From 2b42b93ac6c3befd9f559f8f433b942c9b1af970 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Tue, 10 Feb 2026 13:26:40 -0500 Subject: [PATCH 01/54] dag depth, markers, tests --- .../swagger/ark/v1/indexer.openapi.json | 4 + .../swagger/ark/v1/service.openapi.json | 4 + .../openapi/swagger/ark/v1/types.openapi.json | 4 + api-spec/protobuf/ark/v1/indexer.proto | 1 + api-spec/protobuf/ark/v1/types.proto | 1 + api-spec/protobuf/gen/ark/v1/indexer.pb.go | 13 +- .../protobuf/gen/ark/v1/indexer.pb.rgw.go | 4 +- api-spec/protobuf/gen/ark/v1/types.pb.go | 13 +- internal/core/application/service.go | 11 + internal/core/application/utils.go | 1 + internal/core/domain/marker.go | 31 + internal/core/domain/marker_repo.go | 41 + internal/core/domain/vtxo.go | 2 + internal/core/ports/repo_manager.go | 1 + .../infrastructure/db/postgres/marker_repo.go | 310 ++++++++ .../20260210000000_add_vtxo_depth.down.sql | 22 + .../20260210000000_add_vtxo_depth.up.sql | 23 + .../20260211000000_add_markers.down.sql | 28 + .../20260211000000_add_markers.up.sql | 56 ++ .../db/postgres/sqlc/queries/models.go | 17 + .../db/postgres/sqlc/queries/query.sql.go | 470 +++++++++++- .../infrastructure/db/postgres/sqlc/query.sql | 72 +- .../infrastructure/db/postgres/vtxo_repo.go | 3 + internal/infrastructure/db/service.go | 202 ++++- internal/infrastructure/db/service_test.go | 716 +++++++++++++++++- .../infrastructure/db/sqlite/marker_repo.go | 362 +++++++++ .../20260210000000_add_vtxo_depth.down.sql | 54 ++ .../20260210000000_add_vtxo_depth.up.sql | 22 + .../20260211000000_add_markers.down.sql | 65 ++ .../20260211000000_add_markers.up.sql | 56 ++ .../db/sqlite/sqlc/queries/models.go | 21 +- .../db/sqlite/sqlc/queries/query.sql.go | 518 ++++++++++++- .../infrastructure/db/sqlite/sqlc/query.sql | 72 +- .../infrastructure/db/sqlite/vtxo_repo.go | 7 +- internal/interface/grpc/handlers/indexer.go | 1 + internal/interface/grpc/handlers/parser.go | 1 + 36 files changed, 3174 insertions(+), 55 deletions(-) create mode 100644 internal/core/domain/marker.go create mode 100644 internal/core/domain/marker_repo.go create mode 100644 internal/infrastructure/db/postgres/marker_repo.go create mode 100644 internal/infrastructure/db/postgres/migration/20260210000000_add_vtxo_depth.down.sql create mode 100644 internal/infrastructure/db/postgres/migration/20260210000000_add_vtxo_depth.up.sql create mode 100644 internal/infrastructure/db/postgres/migration/20260211000000_add_markers.down.sql create mode 100644 internal/infrastructure/db/postgres/migration/20260211000000_add_markers.up.sql create mode 100644 internal/infrastructure/db/sqlite/marker_repo.go create mode 100644 internal/infrastructure/db/sqlite/migration/20260210000000_add_vtxo_depth.down.sql create mode 100644 internal/infrastructure/db/sqlite/migration/20260210000000_add_vtxo_depth.up.sql create mode 100644 internal/infrastructure/db/sqlite/migration/20260211000000_add_markers.down.sql create mode 100644 internal/infrastructure/db/sqlite/migration/20260211000000_add_markers.up.sql diff --git a/api-spec/openapi/swagger/ark/v1/indexer.openapi.json b/api-spec/openapi/swagger/ark/v1/indexer.openapi.json index 45c691717..584c70df1 100644 --- a/api-spec/openapi/swagger/ark/v1/indexer.openapi.json +++ b/api-spec/openapi/swagger/ark/v1/indexer.openapi.json @@ -1282,6 +1282,10 @@ "type": "integer", "format": "int64" }, + "depth": { + "type": "integer", + "format": "uint32" + }, "expiresAt": { "type": "integer", "format": "int64" diff --git a/api-spec/openapi/swagger/ark/v1/service.openapi.json b/api-spec/openapi/swagger/ark/v1/service.openapi.json index 67fc9545c..d43056800 100644 --- a/api-spec/openapi/swagger/ark/v1/service.openapi.json +++ b/api-spec/openapi/swagger/ark/v1/service.openapi.json @@ -1436,6 +1436,10 @@ "type": "integer", "format": "int64" }, + "depth": { + "type": "integer", + "format": "uint32" + }, "expiresAt": { "type": "integer", "format": "int64" diff --git a/api-spec/openapi/swagger/ark/v1/types.openapi.json b/api-spec/openapi/swagger/ark/v1/types.openapi.json index 668ec230a..cb43c2d62 100644 --- a/api-spec/openapi/swagger/ark/v1/types.openapi.json +++ b/api-spec/openapi/swagger/ark/v1/types.openapi.json @@ -410,6 +410,10 @@ "type": "integer", "format": "int64" }, + "depth": { + "type": "integer", + "format": "uint32" + }, "expiresAt": { "type": "integer", "format": "int64" diff --git a/api-spec/protobuf/ark/v1/indexer.proto b/api-spec/protobuf/ark/v1/indexer.proto index a74be965b..461c57232 100644 --- a/api-spec/protobuf/ark/v1/indexer.proto +++ b/api-spec/protobuf/ark/v1/indexer.proto @@ -257,6 +257,7 @@ message IndexerVtxo { repeated string commitment_txids = 11; string settled_by = 12; string ark_txid = 13; + uint32 depth = 14; } message IndexerChain { diff --git a/api-spec/protobuf/ark/v1/types.proto b/api-spec/protobuf/ark/v1/types.proto index 58a178fdd..eaf221cb2 100644 --- a/api-spec/protobuf/ark/v1/types.proto +++ b/api-spec/protobuf/ark/v1/types.proto @@ -28,6 +28,7 @@ message Vtxo { string spent_by = 11; string settled_by = 12; string ark_txid = 13; + uint32 depth = 14; } message TxData { diff --git a/api-spec/protobuf/gen/ark/v1/indexer.pb.go b/api-spec/protobuf/gen/ark/v1/indexer.pb.go index 9dfca2ec1..81efee685 100644 --- a/api-spec/protobuf/gen/ark/v1/indexer.pb.go +++ b/api-spec/protobuf/gen/ark/v1/indexer.pb.go @@ -1332,6 +1332,7 @@ type IndexerVtxo struct { CommitmentTxids []string `protobuf:"bytes,11,rep,name=commitment_txids,json=commitmentTxids,proto3" json:"commitment_txids,omitempty"` SettledBy string `protobuf:"bytes,12,opt,name=settled_by,json=settledBy,proto3" json:"settled_by,omitempty"` ArkTxid string `protobuf:"bytes,13,opt,name=ark_txid,json=arkTxid,proto3" json:"ark_txid,omitempty"` + Depth uint32 `protobuf:"varint,14,opt,name=depth,proto3" json:"depth,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1457,6 +1458,13 @@ func (x *IndexerVtxo) GetArkTxid() string { return "" } +func (x *IndexerVtxo) GetDepth() uint32 { + if x != nil { + return x.Depth + } + return 0 +} + type IndexerChain struct { state protoimpl.MessageState `protogen:"open.v1"` Txid string `protobuf:"bytes,1,opt,name=txid,proto3" json:"txid,omitempty"` @@ -2341,7 +2349,7 @@ const file_ark_v1_indexer_proto_rawDesc = "" + "\bchildren\x18\x02 \x03(\v2!.ark.v1.IndexerNode.ChildrenEntryR\bchildren\x1a;\n" + "\rChildrenEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\rR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xb0\x03\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xc6\x03\n" + "\vIndexerVtxo\x123\n" + "\boutpoint\x18\x01 \x01(\v2\x17.ark.v1.IndexerOutpointR\boutpoint\x12\x1d\n" + "\n" + @@ -2360,7 +2368,8 @@ const file_ark_v1_indexer_proto_rawDesc = "" + "\x10commitment_txids\x18\v \x03(\tR\x0fcommitmentTxids\x12\x1d\n" + "\n" + "settled_by\x18\f \x01(\tR\tsettledBy\x12\x19\n" + - "\bark_txid\x18\r \x01(\tR\aarkTxid\"\x8b\x01\n" + + "\bark_txid\x18\r \x01(\tR\aarkTxid\x12\x14\n" + + "\x05depth\x18\x0e \x01(\rR\x05depth\"\x8b\x01\n" + "\fIndexerChain\x12\x12\n" + "\x04txid\x18\x01 \x01(\tR\x04txid\x12\x1d\n" + "\n" + diff --git a/api-spec/protobuf/gen/ark/v1/indexer.pb.rgw.go b/api-spec/protobuf/gen/ark/v1/indexer.pb.rgw.go index 0e7bfee9d..804cfb956 100644 --- a/api-spec/protobuf/gen/ark/v1/indexer.pb.rgw.go +++ b/api-spec/protobuf/gen/ark/v1/indexer.pb.rgw.go @@ -125,7 +125,7 @@ func request_IndexerService_GetConnectors_0(ctx context.Context, marshaler gatew var ( query_params_IndexerService_GetVtxoTree_0 = gateway.QueryParameterParseOptions{ - Filter: trie.New("txid", "vout", "batch_outpoint.txid", "batch_outpoint.vout"), + Filter: trie.New("batch_outpoint.txid", "batch_outpoint.vout", "txid", "vout"), } ) @@ -243,7 +243,7 @@ func request_IndexerService_GetVtxos_0(ctx context.Context, marshaler gateway.Ma var ( query_params_IndexerService_GetVtxoChain_0 = gateway.QueryParameterParseOptions{ - Filter: trie.New("outpoint.txid", "outpoint.vout", "txid", "vout"), + Filter: trie.New("vout", "outpoint.vout", "outpoint.txid", "txid"), } ) diff --git a/api-spec/protobuf/gen/ark/v1/types.pb.go b/api-spec/protobuf/gen/ark/v1/types.pb.go index 5cba0bc1b..2237e6b5c 100644 --- a/api-spec/protobuf/gen/ark/v1/types.pb.go +++ b/api-spec/protobuf/gen/ark/v1/types.pb.go @@ -140,6 +140,7 @@ type Vtxo struct { SpentBy string `protobuf:"bytes,11,opt,name=spent_by,json=spentBy,proto3" json:"spent_by,omitempty"` SettledBy string `protobuf:"bytes,12,opt,name=settled_by,json=settledBy,proto3" json:"settled_by,omitempty"` ArkTxid string `protobuf:"bytes,13,opt,name=ark_txid,json=arkTxid,proto3" json:"ark_txid,omitempty"` + Depth uint32 `protobuf:"varint,14,opt,name=depth,proto3" json:"depth,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -265,6 +266,13 @@ func (x *Vtxo) GetArkTxid() string { return "" } +func (x *Vtxo) GetDepth() uint32 { + if x != nil { + return x.Depth + } + return 0 +} + type TxData struct { state protoimpl.MessageState `protogen:"open.v1"` Txid string `protobuf:"bytes,1,opt,name=txid,proto3" json:"txid,omitempty"` @@ -1512,7 +1520,7 @@ const file_ark_v1_types_proto_rawDesc = "" + "\x04vout\x18\x02 \x01(\rR\x04vout\"l\n" + "\x05Input\x12,\n" + "\boutpoint\x18\x01 \x01(\v2\x10.ark.v1.OutpointR\boutpoint\x125\n" + - "\ftaproot_tree\x18\x02 \x01(\v2\x12.ark.v1.TapscriptsR\vtaprootTree\"\xa2\x03\n" + + "\ftaproot_tree\x18\x02 \x01(\v2\x12.ark.v1.TapscriptsR\vtaprootTree\"\xb8\x03\n" + "\x04Vtxo\x12,\n" + "\boutpoint\x18\x01 \x01(\v2\x10.ark.v1.OutpointR\boutpoint\x12\x16\n" + "\x06amount\x18\x02 \x01(\x04R\x06amount\x12\x16\n" + @@ -1531,7 +1539,8 @@ const file_ark_v1_types_proto_rawDesc = "" + "\bspent_by\x18\v \x01(\tR\aspentBy\x12\x1d\n" + "\n" + "settled_by\x18\f \x01(\tR\tsettledBy\x12\x19\n" + - "\bark_txid\x18\r \x01(\tR\aarkTxid\",\n" + + "\bark_txid\x18\r \x01(\tR\aarkTxid\x12\x14\n" + + "\x05depth\x18\x0e \x01(\rR\x05depth\",\n" + "\x06TxData\x12\x12\n" + "\x04txid\x18\x01 \x01(\tR\x04txid\x12\x0e\n" + "\x02tx\x18\x02 \x01(\tR\x02tx\"\xbe\x02\n" + diff --git a/internal/core/application/service.go b/internal/core/application/service.go index 04c795813..272eb6e9b 100644 --- a/internal/core/application/service.go +++ b/internal/core/application/service.go @@ -321,6 +321,17 @@ func NewService( return } + // Calculate depth for new vtxos: max(parent depths) + 1 + var maxDepth uint32 + for _, v := range spentVtxos { + if v.Depth > maxDepth { + maxDepth = v.Depth + } + } + for i := range newVtxos { + newVtxos[i].Depth = maxDepth + 1 + } + checkpointTxsByOutpoint := make(map[string]TxData) for txid, tx := range offchainTx.CheckpointTxs { // nolint diff --git a/internal/core/application/utils.go b/internal/core/application/utils.go index c98bb4f93..bd03d2663 100644 --- a/internal/core/application/utils.go +++ b/internal/core/application/utils.go @@ -242,6 +242,7 @@ func getNewVtxosFromRound(round *domain.Round) []domain.Vtxo { RootCommitmentTxid: round.CommitmentTxid, CreatedAt: createdAt, ExpiresAt: expireAt, + Depth: 0, // new vtxo from batch starts at depth 0 }) } } diff --git a/internal/core/domain/marker.go b/internal/core/domain/marker.go new file mode 100644 index 000000000..9a540523a --- /dev/null +++ b/internal/core/domain/marker.go @@ -0,0 +1,31 @@ +package domain + +// MarkerInterval is the depth interval at which markers are created. +// VTXOs at depth 0, 100, 200, etc. create new markers. +const MarkerInterval = 100 + +// Marker represents a DAG traversal checkpoint created at regular depth intervals. +// Markers enable compressed traversal of the VTXO chain by allowing jumps of +// MarkerInterval depths instead of traversing each VTXO individually. +type Marker struct { + // ID is the unique identifier for this marker (typically the VTXO outpoint) + ID string + // Depth is the chain depth at which this marker exists (0, 100, 200, ...) + Depth uint32 + // ParentMarkerIDs is a list of marker IDs that this marker descends from + ParentMarkerIDs []string +} + +// IsAtMarkerBoundary returns true if the given depth is at a marker boundary. +func IsAtMarkerBoundary(depth uint32) bool { + return depth%MarkerInterval == 0 +} + +// SweptMarker records when a marker (and all VTXOs it covers) was swept. +// This is an append-only table that enables efficient bulk sweep operations. +type SweptMarker struct { + // MarkerID is the ID of the marker that was swept + MarkerID string + // SweptAt is the Unix timestamp (milliseconds) when the marker was swept + SweptAt int64 +} diff --git a/internal/core/domain/marker_repo.go b/internal/core/domain/marker_repo.go new file mode 100644 index 000000000..b05b26be4 --- /dev/null +++ b/internal/core/domain/marker_repo.go @@ -0,0 +1,41 @@ +package domain + +import "context" + +type MarkerRepository interface { + // AddMarker creates or updates a marker + AddMarker(ctx context.Context, marker Marker) error + // GetMarker retrieves a marker by ID + GetMarker(ctx context.Context, id string) (*Marker, error) + // GetMarkersByDepth retrieves all markers at a specific depth + GetMarkersByDepth(ctx context.Context, depth uint32) ([]Marker, error) + // GetMarkersByDepthRange retrieves all markers within a depth range + GetMarkersByDepthRange(ctx context.Context, minDepth, maxDepth uint32) ([]Marker, error) + // GetMarkersByIds retrieves markers by their IDs + GetMarkersByIds(ctx context.Context, ids []string) ([]Marker, error) + + // SweepMarker marks a marker as swept at the given timestamp + SweepMarker(ctx context.Context, markerID string, sweptAt int64) error + // IsMarkerSwept checks if a marker has been swept + IsMarkerSwept(ctx context.Context, markerID string) (bool, error) + // GetSweptMarkers retrieves swept marker records for the given marker IDs + GetSweptMarkers(ctx context.Context, markerIDs []string) ([]SweptMarker, error) + + // UpdateVtxoMarker updates the marker_id for a VTXO + UpdateVtxoMarker(ctx context.Context, outpoint Outpoint, markerID string) error + // GetVtxosByMarker retrieves all VTXOs associated with a marker + GetVtxosByMarker(ctx context.Context, markerID string) ([]Vtxo, error) + // SweepVtxosByMarker marks all VTXOs with the given marker_id as swept + // Returns the number of VTXOs that were swept (not already swept) + SweepVtxosByMarker(ctx context.Context, markerID string) (int64, error) + + // Chain traversal methods for GetVtxoChain optimization + // GetVtxosByDepthRange retrieves VTXOs within a depth range + GetVtxosByDepthRange(ctx context.Context, minDepth, maxDepth uint32) ([]Vtxo, error) + // GetVtxosByArkTxid retrieves VTXOs created by a specific ark tx + GetVtxosByArkTxid(ctx context.Context, arkTxid string) ([]Vtxo, error) + // GetVtxoChainByMarkers retrieves VTXOs that have markers in the given list + GetVtxoChainByMarkers(ctx context.Context, markerIDs []string) ([]Vtxo, error) + + Close() +} diff --git a/internal/core/domain/vtxo.go b/internal/core/domain/vtxo.go index 8816929c7..02641fa5b 100644 --- a/internal/core/domain/vtxo.go +++ b/internal/core/domain/vtxo.go @@ -50,6 +50,8 @@ type Vtxo struct { Preconfirmed bool ExpiresAt int64 CreatedAt int64 + Depth uint32 // chain depth: 0 for vtxos from batch, increments on each chain + MarkerID string // marker ID for DAG traversal optimization } func (v Vtxo) String() string { diff --git a/internal/core/ports/repo_manager.go b/internal/core/ports/repo_manager.go index 1e69fb4ba..380e6e5c7 100644 --- a/internal/core/ports/repo_manager.go +++ b/internal/core/ports/repo_manager.go @@ -6,6 +6,7 @@ type RepoManager interface { Events() domain.EventRepository Rounds() domain.RoundRepository Vtxos() domain.VtxoRepository + Markers() domain.MarkerRepository ScheduledSession() domain.ScheduledSessionRepo OffchainTxs() domain.OffchainTxRepository Convictions() domain.ConvictionRepository diff --git a/internal/infrastructure/db/postgres/marker_repo.go b/internal/infrastructure/db/postgres/marker_repo.go new file mode 100644 index 000000000..d8c8da7ca --- /dev/null +++ b/internal/infrastructure/db/postgres/marker_repo.go @@ -0,0 +1,310 @@ +package pgdb + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "fmt" + + "github.com/arkade-os/arkd/internal/core/domain" + "github.com/arkade-os/arkd/internal/infrastructure/db/postgres/sqlc/queries" + "github.com/sqlc-dev/pqtype" +) + +type markerRepository struct { + db *sql.DB + querier *queries.Queries +} + +func NewMarkerRepository(config ...interface{}) (domain.MarkerRepository, error) { + if len(config) != 1 { + return nil, fmt.Errorf("invalid config") + } + db, ok := config[0].(*sql.DB) + if !ok { + return nil, fmt.Errorf("cannot open marker repository: invalid config") + } + + return &markerRepository{ + db: db, + querier: queries.New(db), + }, nil +} + +func (m *markerRepository) Close() { + _ = m.db.Close() +} + +func (m *markerRepository) AddMarker(ctx context.Context, marker domain.Marker) error { + parentMarkersJSON, err := json.Marshal(marker.ParentMarkerIDs) + if err != nil { + return fmt.Errorf("failed to marshal parent markers: %w", err) + } + + return m.querier.UpsertMarker(ctx, queries.UpsertMarkerParams{ + ID: marker.ID, + Depth: int32(marker.Depth), + ParentMarkers: pqtype.NullRawMessage{ + RawMessage: parentMarkersJSON, + Valid: true, + }, + }) +} + +func (m *markerRepository) GetMarker(ctx context.Context, id string) (*domain.Marker, error) { + row, err := m.querier.SelectMarker(ctx, id) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, err + } + + marker, err := rowToMarker(row) + if err != nil { + return nil, err + } + return &marker, nil +} + +func (m *markerRepository) GetMarkersByDepth(ctx context.Context, depth uint32) ([]domain.Marker, error) { + rows, err := m.querier.SelectMarkersByDepth(ctx, int32(depth)) + if err != nil { + return nil, err + } + + markers := make([]domain.Marker, 0, len(rows)) + for _, row := range rows { + marker, err := rowToMarker(row) + if err != nil { + return nil, err + } + markers = append(markers, marker) + } + return markers, nil +} + +func (m *markerRepository) GetMarkersByDepthRange(ctx context.Context, minDepth, maxDepth uint32) ([]domain.Marker, error) { + rows, err := m.querier.SelectMarkersByDepthRange(ctx, queries.SelectMarkersByDepthRangeParams{ + MinDepth: int32(minDepth), + MaxDepth: int32(maxDepth), + }) + if err != nil { + return nil, err + } + + markers := make([]domain.Marker, 0, len(rows)) + for _, row := range rows { + marker, err := rowToMarker(row) + if err != nil { + return nil, err + } + markers = append(markers, marker) + } + return markers, nil +} + +func (m *markerRepository) GetMarkersByIds(ctx context.Context, ids []string) ([]domain.Marker, error) { + if len(ids) == 0 { + return nil, nil + } + + rows, err := m.querier.SelectMarkersByIds(ctx, ids) + if err != nil { + return nil, err + } + + markers := make([]domain.Marker, 0, len(rows)) + for _, row := range rows { + marker, err := rowToMarker(row) + if err != nil { + return nil, err + } + markers = append(markers, marker) + } + return markers, nil +} + +func (m *markerRepository) SweepMarker(ctx context.Context, markerID string, sweptAt int64) error { + return m.querier.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ + MarkerID: markerID, + SweptAt: sweptAt, + }) +} + +func (m *markerRepository) IsMarkerSwept(ctx context.Context, markerID string) (bool, error) { + result, err := m.querier.IsMarkerSwept(ctx, markerID) + if err != nil { + return false, err + } + return result, nil +} + +func (m *markerRepository) GetSweptMarkers(ctx context.Context, markerIDs []string) ([]domain.SweptMarker, error) { + if len(markerIDs) == 0 { + return nil, nil + } + + rows, err := m.querier.SelectSweptMarkersByIds(ctx, markerIDs) + if err != nil { + return nil, err + } + + sweptMarkers := make([]domain.SweptMarker, 0, len(rows)) + for _, row := range rows { + sweptMarkers = append(sweptMarkers, domain.SweptMarker{ + MarkerID: row.MarkerID, + SweptAt: row.SweptAt, + }) + } + return sweptMarkers, nil +} + +func (m *markerRepository) UpdateVtxoMarker(ctx context.Context, outpoint domain.Outpoint, markerID string) error { + return m.querier.UpdateVtxoMarkerId(ctx, queries.UpdateVtxoMarkerIdParams{ + MarkerID: sql.NullString{String: markerID, Valid: len(markerID) > 0}, + Txid: outpoint.Txid, + Vout: int32(outpoint.VOut), + }) +} + +func (m *markerRepository) GetVtxosByMarker(ctx context.Context, markerID string) ([]domain.Vtxo, error) { + rows, err := m.querier.SelectVtxosByMarkerId(ctx, sql.NullString{String: markerID, Valid: len(markerID) > 0}) + if err != nil { + return nil, err + } + + vtxos := make([]domain.Vtxo, 0, len(rows)) + for _, row := range rows { + vtxos = append(vtxos, rowToVtxoFromMarkerQuery(row)) + } + return vtxos, nil +} + +func (m *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID string) (int64, error) { + return m.querier.SweepVtxosByMarkerId(ctx, sql.NullString{String: markerID, Valid: len(markerID) > 0}) +} + +func (m *markerRepository) GetVtxosByDepthRange(ctx context.Context, minDepth, maxDepth uint32) ([]domain.Vtxo, error) { + rows, err := m.querier.SelectVtxosByDepthRange(ctx, queries.SelectVtxosByDepthRangeParams{ + MinDepth: int32(minDepth), + MaxDepth: int32(maxDepth), + }) + if err != nil { + return nil, err + } + + vtxos := make([]domain.Vtxo, 0, len(rows)) + for _, row := range rows { + vtxos = append(vtxos, rowToVtxoFromVtxoVw(row)) + } + return vtxos, nil +} + +func (m *markerRepository) GetVtxosByArkTxid(ctx context.Context, arkTxid string) ([]domain.Vtxo, error) { + rows, err := m.querier.SelectVtxosByArkTxid(ctx, arkTxid) + if err != nil { + return nil, err + } + + vtxos := make([]domain.Vtxo, 0, len(rows)) + for _, row := range rows { + vtxos = append(vtxos, rowToVtxoFromVtxoVw(row)) + } + return vtxos, nil +} + +func (m *markerRepository) GetVtxoChainByMarkers(ctx context.Context, markerIDs []string) ([]domain.Vtxo, error) { + if len(markerIDs) == 0 { + return nil, nil + } + + rows, err := m.querier.SelectVtxoChainByMarker(ctx, markerIDs) + if err != nil { + return nil, err + } + + vtxos := make([]domain.Vtxo, 0, len(rows)) + for _, row := range rows { + vtxos = append(vtxos, rowToVtxoFromVtxoVw(row)) + } + return vtxos, nil +} + +// rowToVtxoFromVtxoVw converts a VtxoVw (used in multiple query results) to domain.Vtxo +func rowToVtxoFromVtxoVw(row queries.VtxoVw) domain.Vtxo { + return domain.Vtxo{ + Outpoint: domain.Outpoint{ + Txid: row.Txid, + VOut: uint32(row.Vout), + }, + Amount: uint64(row.Amount), + PubKey: row.Pubkey, + RootCommitmentTxid: row.CommitmentTxid, + CommitmentTxids: parseCommitments(row.Commitments, []byte(",")), + SettledBy: row.SettledBy.String, + ArkTxid: row.ArkTxid.String, + SpentBy: row.SpentBy.String, + Spent: row.Spent, + Unrolled: row.Unrolled, + Swept: row.Swept, + Preconfirmed: row.Preconfirmed, + ExpiresAt: row.ExpiresAt, + CreatedAt: row.CreatedAt, + Depth: uint32(row.Depth), + MarkerID: row.MarkerID.String, + } +} + +func rowToMarker(row queries.Marker) (domain.Marker, error) { + var parentMarkerIDs []string + if row.ParentMarkers.Valid && len(row.ParentMarkers.RawMessage) > 0 { + if err := json.Unmarshal(row.ParentMarkers.RawMessage, &parentMarkerIDs); err != nil { + return domain.Marker{}, fmt.Errorf("failed to unmarshal parent markers: %w", err) + } + } + + return domain.Marker{ + ID: row.ID, + Depth: uint32(row.Depth), + ParentMarkerIDs: parentMarkerIDs, + }, nil +} + +func rowToVtxoFromMarkerQuery(row queries.SelectVtxosByMarkerIdRow) domain.Vtxo { + return domain.Vtxo{ + Outpoint: domain.Outpoint{ + Txid: row.VtxoVw.Txid, + VOut: uint32(row.VtxoVw.Vout), + }, + Amount: uint64(row.VtxoVw.Amount), + PubKey: row.VtxoVw.Pubkey, + RootCommitmentTxid: row.VtxoVw.CommitmentTxid, + CommitmentTxids: parseCommitments(row.VtxoVw.Commitments, []byte(",")), + SettledBy: row.VtxoVw.SettledBy.String, + ArkTxid: row.VtxoVw.ArkTxid.String, + SpentBy: row.VtxoVw.SpentBy.String, + Spent: row.VtxoVw.Spent, + Unrolled: row.VtxoVw.Unrolled, + Swept: row.VtxoVw.Swept, + Preconfirmed: row.VtxoVw.Preconfirmed, + ExpiresAt: row.VtxoVw.ExpiresAt, + CreatedAt: row.VtxoVw.CreatedAt, + Depth: uint32(row.VtxoVw.Depth), + MarkerID: row.VtxoVw.MarkerID.String, + } +} + +// parseCommitmentsBytes is a local helper function that handles nil slices +func parseCommitmentsBytes(commitments []byte, separator []byte) []string { + if len(commitments) == 0 { + return nil + } + parts := bytes.Split(commitments, separator) + commitmentsStr := make([]string, 0, len(parts)) + for _, p := range parts { + commitmentsStr = append(commitmentsStr, string(p)) + } + return commitmentsStr +} diff --git a/internal/infrastructure/db/postgres/migration/20260210000000_add_vtxo_depth.down.sql b/internal/infrastructure/db/postgres/migration/20260210000000_add_vtxo_depth.down.sql new file mode 100644 index 000000000..94fd10573 --- /dev/null +++ b/internal/infrastructure/db/postgres/migration/20260210000000_add_vtxo_depth.down.sql @@ -0,0 +1,22 @@ +-- Recreate views without depth column +DROP VIEW IF EXISTS intent_with_inputs_vw; +DROP VIEW IF EXISTS vtxo_vw; + +ALTER TABLE vtxo DROP COLUMN IF EXISTS depth; + +CREATE VIEW vtxo_vw AS +SELECT v.*, string_agg(vc.commitment_txid, ',') AS commitments +FROM vtxo v +LEFT JOIN vtxo_commitment_txid vc +ON v.txid = vc.vtxo_txid AND v.vout = vc.vtxo_vout +GROUP BY v.txid, v.vout; + +CREATE VIEW intent_with_inputs_vw AS +SELECT vtxo_vw.*, + intent.id, + intent.round_id, + intent.proof, + intent.message +FROM intent +LEFT OUTER JOIN vtxo_vw +ON intent.id = vtxo_vw.intent_id; diff --git a/internal/infrastructure/db/postgres/migration/20260210000000_add_vtxo_depth.up.sql b/internal/infrastructure/db/postgres/migration/20260210000000_add_vtxo_depth.up.sql new file mode 100644 index 000000000..2585dfb5f --- /dev/null +++ b/internal/infrastructure/db/postgres/migration/20260210000000_add_vtxo_depth.up.sql @@ -0,0 +1,23 @@ +ALTER TABLE vtxo +ADD COLUMN IF NOT EXISTS depth INTEGER NOT NULL DEFAULT 0; + +-- Recreate views to include the new depth column +DROP VIEW IF EXISTS intent_with_inputs_vw; +DROP VIEW IF EXISTS vtxo_vw; + +CREATE VIEW vtxo_vw AS +SELECT v.*, string_agg(vc.commitment_txid, ',') AS commitments +FROM vtxo v +LEFT JOIN vtxo_commitment_txid vc +ON v.txid = vc.vtxo_txid AND v.vout = vc.vtxo_vout +GROUP BY v.txid, v.vout; + +CREATE VIEW intent_with_inputs_vw AS +SELECT vtxo_vw.*, + intent.id, + intent.round_id, + intent.proof, + intent.message +FROM intent +LEFT OUTER JOIN vtxo_vw +ON intent.id = vtxo_vw.intent_id; diff --git a/internal/infrastructure/db/postgres/migration/20260211000000_add_markers.down.sql b/internal/infrastructure/db/postgres/migration/20260211000000_add_markers.down.sql new file mode 100644 index 000000000..43602d209 --- /dev/null +++ b/internal/infrastructure/db/postgres/migration/20260211000000_add_markers.down.sql @@ -0,0 +1,28 @@ +-- Drop marker_id column from vtxo +DROP INDEX IF EXISTS idx_vtxo_marker_id; +ALTER TABLE vtxo DROP COLUMN IF EXISTS marker_id; + +-- Drop marker tables +DROP TABLE IF EXISTS swept_marker; +DROP TABLE IF EXISTS marker; + +-- Recreate views without marker_id +DROP VIEW IF EXISTS intent_with_inputs_vw; +DROP VIEW IF EXISTS vtxo_vw; + +CREATE VIEW vtxo_vw AS +SELECT v.*, string_agg(vc.commitment_txid, ',') AS commitments +FROM vtxo v +LEFT JOIN vtxo_commitment_txid vc +ON v.txid = vc.vtxo_txid AND v.vout = vc.vtxo_vout +GROUP BY v.txid, v.vout; + +CREATE VIEW intent_with_inputs_vw AS +SELECT vtxo_vw.*, + intent.id, + intent.round_id, + intent.proof, + intent.message +FROM intent +LEFT OUTER JOIN vtxo_vw +ON intent.id = vtxo_vw.intent_id; diff --git a/internal/infrastructure/db/postgres/migration/20260211000000_add_markers.up.sql b/internal/infrastructure/db/postgres/migration/20260211000000_add_markers.up.sql new file mode 100644 index 000000000..d1bda68b3 --- /dev/null +++ b/internal/infrastructure/db/postgres/migration/20260211000000_add_markers.up.sql @@ -0,0 +1,56 @@ +-- Create markers table +CREATE TABLE IF NOT EXISTS marker ( + id TEXT PRIMARY KEY, + depth INTEGER NOT NULL, + parent_markers JSONB -- JSON array of parent marker IDs +); +CREATE INDEX IF NOT EXISTS idx_marker_depth ON marker(depth); + +-- Create swept_markers table (append-only) +CREATE TABLE IF NOT EXISTS swept_marker ( + marker_id TEXT PRIMARY KEY REFERENCES marker(id), + swept_at BIGINT NOT NULL +); + +-- Add marker_id column to vtxo table +ALTER TABLE vtxo ADD COLUMN marker_id TEXT REFERENCES marker(id); +CREATE INDEX IF NOT EXISTS idx_vtxo_marker_id ON vtxo(marker_id); + +-- Recreate views to include the new marker_id column +DROP VIEW IF EXISTS intent_with_inputs_vw; +DROP VIEW IF EXISTS vtxo_vw; + +CREATE VIEW vtxo_vw AS +SELECT v.*, string_agg(vc.commitment_txid, ',') AS commitments +FROM vtxo v +LEFT JOIN vtxo_commitment_txid vc +ON v.txid = vc.vtxo_txid AND v.vout = vc.vtxo_vout +GROUP BY v.txid, v.vout; + +CREATE VIEW intent_with_inputs_vw AS +SELECT vtxo_vw.*, + intent.id, + intent.round_id, + intent.proof, + intent.message +FROM intent +LEFT OUTER JOIN vtxo_vw +ON intent.id = vtxo_vw.intent_id; + +-- Backfill markers for existing VTXOs based on their depth +-- VTXOs at depth 0, 100, 200, ... get their own markers +-- Other VTXOs will have their marker_id set during PR 5 (marker assignment logic) + +-- First, create markers for all existing VTXOs at marker boundary depths (depth % 100 == 0) +INSERT INTO marker (id, depth, parent_markers) +SELECT + v.txid || ':' || v.vout, -- Use VTXO outpoint as marker ID + v.depth, + '[]'::jsonb -- Empty parent markers for initial backfill +FROM vtxo v +WHERE v.depth % 100 = 0; + +-- Assign marker_id to VTXOs at boundary depths +UPDATE vtxo +SET marker_id = txid || ':' || vout +WHERE depth % 100 = 0; diff --git a/internal/infrastructure/db/postgres/sqlc/queries/models.go b/internal/infrastructure/db/postgres/sqlc/queries/models.go index a365f9500..d9e276c09 100644 --- a/internal/infrastructure/db/postgres/sqlc/queries/models.go +++ b/internal/infrastructure/db/postgres/sqlc/queries/models.go @@ -64,6 +64,8 @@ type IntentWithInputsVw struct { ArkTxid sql.NullString IntentID sql.NullString UpdatedAt sql.NullInt64 + Depth sql.NullInt32 + MarkerID sql.NullString Commitments []byte ID sql.NullString RoundID sql.NullString @@ -82,6 +84,12 @@ type IntentWithReceiversVw struct { Message sql.NullString } +type Marker struct { + ID string + Depth int32 + ParentMarkers pqtype.NullRawMessage +} + type MarketHour struct { ID int32 StartTime int64 @@ -186,6 +194,11 @@ type ScheduledSession struct { UpdatedAt int64 } +type SweptMarker struct { + MarkerID string + SweptAt int64 +} + type Tx struct { Txid string Tx string @@ -212,6 +225,8 @@ type Vtxo struct { ArkTxid sql.NullString IntentID sql.NullString UpdatedAt int64 + Depth int32 + MarkerID sql.NullString } type VtxoCommitmentTxid struct { @@ -237,5 +252,7 @@ type VtxoVw struct { ArkTxid sql.NullString IntentID sql.NullString UpdatedAt int64 + Depth int32 + MarkerID sql.NullString Commitments []byte } diff --git a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go index 0e9f723f9..2aabead78 100644 --- a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go @@ -86,6 +86,22 @@ func (q *Queries) ClearScheduledSession(ctx context.Context) error { return err } +const insertSweptMarker = `-- name: InsertSweptMarker :exec +INSERT INTO swept_marker (marker_id, swept_at) +VALUES ($1, $2) +ON CONFLICT(marker_id) DO NOTHING +` + +type InsertSweptMarkerParams struct { + MarkerID string + SweptAt int64 +} + +func (q *Queries) InsertSweptMarker(ctx context.Context, arg InsertSweptMarkerParams) error { + _, err := q.db.ExecContext(ctx, insertSweptMarker, arg.MarkerID, arg.SweptAt) + return err +} + const insertVtxoCommitmentTxid = `-- name: InsertVtxoCommitmentTxid :exec INSERT INTO vtxo_commitment_txid (vtxo_txid, vtxo_vout, commitment_txid) VALUES ($1, $2, $3) @@ -102,6 +118,17 @@ func (q *Queries) InsertVtxoCommitmentTxid(ctx context.Context, arg InsertVtxoCo return err } +const isMarkerSwept = `-- name: IsMarkerSwept :one +SELECT EXISTS(SELECT 1 FROM swept_marker WHERE marker_id = $1) AS is_swept +` + +func (q *Queries) IsMarkerSwept(ctx context.Context, markerID string) (bool, error) { + row := q.db.QueryRowContext(ctx, isMarkerSwept, markerID) + var is_swept bool + err := row.Scan(&is_swept) + return is_swept, err +} + const selectActiveScriptConvictions = `-- name: SelectActiveScriptConvictions :many SELECT id, type, created_at, expires_at, crime_type, crime_round_id, crime_reason, pardoned, script FROM conviction WHERE script = $1 @@ -176,7 +203,7 @@ func (q *Queries) SelectAllRoundIds(ctx context.Context) ([]string, error) { } const selectAllVtxos = `-- name: SelectAllVtxos :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.commitments FROM vtxo_vw +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw ` type SelectAllVtxosRow struct { @@ -209,6 +236,8 @@ func (q *Queries) SelectAllVtxos(ctx context.Context) ([]SelectAllVtxosRow, erro &i.VtxoVw.ArkTxid, &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -412,8 +441,105 @@ func (q *Queries) SelectLatestScheduledSession(ctx context.Context) (ScheduledSe return i, err } +const selectMarker = `-- name: SelectMarker :one +SELECT id, depth, parent_markers FROM marker WHERE id = $1 +` + +func (q *Queries) SelectMarker(ctx context.Context, id string) (Marker, error) { + row := q.db.QueryRowContext(ctx, selectMarker, id) + var i Marker + err := row.Scan(&i.ID, &i.Depth, &i.ParentMarkers) + return i, err +} + +const selectMarkersByDepth = `-- name: SelectMarkersByDepth :many +SELECT id, depth, parent_markers FROM marker WHERE depth = $1 +` + +func (q *Queries) SelectMarkersByDepth(ctx context.Context, depth int32) ([]Marker, error) { + rows, err := q.db.QueryContext(ctx, selectMarkersByDepth, depth) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Marker + for rows.Next() { + var i Marker + if err := rows.Scan(&i.ID, &i.Depth, &i.ParentMarkers); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const selectMarkersByDepthRange = `-- name: SelectMarkersByDepthRange :many +SELECT id, depth, parent_markers FROM marker WHERE depth >= $1 AND depth <= $2 ORDER BY depth +` + +type SelectMarkersByDepthRangeParams struct { + MinDepth int32 + MaxDepth int32 +} + +func (q *Queries) SelectMarkersByDepthRange(ctx context.Context, arg SelectMarkersByDepthRangeParams) ([]Marker, error) { + rows, err := q.db.QueryContext(ctx, selectMarkersByDepthRange, arg.MinDepth, arg.MaxDepth) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Marker + for rows.Next() { + var i Marker + if err := rows.Scan(&i.ID, &i.Depth, &i.ParentMarkers); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const selectMarkersByIds = `-- name: SelectMarkersByIds :many +SELECT id, depth, parent_markers FROM marker WHERE id = ANY($1::text[]) +` + +func (q *Queries) SelectMarkersByIds(ctx context.Context, ids []string) ([]Marker, error) { + rows, err := q.db.QueryContext(ctx, selectMarkersByIds, pq.Array(ids)) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Marker + for rows.Next() { + var i Marker + if err := rows.Scan(&i.ID, &i.Depth, &i.ParentMarkers); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const selectNotUnrolledVtxos = `-- name: SelectNotUnrolledVtxos :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false ` type SelectNotUnrolledVtxosRow struct { @@ -446,6 +572,8 @@ func (q *Queries) SelectNotUnrolledVtxos(ctx context.Context) ([]SelectNotUnroll &i.VtxoVw.ArkTxid, &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -462,7 +590,7 @@ func (q *Queries) SelectNotUnrolledVtxos(ctx context.Context) ([]SelectNotUnroll } const selectNotUnrolledVtxosWithPubkey = `-- name: SelectNotUnrolledVtxosWithPubkey :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false AND pubkey = $1 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false AND pubkey = $1 ` type SelectNotUnrolledVtxosWithPubkeyRow struct { @@ -495,6 +623,8 @@ func (q *Queries) SelectNotUnrolledVtxosWithPubkey(ctx context.Context, pubkey s &i.VtxoVw.ArkTxid, &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -555,7 +685,7 @@ func (q *Queries) SelectOffchainTx(ctx context.Context, txid string) ([]SelectOf } const selectPendingSpentVtxo = `-- name: SelectPendingSpentVtxo :one -SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.commitments +SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.marker_id, v.commitments FROM vtxo_vw v WHERE v.txid = $1 AND v.vout = $2 AND v.spent = TRUE AND v.unrolled = FALSE and COALESCE(v.settled_by, '') = '' @@ -589,13 +719,15 @@ func (q *Queries) SelectPendingSpentVtxo(ctx context.Context, arg SelectPendingS &i.ArkTxid, &i.IntentID, &i.UpdatedAt, + &i.Depth, + &i.MarkerID, &i.Commitments, ) return i, err } const selectPendingSpentVtxosWithPubkeys = `-- name: SelectPendingSpentVtxosWithPubkeys :many -SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.commitments +SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.marker_id, v.commitments FROM vtxo_vw v WHERE v.spent = TRUE AND v.unrolled = FALSE and COALESCE(v.settled_by, '') = '' AND v.pubkey = ANY($1::varchar[]) @@ -638,6 +770,8 @@ func (q *Queries) SelectPendingSpentVtxosWithPubkeys(ctx context.Context, arg Se &i.ArkTxid, &i.IntentID, &i.UpdatedAt, + &i.Depth, + &i.MarkerID, &i.Commitments, ); err != nil { return nil, err @@ -937,7 +1071,7 @@ func (q *Queries) SelectRoundVtxoTree(ctx context.Context, txid string) ([]Tx, e } const selectRoundVtxoTreeLeaves = `-- name: SelectRoundVtxoTreeLeaves :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.commitments FROM vtxo_vw WHERE commitment_txid = $1 AND preconfirmed = false +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE commitment_txid = $1 AND preconfirmed = false ` type SelectRoundVtxoTreeLeavesRow struct { @@ -970,6 +1104,8 @@ func (q *Queries) SelectRoundVtxoTreeLeaves(ctx context.Context, commitmentTxid &i.VtxoVw.ArkTxid, &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -990,7 +1126,7 @@ SELECT round.id, round.starting_timestamp, round.ending_timestamp, round.ended, round_intents_vw.id, round_intents_vw.round_id, round_intents_vw.proof, round_intents_vw.message, round_txs_vw.txid, round_txs_vw.tx, round_txs_vw.round_id, round_txs_vw.type, round_txs_vw.position, round_txs_vw.children, intent_with_receivers_vw.intent_id, intent_with_receivers_vw.pubkey, intent_with_receivers_vw.onchain_address, intent_with_receivers_vw.amount, intent_with_receivers_vw.id, intent_with_receivers_vw.round_id, intent_with_receivers_vw.proof, intent_with_receivers_vw.message, - intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message + intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.marker_id, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message FROM round LEFT OUTER JOIN round_intents_vw ON round.id=round_intents_vw.round_id LEFT OUTER JOIN round_txs_vw ON round.id=round_txs_vw.round_id @@ -1062,6 +1198,8 @@ func (q *Queries) SelectRoundWithId(ctx context.Context, id string) ([]SelectRou &i.IntentWithInputsVw.ArkTxid, &i.IntentWithInputsVw.IntentID, &i.IntentWithInputsVw.UpdatedAt, + &i.IntentWithInputsVw.Depth, + &i.IntentWithInputsVw.MarkerID, &i.IntentWithInputsVw.Commitments, &i.IntentWithInputsVw.ID, &i.IntentWithInputsVw.RoundID, @@ -1086,7 +1224,7 @@ SELECT round.id, round.starting_timestamp, round.ending_timestamp, round.ended, round_intents_vw.id, round_intents_vw.round_id, round_intents_vw.proof, round_intents_vw.message, round_txs_vw.txid, round_txs_vw.tx, round_txs_vw.round_id, round_txs_vw.type, round_txs_vw.position, round_txs_vw.children, intent_with_receivers_vw.intent_id, intent_with_receivers_vw.pubkey, intent_with_receivers_vw.onchain_address, intent_with_receivers_vw.amount, intent_with_receivers_vw.id, intent_with_receivers_vw.round_id, intent_with_receivers_vw.proof, intent_with_receivers_vw.message, - intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message + intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.marker_id, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message FROM round LEFT OUTER JOIN round_intents_vw ON round.id=round_intents_vw.round_id LEFT OUTER JOIN round_txs_vw ON round.id=round_txs_vw.round_id @@ -1160,6 +1298,8 @@ func (q *Queries) SelectRoundWithTxid(ctx context.Context, txid string) ([]Selec &i.IntentWithInputsVw.ArkTxid, &i.IntentWithInputsVw.IntentID, &i.IntentWithInputsVw.UpdatedAt, + &i.IntentWithInputsVw.Depth, + &i.IntentWithInputsVw.MarkerID, &i.IntentWithInputsVw.Commitments, &i.IntentWithInputsVw.ID, &i.IntentWithInputsVw.RoundID, @@ -1239,7 +1379,7 @@ func (q *Queries) SelectSweepableRounds(ctx context.Context) ([]string, error) { } const selectSweepableUnrolledVtxos = `-- name: SelectSweepableUnrolledVtxos :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.commitments FROM vtxo_vw WHERE spent = true AND unrolled = true AND swept = false AND COALESCE(settled_by, '') = '' +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE spent = true AND unrolled = true AND swept = false AND COALESCE(settled_by, '') = '' ` type SelectSweepableUnrolledVtxosRow struct { @@ -1272,6 +1412,8 @@ func (q *Queries) SelectSweepableUnrolledVtxos(ctx context.Context) ([]SelectSwe &i.VtxoVw.ArkTxid, &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -1323,6 +1465,44 @@ func (q *Queries) SelectSweepableVtxoOutpointsByCommitmentTxid(ctx context.Conte return items, nil } +const selectSweptMarker = `-- name: SelectSweptMarker :one +SELECT marker_id, swept_at FROM swept_marker WHERE marker_id = $1 +` + +func (q *Queries) SelectSweptMarker(ctx context.Context, markerID string) (SweptMarker, error) { + row := q.db.QueryRowContext(ctx, selectSweptMarker, markerID) + var i SweptMarker + err := row.Scan(&i.MarkerID, &i.SweptAt) + return i, err +} + +const selectSweptMarkersByIds = `-- name: SelectSweptMarkersByIds :many +SELECT marker_id, swept_at FROM swept_marker WHERE marker_id = ANY($1::text[]) +` + +func (q *Queries) SelectSweptMarkersByIds(ctx context.Context, markerIds []string) ([]SweptMarker, error) { + rows, err := q.db.QueryContext(ctx, selectSweptMarkersByIds, pq.Array(markerIds)) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SweptMarker + for rows.Next() { + var i SweptMarker + if err := rows.Scan(&i.MarkerID, &i.SweptAt); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const selectSweptRoundsConnectorAddress = `-- name: SelectSweptRoundsConnectorAddress :many SELECT round.connector_address FROM round WHERE round.swept = true AND round.failed = false AND round.ended = true AND round.connector_address <> '' @@ -1388,7 +1568,7 @@ func (q *Queries) SelectTxs(ctx context.Context, dollar_1 []string) ([]SelectTxs } const selectVtxo = `-- name: SelectVtxo :one -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.commitments FROM vtxo_vw WHERE txid = $1 AND vout = $2 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE txid = $1 AND vout = $2 ` type SelectVtxoParams struct { @@ -1420,11 +1600,63 @@ func (q *Queries) SelectVtxo(ctx context.Context, arg SelectVtxoParams) (SelectV &i.VtxoVw.ArkTxid, &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, &i.VtxoVw.Commitments, ) return i, err } +const selectVtxoChainByMarker = `-- name: SelectVtxoChainByMarker :many +SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, marker_id, commitments FROM vtxo_vw +WHERE marker_id = ANY($1::TEXT[]) +ORDER BY depth DESC +` + +// Get VTXOs that share the same marker or have markers in the parent chain +func (q *Queries) SelectVtxoChainByMarker(ctx context.Context, markerIds []string) ([]VtxoVw, error) { + rows, err := q.db.QueryContext(ctx, selectVtxoChainByMarker, pq.Array(markerIds)) + if err != nil { + return nil, err + } + defer rows.Close() + var items []VtxoVw + for rows.Next() { + var i VtxoVw + if err := rows.Scan( + &i.Txid, + &i.Vout, + &i.Pubkey, + &i.Amount, + &i.ExpiresAt, + &i.CreatedAt, + &i.CommitmentTxid, + &i.SpentBy, + &i.Spent, + &i.Unrolled, + &i.Swept, + &i.Preconfirmed, + &i.SettledBy, + &i.ArkTxid, + &i.IntentID, + &i.UpdatedAt, + &i.Depth, + &i.MarkerID, + &i.Commitments, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const selectVtxoPubKeysByCommitmentTxid = `-- name: SelectVtxoPubKeysByCommitmentTxid :many SELECT DISTINCT v.pubkey FROM vtxo_vw v @@ -1461,6 +1693,162 @@ func (q *Queries) SelectVtxoPubKeysByCommitmentTxid(ctx context.Context, arg Sel return items, nil } +const selectVtxosByArkTxid = `-- name: SelectVtxosByArkTxid :many +SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, marker_id, commitments FROM vtxo_vw WHERE txid = $1 +` + +// Get all VTXOs created by a specific ark tx (offchain tx) +func (q *Queries) SelectVtxosByArkTxid(ctx context.Context, arkTxid string) ([]VtxoVw, error) { + rows, err := q.db.QueryContext(ctx, selectVtxosByArkTxid, arkTxid) + if err != nil { + return nil, err + } + defer rows.Close() + var items []VtxoVw + for rows.Next() { + var i VtxoVw + if err := rows.Scan( + &i.Txid, + &i.Vout, + &i.Pubkey, + &i.Amount, + &i.ExpiresAt, + &i.CreatedAt, + &i.CommitmentTxid, + &i.SpentBy, + &i.Spent, + &i.Unrolled, + &i.Swept, + &i.Preconfirmed, + &i.SettledBy, + &i.ArkTxid, + &i.IntentID, + &i.UpdatedAt, + &i.Depth, + &i.MarkerID, + &i.Commitments, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const selectVtxosByDepthRange = `-- name: SelectVtxosByDepthRange :many + +SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, marker_id, commitments FROM vtxo_vw +WHERE depth >= $1 AND depth <= $2 +ORDER BY depth DESC +` + +type SelectVtxosByDepthRangeParams struct { + MinDepth int32 + MaxDepth int32 +} + +// Chain traversal queries for GetVtxoChain optimization +// Get all VTXOs within a depth range, useful for filling gaps between markers +func (q *Queries) SelectVtxosByDepthRange(ctx context.Context, arg SelectVtxosByDepthRangeParams) ([]VtxoVw, error) { + rows, err := q.db.QueryContext(ctx, selectVtxosByDepthRange, arg.MinDepth, arg.MaxDepth) + if err != nil { + return nil, err + } + defer rows.Close() + var items []VtxoVw + for rows.Next() { + var i VtxoVw + if err := rows.Scan( + &i.Txid, + &i.Vout, + &i.Pubkey, + &i.Amount, + &i.ExpiresAt, + &i.CreatedAt, + &i.CommitmentTxid, + &i.SpentBy, + &i.Spent, + &i.Unrolled, + &i.Swept, + &i.Preconfirmed, + &i.SettledBy, + &i.ArkTxid, + &i.IntentID, + &i.UpdatedAt, + &i.Depth, + &i.MarkerID, + &i.Commitments, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const selectVtxosByMarkerId = `-- name: SelectVtxosByMarkerId :many +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE marker_id = $1 +` + +type SelectVtxosByMarkerIdRow struct { + VtxoVw VtxoVw +} + +func (q *Queries) SelectVtxosByMarkerId(ctx context.Context, markerID sql.NullString) ([]SelectVtxosByMarkerIdRow, error) { + rows, err := q.db.QueryContext(ctx, selectVtxosByMarkerId, markerID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SelectVtxosByMarkerIdRow + for rows.Next() { + var i SelectVtxosByMarkerIdRow + if err := rows.Scan( + &i.VtxoVw.Txid, + &i.VtxoVw.Vout, + &i.VtxoVw.Pubkey, + &i.VtxoVw.Amount, + &i.VtxoVw.ExpiresAt, + &i.VtxoVw.CreatedAt, + &i.VtxoVw.CommitmentTxid, + &i.VtxoVw.SpentBy, + &i.VtxoVw.Spent, + &i.VtxoVw.Unrolled, + &i.VtxoVw.Swept, + &i.VtxoVw.Preconfirmed, + &i.VtxoVw.SettledBy, + &i.VtxoVw.ArkTxid, + &i.VtxoVw.IntentID, + &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, + &i.VtxoVw.Commitments, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const selectVtxosOutpointsByArkTxidRecursive = `-- name: SelectVtxosOutpointsByArkTxidRecursive :many WITH RECURSIVE descendants_chain AS ( -- seed @@ -1523,7 +1911,7 @@ func (q *Queries) SelectVtxosOutpointsByArkTxidRecursive(ctx context.Context, tx } const selectVtxosWithPubkeys = `-- name: SelectVtxosWithPubkeys :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.commitments FROM vtxo_vw +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE vtxo_vw.pubkey = ANY($1::varchar[]) AND vtxo_vw.updated_at >= $2::bigint AND ($3::bigint = 0 OR vtxo_vw.updated_at <= $3::bigint) @@ -1565,6 +1953,8 @@ func (q *Queries) SelectVtxosWithPubkeys(ctx context.Context, arg SelectVtxosWit &i.VtxoVw.ArkTxid, &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -1580,6 +1970,19 @@ func (q *Queries) SelectVtxosWithPubkeys(ctx context.Context, arg SelectVtxosWit return items, nil } +const sweepVtxosByMarkerId = `-- name: SweepVtxosByMarkerId :execrows +UPDATE vtxo SET swept = true, updated_at = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT +WHERE marker_id = $1 AND swept = false +` + +func (q *Queries) SweepVtxosByMarkerId(ctx context.Context, markerID sql.NullString) (int64, error) { + result, err := q.db.ExecContext(ctx, sweepVtxosByMarkerId, markerID) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + const updateConvictionPardoned = `-- name: UpdateConvictionPardoned :exec UPDATE conviction SET pardoned = true WHERE id = $1 ` @@ -1619,6 +2022,21 @@ func (q *Queries) UpdateVtxoIntentId(ctx context.Context, arg UpdateVtxoIntentId return err } +const updateVtxoMarkerId = `-- name: UpdateVtxoMarkerId :exec +UPDATE vtxo SET marker_id = $1 WHERE txid = $2 AND vout = $3 +` + +type UpdateVtxoMarkerIdParams struct { + MarkerID sql.NullString + Txid string + Vout int32 +} + +func (q *Queries) UpdateVtxoMarkerId(ctx context.Context, arg UpdateVtxoMarkerIdParams) error { + _, err := q.db.ExecContext(ctx, updateVtxoMarkerId, arg.MarkerID, arg.Txid, arg.Vout) + return err +} + const updateVtxoSettled = `-- name: UpdateVtxoSettled :exec UPDATE vtxo SET spent = true, spent_by = $1, settled_by = $2, updated_at = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT WHERE txid = $3 AND vout = $4 @@ -1788,6 +2206,27 @@ func (q *Queries) UpsertIntent(ctx context.Context, arg UpsertIntentParams) erro return err } +const upsertMarker = `-- name: UpsertMarker :exec + +INSERT INTO marker (id, depth, parent_markers) +VALUES ($1, $2, $3) +ON CONFLICT(id) DO UPDATE SET + depth = EXCLUDED.depth, + parent_markers = EXCLUDED.parent_markers +` + +type UpsertMarkerParams struct { + ID string + Depth int32 + ParentMarkers pqtype.NullRawMessage +} + +// Marker queries +func (q *Queries) UpsertMarker(ctx context.Context, arg UpsertMarkerParams) error { + _, err := q.db.ExecContext(ctx, upsertMarker, arg.ID, arg.Depth, arg.ParentMarkers) + return err +} + const upsertOffchainTx = `-- name: UpsertOffchainTx :exec INSERT INTO offchain_tx (txid, tx, starting_timestamp, ending_timestamp, expiry_timestamp, fail_reason, stage_code) VALUES ($1, $2, $3, $4, $5, $6, $7) @@ -1975,11 +2414,11 @@ func (q *Queries) UpsertTx(ctx context.Context, arg UpsertTxParams) error { const upsertVtxo = `-- name: UpsertVtxo :exec INSERT INTO vtxo ( txid, vout, pubkey, amount, commitment_txid, settled_by, ark_txid, - spent_by, spent, unrolled, swept, preconfirmed, expires_at, created_at, updated_at + spent_by, spent, unrolled, swept, preconfirmed, expires_at, created_at, updated_at, depth ) VALUES ( $1, $2, $3, $4, $5, $6, $7, - $8, $9, $10, $11, $12, $13, $14, (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT + $8, $9, $10, $11, $12, $13, $14, (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, $15 ) ON CONFLICT(txid, vout) DO UPDATE SET pubkey = EXCLUDED.pubkey, amount = EXCLUDED.amount, @@ -1993,7 +2432,8 @@ VALUES ( preconfirmed = EXCLUDED.preconfirmed, expires_at = EXCLUDED.expires_at, created_at = EXCLUDED.created_at, - updated_at = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT + updated_at = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, + depth = EXCLUDED.depth ` type UpsertVtxoParams struct { @@ -2011,6 +2451,7 @@ type UpsertVtxoParams struct { Preconfirmed bool ExpiresAt int64 CreatedAt int64 + Depth int32 } func (q *Queries) UpsertVtxo(ctx context.Context, arg UpsertVtxoParams) error { @@ -2029,6 +2470,7 @@ func (q *Queries) UpsertVtxo(ctx context.Context, arg UpsertVtxoParams) error { arg.Preconfirmed, arg.ExpiresAt, arg.CreatedAt, + arg.Depth, ) return err } diff --git a/internal/infrastructure/db/postgres/sqlc/query.sql b/internal/infrastructure/db/postgres/sqlc/query.sql index 18f58e81f..cb90f6cf3 100644 --- a/internal/infrastructure/db/postgres/sqlc/query.sql +++ b/internal/infrastructure/db/postgres/sqlc/query.sql @@ -48,11 +48,11 @@ ON CONFLICT(intent_id, pubkey, onchain_address) DO UPDATE SET -- name: UpsertVtxo :exec INSERT INTO vtxo ( txid, vout, pubkey, amount, commitment_txid, settled_by, ark_txid, - spent_by, spent, unrolled, swept, preconfirmed, expires_at, created_at, updated_at + spent_by, spent, unrolled, swept, preconfirmed, expires_at, created_at, updated_at, depth ) VALUES ( @txid, @vout, @pubkey, @amount, @commitment_txid, @settled_by, @ark_txid, - @spent_by, @spent, @unrolled, @swept, @preconfirmed, @expires_at, @created_at, (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT + @spent_by, @spent, @unrolled, @swept, @preconfirmed, @expires_at, @created_at, (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, @depth ) ON CONFLICT(txid, vout) DO UPDATE SET pubkey = EXCLUDED.pubkey, amount = EXCLUDED.amount, @@ -66,7 +66,8 @@ VALUES ( preconfirmed = EXCLUDED.preconfirmed, expires_at = EXCLUDED.expires_at, created_at = EXCLUDED.created_at, - updated_at = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT; + updated_at = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, + depth = EXCLUDED.depth; -- name: InsertVtxoCommitmentTxid :exec INSERT INTO vtxo_commitment_txid (vtxo_txid, vtxo_vout, commitment_txid) @@ -423,4 +424,67 @@ VALUES ('', '', '', ''); -- name: SelectIntentByTxid :one SELECT id, txid, proof, message FROM intent -WHERE txid = @txid; \ No newline at end of file +WHERE txid = @txid; + +-- Marker queries + +-- name: UpsertMarker :exec +INSERT INTO marker (id, depth, parent_markers) +VALUES (@id, @depth, @parent_markers) +ON CONFLICT(id) DO UPDATE SET + depth = EXCLUDED.depth, + parent_markers = EXCLUDED.parent_markers; + +-- name: SelectMarker :one +SELECT * FROM marker WHERE id = @id; + +-- name: SelectMarkersByDepth :many +SELECT * FROM marker WHERE depth = @depth; + +-- name: SelectMarkersByDepthRange :many +SELECT * FROM marker WHERE depth >= @min_depth AND depth <= @max_depth ORDER BY depth; + +-- name: SelectMarkersByIds :many +SELECT * FROM marker WHERE id = ANY(@ids::text[]); + +-- name: InsertSweptMarker :exec +INSERT INTO swept_marker (marker_id, swept_at) +VALUES (@marker_id, @swept_at) +ON CONFLICT(marker_id) DO NOTHING; + +-- name: SelectSweptMarker :one +SELECT * FROM swept_marker WHERE marker_id = @marker_id; + +-- name: SelectSweptMarkersByIds :many +SELECT * FROM swept_marker WHERE marker_id = ANY(@marker_ids::text[]); + +-- name: IsMarkerSwept :one +SELECT EXISTS(SELECT 1 FROM swept_marker WHERE marker_id = @marker_id) AS is_swept; + +-- name: UpdateVtxoMarkerId :exec +UPDATE vtxo SET marker_id = @marker_id WHERE txid = @txid AND vout = @vout; + +-- name: SelectVtxosByMarkerId :many +SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE marker_id = @marker_id; + +-- name: SweepVtxosByMarkerId :execrows +UPDATE vtxo SET swept = true, updated_at = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT +WHERE marker_id = @marker_id AND swept = false; + +-- Chain traversal queries for GetVtxoChain optimization + +-- name: SelectVtxosByDepthRange :many +-- Get all VTXOs within a depth range, useful for filling gaps between markers +SELECT * FROM vtxo_vw +WHERE depth >= @min_depth AND depth <= @max_depth +ORDER BY depth DESC; + +-- name: SelectVtxosByArkTxid :many +-- Get all VTXOs created by a specific ark tx (offchain tx) +SELECT * FROM vtxo_vw WHERE txid = @ark_txid; + +-- name: SelectVtxoChainByMarker :many +-- Get VTXOs that share the same marker or have markers in the parent chain +SELECT * FROM vtxo_vw +WHERE marker_id = ANY(@marker_ids::TEXT[]) +ORDER BY depth DESC; \ No newline at end of file diff --git a/internal/infrastructure/db/postgres/vtxo_repo.go b/internal/infrastructure/db/postgres/vtxo_repo.go index 7aa5a4199..bfd090527 100644 --- a/internal/infrastructure/db/postgres/vtxo_repo.go +++ b/internal/infrastructure/db/postgres/vtxo_repo.go @@ -62,6 +62,7 @@ func (v *vtxoRepository) AddVtxos(ctx context.Context, vtxos []domain.Vtxo) erro ArkTxid: sql.NullString{ String: vtxo.ArkTxid, Valid: len(vtxo.ArkTxid) > 0, }, + Depth: int32(vtxo.Depth), }, ); err != nil { return err @@ -524,6 +525,8 @@ func rowToVtxo(row queries.VtxoVw) domain.Vtxo { Preconfirmed: row.Preconfirmed, ExpiresAt: row.ExpiresAt, CreatedAt: row.CreatedAt, + Depth: uint32(row.Depth), + MarkerID: row.MarkerID.String, } } diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index 8f60f6612..e17bbe1b1 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -73,6 +73,10 @@ var ( "sqlite": sqlitedb.NewIntentFeesRepository, "postgres": pgdb.NewIntentFeesRepository, } + markerStoreTypes = map[string]func(...interface{}) (domain.MarkerRepository, error){ + "sqlite": sqlitedb.NewMarkerRepository, + "postgres": pgdb.NewMarkerRepository, + } ) const ( @@ -91,6 +95,7 @@ type service struct { eventStore domain.EventRepository roundStore domain.RoundRepository vtxoStore domain.VtxoRepository + markerStore domain.MarkerRepository scheduledSessionStore domain.ScheduledSessionRepo offchainTxStore domain.OffchainTxRepository convictionStore domain.ConvictionRepository @@ -127,10 +132,12 @@ func NewService(config ServiceConfig, txDecoder ports.TxDecoder) (ports.RepoMana if !ok { return nil, fmt.Errorf("invalid data store type: %s", config.DataStoreType) } + markerStoreFactory := markerStoreTypes[config.DataStoreType] // optional, may be nil for badger var eventStore domain.EventRepository var roundStore domain.RoundRepository var vtxoStore domain.VtxoRepository + var markerStore domain.MarkerRepository var scheduledSessionStore domain.ScheduledSessionRepo var offchainTxStore domain.OffchainTxRepository var convictionStore domain.ConvictionRepository @@ -268,6 +275,12 @@ func NewService(config ServiceConfig, txDecoder ports.TxDecoder) (ports.RepoMana if err != nil { return nil, fmt.Errorf("failed to create intent fees store: %w", err) } + if markerStoreFactory != nil { + markerStore, err = markerStoreFactory(db) + if err != nil { + return nil, fmt.Errorf("failed to create marker store: %w", err) + } + } case "sqlite": if len(config.DataStoreConfig) != 1 { return nil, fmt.Errorf("invalid data store config") @@ -332,12 +345,19 @@ func NewService(config ServiceConfig, txDecoder ports.TxDecoder) (ports.RepoMana if err != nil { return nil, fmt.Errorf("failed to create intent fees store: %w", err) } + if markerStoreFactory != nil { + markerStore, err = markerStoreFactory(db) + if err != nil { + return nil, fmt.Errorf("failed to create marker store: %w", err) + } + } } svc := &service{ eventStore: eventStore, roundStore: roundStore, vtxoStore: vtxoStore, + markerStore: markerStore, scheduledSessionStore: scheduledSessionStore, offchainTxStore: offchainTxStore, txDecoder: txDecoder, @@ -368,6 +388,10 @@ func (s *service) Vtxos() domain.VtxoRepository { return s.vtxoStore } +func (s *service) Markers() domain.MarkerRepository { + return s.markerStore +} + func (s *service) ScheduledSession() domain.ScheduledSessionRepo { return s.scheduledSessionStore } @@ -388,6 +412,9 @@ func (s *service) Close() { s.eventStore.Close() s.roundStore.Close() s.vtxoStore.Close() + if s.markerStore != nil { + s.markerStore.Close() + } s.scheduledSessionStore.Close() s.offchainTxStore.Close() s.convictionStore.Close() @@ -413,11 +440,21 @@ func (s *service) updateProjectionsAfterRoundEvents(events []domain.Event) { if lastEvent.GetType() == domain.EventTypeBatchSwept { event := lastEvent.(domain.BatchSwept) allSweptVtxos := append(event.LeafVtxos, event.PreconfirmedVtxos...) - sweptCount, err := repo.SweepVtxos(ctx, allSweptVtxos) - if err != nil { - log.WithError(err).Warn("failed to sweep vtxos") + + // Try marker-based sweeping first if marker store is available + if s.markerStore != nil { + sweptCount := s.sweepVtxosWithMarkers(ctx, allSweptVtxos) + if sweptCount > 0 { + log.Debugf("swept %d vtxos using marker-based sweeping", sweptCount) + } } else { - log.Debugf("swept %d vtxos", sweptCount) + // Fall back to individual VTXO sweeping + sweptCount, err := repo.SweepVtxos(ctx, allSweptVtxos) + if err != nil { + log.WithError(err).Warn("failed to sweep vtxos") + } else { + log.Debugf("swept %d vtxos", sweptCount) + } } if event.FullySwept { @@ -453,6 +490,27 @@ func (s *service) updateProjectionsAfterRoundEvents(events []domain.Event) { log.Debugf("added %d new vtxos", len(newVtxos)) break } + + // Create root markers for batch VTXOs (depth 0 is always at marker boundary) + if s.markerStore != nil { + for _, vtxo := range newVtxos { + // Each batch VTXO at depth 0 gets its own root marker + markerID := vtxo.Outpoint.String() + marker := domain.Marker{ + ID: markerID, + Depth: 0, + ParentMarkerIDs: nil, // Root markers have no parents + } + if err := s.markerStore.AddMarker(ctx, marker); err != nil { + log.WithError(err).Warnf("failed to create root marker for vtxo %s", vtxo.Outpoint.String()) + continue + } + if err := s.markerStore.UpdateVtxoMarker(ctx, vtxo.Outpoint, markerID); err != nil { + log.WithError(err).Warnf("failed to update marker_id for vtxo %s", vtxo.Outpoint.String()) + } + } + log.Debugf("created %d root markers for batch vtxos", len(newVtxos)) + } } } @@ -494,6 +552,62 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) return } + // Get spent VTXO outpoints from checkpoint txs to calculate depth + spentOutpoints := make([]domain.Outpoint, 0) + for _, tx := range offchainTx.CheckpointTxs { + _, ins, _, err := s.txDecoder.DecodeTx(tx) + if err != nil { + log.WithError(err).Warn("failed to decode checkpoint tx for depth calculation") + continue + } + spentOutpoints = append(spentOutpoints, ins...) + } + + // Get spent VTXOs to calculate new depth + var newDepth uint32 + var parentMarkerIDs []string + if len(spentOutpoints) > 0 { + spentVtxos, err := s.vtxoStore.GetVtxos(ctx, spentOutpoints) + if err != nil { + log.WithError(err).Warn("failed to get spent vtxos for depth calculation") + } else { + // Calculate depth: max(parent depths) + 1 + var maxDepth uint32 + parentMarkerSet := make(map[string]struct{}) + for _, v := range spentVtxos { + if v.Depth > maxDepth { + maxDepth = v.Depth + } + // Collect parent marker IDs for marker linking (will be used at boundary) + // Note: We need to get marker_id from the VTXO, which requires the field to be added to domain.Vtxo + // For now, we'll create markers without parent links - this can be enhanced in a follow-up + } + newDepth = maxDepth + 1 + for id := range parentMarkerSet { + parentMarkerIDs = append(parentMarkerIDs, id) + } + } + } + + // Create marker if at boundary depth + var markerID string + if s.markerStore != nil && domain.IsAtMarkerBoundary(newDepth) { + // Create marker ID from the first output (the ark tx id + first vtxo vout) + markerID = fmt.Sprintf("%s:marker:%d", txid, newDepth) + marker := domain.Marker{ + ID: markerID, + Depth: newDepth, + ParentMarkerIDs: parentMarkerIDs, + } + if err := s.markerStore.AddMarker(ctx, marker); err != nil { + log.WithError(err).Warn("failed to create marker for chained vtxo") + // Continue without marker - non-fatal + markerID = "" + } else { + log.Debugf("created marker %s at depth %d", markerID, newDepth) + } + } + // once the offchain tx is finalized, the user signed the checkpoint txs // thus, we can create the new vtxos in the db. newVtxos := make([]domain.Vtxo, 0, len(outs)) @@ -517,6 +631,7 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) RootCommitmentTxid: offchainTx.RootCommitmentTxId, Preconfirmed: true, CreatedAt: offchainTx.StartingTimestamp, + Depth: newDepth, // mark the vtxo as "swept" if it is below dust limit to prevent it from being spent again in a future offchain tx // the only way to spend a swept vtxo is by collecting enough dust to cover the minSettlementVtxoAmount and then settle. // because sub-dust vtxos are using OP_RETURN output script, they can't be unilaterally exited. @@ -528,7 +643,16 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) log.WithError(err).Warn("failed to add vtxos") return } - log.Debugf("added %d vtxos", len(newVtxos)) + log.Debugf("added %d vtxos at depth %d", len(newVtxos), newDepth) + + // Update marker_id for VTXOs at boundary depth + if markerID != "" && s.markerStore != nil { + for _, vtxo := range newVtxos { + if err := s.markerStore.UpdateVtxoMarker(ctx, vtxo.Outpoint, markerID); err != nil { + log.WithError(err).Warnf("failed to update marker_id for vtxo %s", vtxo.Outpoint.String()) + } + } + } } } @@ -602,6 +726,74 @@ func getNewVtxosFromRound(round *domain.Round) []domain.Vtxo { return vtxos } +// sweepVtxosWithMarkers performs marker-based sweeping for VTXOs. +// It groups VTXOs by their marker, sweeps each marker, then bulk-updates all VTXOs. +// Returns the total count of VTXOs swept. +func (s *service) sweepVtxosWithMarkers(ctx context.Context, vtxoOutpoints []domain.Outpoint) int64 { + if len(vtxoOutpoints) == 0 { + return 0 + } + + // Get VTXOs to find their markers + vtxos, err := s.vtxoStore.GetVtxos(ctx, vtxoOutpoints) + if err != nil { + log.WithError(err).Warn("failed to get vtxos for marker-based sweep") + // Fall back to individual sweep + count, _ := s.vtxoStore.SweepVtxos(ctx, vtxoOutpoints) + return int64(count) + } + + // Group VTXOs by marker ID + markerVtxos := make(map[string][]domain.Outpoint) + noMarkerVtxos := make([]domain.Outpoint, 0) + + for _, vtxo := range vtxos { + if vtxo.MarkerID != "" { + markerVtxos[vtxo.MarkerID] = append(markerVtxos[vtxo.MarkerID], vtxo.Outpoint) + } else { + noMarkerVtxos = append(noMarkerVtxos, vtxo.Outpoint) + } + } + + var totalSwept int64 + sweptAt := time.Now().Unix() + + // Sweep each marker + for markerID := range markerVtxos { + // Mark the marker as swept + if err := s.markerStore.SweepMarker(ctx, markerID, sweptAt); err != nil { + log.WithError(err).Warnf("failed to sweep marker %s", markerID) + // Fall back to individual sweep for this marker's VTXOs + count, _ := s.vtxoStore.SweepVtxos(ctx, markerVtxos[markerID]) + totalSwept += int64(count) + continue + } + + // Bulk sweep all VTXOs with this marker + count, err := s.markerStore.SweepVtxosByMarker(ctx, markerID) + if err != nil { + log.WithError(err).Warnf("failed to bulk sweep vtxos for marker %s", markerID) + // Fall back to individual sweep + count, _ := s.vtxoStore.SweepVtxos(ctx, markerVtxos[markerID]) + totalSwept += int64(count) + continue + } + totalSwept += count + log.Debugf("swept marker %s with %d vtxos", markerID, count) + } + + // Sweep VTXOs without markers individually + if len(noMarkerVtxos) > 0 { + count, err := s.vtxoStore.SweepVtxos(ctx, noMarkerVtxos) + if err != nil { + log.WithError(err).Warn("failed to sweep vtxos without markers") + } + totalSwept += int64(count) + } + + return totalSwept +} + func initBadgerArkRepository(args ...interface{}) (badgerdb.ArkRepository, error) { if arkRepo == nil { repo, err := badgerdb.NewArkRepository(args...) diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index 1c29c0a0f..1003ba174 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -186,6 +186,12 @@ func TestService(t *testing.T) { testEventRepository(t, svc) testRoundRepository(t, svc) testVtxoRepository(t, svc) + testMarkerBasicOperations(t, svc) + testMarkerSweep(t, svc) + testVtxoMarkerAssociation(t, svc) + testSweepVtxosByMarker(t, svc) + testMarkerDepthRangeQueries(t, svc) + testMarkerChainTraversal(t, svc) testOffchainTxRepository(t, svc) testScheduledSessionRepository(t, svc) testConvictionRepository(t, svc) @@ -653,6 +659,7 @@ func testVtxoRepository(t *testing.T, svc ports.RepoManager) { RootCommitmentTxid: commitmentTxid, CommitmentTxids: []string{commitmentTxid, "cmt1", "cmt2"}, Preconfirmed: true, + Depth: 2, // chained vtxo at depth 2 }, { Outpoint: domain.Outpoint{ @@ -663,6 +670,7 @@ func testVtxoRepository(t *testing.T, svc ports.RepoManager) { Amount: 2000, RootCommitmentTxid: commitmentTxid, CommitmentTxids: []string{commitmentTxid}, + Depth: 0, // batch vtxo at depth 0 }, } newVtxos := append(userVtxos, domain.Vtxo{ @@ -674,6 +682,7 @@ func testVtxoRepository(t *testing.T, svc ports.RepoManager) { Amount: 2000, RootCommitmentTxid: commitmentTxid, CommitmentTxids: []string{commitmentTxid}, + Depth: 1, // chained vtxo at depth 1 }) arkTxid := randomString(32) @@ -704,7 +713,7 @@ func testVtxoRepository(t *testing.T, svc ports.RepoManager) { vtxos, err = svc.Vtxos().GetAllVtxos(ctx) require.NoError(t, err) - require.Equal(t, 5, len(vtxos)) + require.Equal(t, numberOfVtxos+len(newVtxos), len(vtxos)) vtxos, err = svc.Vtxos().GetVtxos(ctx, vtxoKeys) require.NoError(t, err) @@ -1262,6 +1271,710 @@ func testVtxoRepository(t *testing.T, svc ports.RepoManager) { require.NoError(t, err) require.Equal(t, recoverableBefore+uint64(111), recoverableAfter) }) + + t.Run("test_vtxo_depth", func(t *testing.T) { + ctx := context.Background() + commitmentTxid := randomString(32) + + // Create vtxos with different depths to simulate a chain + // Batch vtxo at depth 0 + batchVtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 1000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 0, + } + + // First chain at depth 1 + chainedVtxo1 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 900, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid, randomString(32)}, + Depth: 1, + } + + // Second chain at depth 2 + chainedVtxo2 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 800, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid, randomString(32), randomString(32)}, + Depth: 2, + } + + // Deep chain at depth 100 + deepVtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 500, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 100, + } + + vtxosToAdd := []domain.Vtxo{batchVtxo, chainedVtxo1, chainedVtxo2, deepVtxo} + err := svc.Vtxos().AddVtxos(ctx, vtxosToAdd) + require.NoError(t, err) + + // Retrieve and verify depths are preserved + outpoints := []domain.Outpoint{ + batchVtxo.Outpoint, + chainedVtxo1.Outpoint, + chainedVtxo2.Outpoint, + deepVtxo.Outpoint, + } + retrievedVtxos, err := svc.Vtxos().GetVtxos(ctx, outpoints) + require.NoError(t, err) + require.Len(t, retrievedVtxos, 4) + + // Create a map for easier lookup + vtxoByOutpoint := make(map[string]domain.Vtxo) + for _, v := range retrievedVtxos { + vtxoByOutpoint[v.Outpoint.String()] = v + } + + // Verify each vtxo has correct depth + require.Equal(t, uint32(0), vtxoByOutpoint[batchVtxo.Outpoint.String()].Depth) + require.Equal(t, uint32(1), vtxoByOutpoint[chainedVtxo1.Outpoint.String()].Depth) + require.Equal(t, uint32(2), vtxoByOutpoint[chainedVtxo2.Outpoint.String()].Depth) + require.Equal(t, uint32(100), vtxoByOutpoint[deepVtxo.Outpoint.String()].Depth) + }) +} + +func testMarkerBasicOperations(t *testing.T, svc ports.RepoManager) { + t.Run("test_marker_basic_operations", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + + // Create markers with AddMarker + marker1 := domain.Marker{ + ID: randomString(32), + Depth: 0, + ParentMarkerIDs: nil, + } + marker2 := domain.Marker{ + ID: randomString(32), + Depth: 100, + ParentMarkerIDs: []string{marker1.ID}, + } + marker3 := domain.Marker{ + ID: randomString(32), + Depth: 100, + ParentMarkerIDs: []string{marker1.ID}, + } + marker4 := domain.Marker{ + ID: randomString(32), + Depth: 200, + ParentMarkerIDs: []string{marker2.ID, marker3.ID}, + } + + err := svc.Markers().AddMarker(ctx, marker1) + require.NoError(t, err) + err = svc.Markers().AddMarker(ctx, marker2) + require.NoError(t, err) + err = svc.Markers().AddMarker(ctx, marker3) + require.NoError(t, err) + err = svc.Markers().AddMarker(ctx, marker4) + require.NoError(t, err) + + // Test GetMarker - retrieve single marker and verify all fields + retrievedMarker1, err := svc.Markers().GetMarker(ctx, marker1.ID) + require.NoError(t, err) + require.NotNil(t, retrievedMarker1) + require.Equal(t, marker1.ID, retrievedMarker1.ID) + require.Equal(t, marker1.Depth, retrievedMarker1.Depth) + require.Empty(t, retrievedMarker1.ParentMarkerIDs) + + retrievedMarker2, err := svc.Markers().GetMarker(ctx, marker2.ID) + require.NoError(t, err) + require.NotNil(t, retrievedMarker2) + require.Equal(t, marker2.ID, retrievedMarker2.ID) + require.Equal(t, marker2.Depth, retrievedMarker2.Depth) + require.ElementsMatch(t, marker2.ParentMarkerIDs, retrievedMarker2.ParentMarkerIDs) + + retrievedMarker4, err := svc.Markers().GetMarker(ctx, marker4.ID) + require.NoError(t, err) + require.NotNil(t, retrievedMarker4) + require.Equal(t, marker4.ID, retrievedMarker4.ID) + require.Equal(t, marker4.Depth, retrievedMarker4.Depth) + require.ElementsMatch(t, marker4.ParentMarkerIDs, retrievedMarker4.ParentMarkerIDs) + + // Test GetMarker with non-existent ID + nonExistent, err := svc.Markers().GetMarker(ctx, "nonexistent") + require.NoError(t, err) + require.Nil(t, nonExistent) + + // Test GetMarkersByDepth - markers at same depth + markersAtDepth100, err := svc.Markers().GetMarkersByDepth(ctx, 100) + require.NoError(t, err) + require.Len(t, markersAtDepth100, 2) + markerIdsAtDepth100 := []string{markersAtDepth100[0].ID, markersAtDepth100[1].ID} + require.ElementsMatch(t, []string{marker2.ID, marker3.ID}, markerIdsAtDepth100) + + markersAtDepth0, err := svc.Markers().GetMarkersByDepth(ctx, 0) + require.NoError(t, err) + require.GreaterOrEqual(t, len(markersAtDepth0), 1) + var foundMarker1 bool + for _, m := range markersAtDepth0 { + if m.ID == marker1.ID { + foundMarker1 = true + break + } + } + require.True(t, foundMarker1) + + markersAtDepth200, err := svc.Markers().GetMarkersByDepth(ctx, 200) + require.NoError(t, err) + require.GreaterOrEqual(t, len(markersAtDepth200), 1) + var foundMarker4 bool + for _, m := range markersAtDepth200 { + if m.ID == marker4.ID { + foundMarker4 = true + break + } + } + require.True(t, foundMarker4) + + // Test GetMarkersByIds - batch retrieve + markersById, err := svc.Markers().GetMarkersByIds(ctx, []string{marker1.ID, marker3.ID, marker4.ID}) + require.NoError(t, err) + require.Len(t, markersById, 3) + retrievedIds := make([]string, len(markersById)) + for i, m := range markersById { + retrievedIds[i] = m.ID + } + require.ElementsMatch(t, []string{marker1.ID, marker3.ID, marker4.ID}, retrievedIds) + + // Test GetMarkersByIds with empty slice + emptyMarkers, err := svc.Markers().GetMarkersByIds(ctx, []string{}) + require.NoError(t, err) + require.Nil(t, emptyMarkers) + + // Test GetMarkersByIds with non-existent IDs mixed with valid + mixedMarkers, err := svc.Markers().GetMarkersByIds(ctx, []string{marker1.ID, "nonexistent"}) + require.NoError(t, err) + require.Len(t, mixedMarkers, 1) + require.Equal(t, marker1.ID, mixedMarkers[0].ID) + }) +} + +func testMarkerSweep(t *testing.T, svc ports.RepoManager) { + t.Run("test_marker_sweep", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + + // Create a marker + marker := domain.Marker{ + ID: randomString(32), + Depth: 0, + ParentMarkerIDs: nil, + } + err := svc.Markers().AddMarker(ctx, marker) + require.NoError(t, err) + + // Verify marker is not swept initially + isSwept, err := svc.Markers().IsMarkerSwept(ctx, marker.ID) + require.NoError(t, err) + require.False(t, isSwept) + + // Sweep the marker + sweptAt := time.Now().UnixMilli() + err = svc.Markers().SweepMarker(ctx, marker.ID, sweptAt) + require.NoError(t, err) + + // Verify IsMarkerSwept returns true + isSwept, err = svc.Markers().IsMarkerSwept(ctx, marker.ID) + require.NoError(t, err) + require.True(t, isSwept) + + // Verify GetSweptMarkers returns correct record + sweptMarkers, err := svc.Markers().GetSweptMarkers(ctx, []string{marker.ID}) + require.NoError(t, err) + require.Len(t, sweptMarkers, 1) + require.Equal(t, marker.ID, sweptMarkers[0].MarkerID) + require.Equal(t, sweptAt, sweptMarkers[0].SweptAt) + + // Test idempotency - sweeping again should not error (ON CONFLICT DO NOTHING) + err = svc.Markers().SweepMarker(ctx, marker.ID, sweptAt+1000) + require.NoError(t, err) + + // Verify the original swept_at is preserved (not updated) + sweptMarkers, err = svc.Markers().GetSweptMarkers(ctx, []string{marker.ID}) + require.NoError(t, err) + require.Len(t, sweptMarkers, 1) + require.Equal(t, sweptAt, sweptMarkers[0].SweptAt) + + // Test GetSweptMarkers with multiple markers + marker2 := domain.Marker{ + ID: randomString(32), + Depth: 100, + ParentMarkerIDs: []string{marker.ID}, + } + err = svc.Markers().AddMarker(ctx, marker2) + require.NoError(t, err) + + sweptAt2 := time.Now().UnixMilli() + err = svc.Markers().SweepMarker(ctx, marker2.ID, sweptAt2) + require.NoError(t, err) + + sweptMarkers, err = svc.Markers().GetSweptMarkers(ctx, []string{marker.ID, marker2.ID}) + require.NoError(t, err) + require.Len(t, sweptMarkers, 2) + + // Test GetSweptMarkers with empty slice + emptySwept, err := svc.Markers().GetSweptMarkers(ctx, []string{}) + require.NoError(t, err) + require.Nil(t, emptySwept) + + // Test IsMarkerSwept for non-existent marker + isSwept, err = svc.Markers().IsMarkerSwept(ctx, "nonexistent") + require.NoError(t, err) + require.False(t, isSwept) + }) +} + +func testVtxoMarkerAssociation(t *testing.T, svc ports.RepoManager) { + t.Run("test_vtxo_marker_association", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + commitmentTxid := randomString(32) + + // Create a marker + markerID := randomString(32) + marker := domain.Marker{ + ID: markerID, + Depth: 0, + ParentMarkerIDs: nil, + } + err := svc.Markers().AddMarker(ctx, marker) + require.NoError(t, err) + + // Add VTXOs without marker_id + vtxo1 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 1000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 0, + } + vtxo2 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 2000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 50, + } + vtxo3 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey2, + Amount: 3000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 75, + } + + err = svc.Vtxos().AddVtxos(ctx, []domain.Vtxo{vtxo1, vtxo2, vtxo3}) + require.NoError(t, err) + + // Verify VTXOs initially have no marker_id + retrievedVtxos, err := svc.Vtxos().GetVtxos(ctx, []domain.Outpoint{vtxo1.Outpoint}) + require.NoError(t, err) + require.Len(t, retrievedVtxos, 1) + require.Empty(t, retrievedVtxos[0].MarkerID) + + // Call UpdateVtxoMarker to associate VTXOs with marker + err = svc.Markers().UpdateVtxoMarker(ctx, vtxo1.Outpoint, markerID) + require.NoError(t, err) + err = svc.Markers().UpdateVtxoMarker(ctx, vtxo2.Outpoint, markerID) + require.NoError(t, err) + + // Verify GetVtxosByMarker returns the associated VTXOs + vtxosByMarker, err := svc.Markers().GetVtxosByMarker(ctx, markerID) + require.NoError(t, err) + require.Len(t, vtxosByMarker, 2) + outpoints := []string{vtxosByMarker[0].Outpoint.String(), vtxosByMarker[1].Outpoint.String()} + require.ElementsMatch(t, []string{vtxo1.Outpoint.String(), vtxo2.Outpoint.String()}, outpoints) + + // Verify VTXO.MarkerID field is populated when retrieved via GetVtxos + retrievedVtxos, err = svc.Vtxos().GetVtxos(ctx, []domain.Outpoint{vtxo1.Outpoint, vtxo2.Outpoint}) + require.NoError(t, err) + require.Len(t, retrievedVtxos, 2) + for _, v := range retrievedVtxos { + require.Equal(t, markerID, v.MarkerID) + } + + // Verify vtxo3 still has no marker + retrievedVtxos, err = svc.Vtxos().GetVtxos(ctx, []domain.Outpoint{vtxo3.Outpoint}) + require.NoError(t, err) + require.Len(t, retrievedVtxos, 1) + require.Empty(t, retrievedVtxos[0].MarkerID) + + // Test GetVtxosByMarker with non-existent marker + vtxosByNonExistent, err := svc.Markers().GetVtxosByMarker(ctx, "nonexistent") + require.NoError(t, err) + require.Empty(t, vtxosByNonExistent) + }) +} + +func testSweepVtxosByMarker(t *testing.T, svc ports.RepoManager) { + t.Run("test_sweep_vtxos_by_marker", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + commitmentTxid := randomString(32) + + // Create a marker + markerID := randomString(32) + marker := domain.Marker{ + ID: markerID, + Depth: 0, + ParentMarkerIDs: nil, + } + err := svc.Markers().AddMarker(ctx, marker) + require.NoError(t, err) + + // Add 5 VTXOs - 3 unswept, 2 already swept + vtxos := make([]domain.Vtxo, 5) + for i := 0; i < 5; i++ { + vtxos[i] = domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: uint64(1000 * (i + 1)), + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: uint32(i * 10), + Swept: i >= 3, // vtxos[3] and vtxos[4] are already swept + } + } + + err = svc.Vtxos().AddVtxos(ctx, vtxos) + require.NoError(t, err) + + // Associate all VTXOs with the marker + for _, v := range vtxos { + err = svc.Markers().UpdateVtxoMarker(ctx, v.Outpoint, markerID) + require.NoError(t, err) + } + + // Verify initial state + vtxosByMarker, err := svc.Markers().GetVtxosByMarker(ctx, markerID) + require.NoError(t, err) + require.Len(t, vtxosByMarker, 5) + + sweptCount := 0 + for _, v := range vtxosByMarker { + if v.Swept { + sweptCount++ + } + } + require.Equal(t, 2, sweptCount) + + // Call SweepVtxosByMarker + count, err := svc.Markers().SweepVtxosByMarker(ctx, markerID) + require.NoError(t, err) + require.Equal(t, int64(3), count) // Only 3 were newly swept + + // Verify all 5 VTXOs now have swept=true + vtxosByMarker, err = svc.Markers().GetVtxosByMarker(ctx, markerID) + require.NoError(t, err) + require.Len(t, vtxosByMarker, 5) + for _, v := range vtxosByMarker { + require.True(t, v.Swept, "VTXO %s should be swept", v.Outpoint.String()) + } + + // Call SweepVtxosByMarker again - should return 0 (all already swept) + count, err = svc.Markers().SweepVtxosByMarker(ctx, markerID) + require.NoError(t, err) + require.Equal(t, int64(0), count) + + // Test with non-existent marker + count, err = svc.Markers().SweepVtxosByMarker(ctx, "nonexistent") + require.NoError(t, err) + require.Equal(t, int64(0), count) + }) +} + +func testMarkerDepthRangeQueries(t *testing.T, svc ports.RepoManager) { + t.Run("test_marker_depth_range_queries", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + commitmentTxid := randomString(32) + + // Add markers at depths 0, 100, 200, 300 with unique IDs + markerDepth0 := domain.Marker{ + ID: "range_test_" + randomString(16), + Depth: 0, + ParentMarkerIDs: nil, + } + markerDepth100 := domain.Marker{ + ID: "range_test_" + randomString(16), + Depth: 100, + ParentMarkerIDs: []string{markerDepth0.ID}, + } + markerDepth200 := domain.Marker{ + ID: "range_test_" + randomString(16), + Depth: 200, + ParentMarkerIDs: []string{markerDepth100.ID}, + } + markerDepth300 := domain.Marker{ + ID: "range_test_" + randomString(16), + Depth: 300, + ParentMarkerIDs: []string{markerDepth200.ID}, + } + + err := svc.Markers().AddMarker(ctx, markerDepth0) + require.NoError(t, err) + err = svc.Markers().AddMarker(ctx, markerDepth100) + require.NoError(t, err) + err = svc.Markers().AddMarker(ctx, markerDepth200) + require.NoError(t, err) + err = svc.Markers().AddMarker(ctx, markerDepth300) + require.NoError(t, err) + + // Test GetMarkersByDepthRange(50, 250) - should return markers at 100 and 200 + markersInRange, err := svc.Markers().GetMarkersByDepthRange(ctx, 50, 250) + require.NoError(t, err) + + // Filter to only our test markers + var ourMarkers []domain.Marker + testMarkerIDs := map[string]bool{ + markerDepth0.ID: true, + markerDepth100.ID: true, + markerDepth200.ID: true, + markerDepth300.ID: true, + } + for _, m := range markersInRange { + if testMarkerIDs[m.ID] { + ourMarkers = append(ourMarkers, m) + } + } + require.Len(t, ourMarkers, 2) + foundDepths := []uint32{ourMarkers[0].Depth, ourMarkers[1].Depth} + require.ElementsMatch(t, []uint32{100, 200}, foundDepths) + + // Test range that includes all + markersInRange, err = svc.Markers().GetMarkersByDepthRange(ctx, 0, 300) + require.NoError(t, err) + ourMarkers = nil + for _, m := range markersInRange { + if testMarkerIDs[m.ID] { + ourMarkers = append(ourMarkers, m) + } + } + require.Len(t, ourMarkers, 4) + + // Test range that includes none of our test markers + markersInRange, err = svc.Markers().GetMarkersByDepthRange(ctx, 350, 400) + require.NoError(t, err) + ourMarkers = nil + for _, m := range markersInRange { + if testMarkerIDs[m.ID] { + ourMarkers = append(ourMarkers, m) + } + } + require.Empty(t, ourMarkers) + + // Add VTXOs at depths 0, 50, 100, 150 with unique IDs + vtxoDepth0 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "vtxo_range_" + randomString(24), VOut: 0}, + PubKey: pubkey, + Amount: 1000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 0, + } + vtxoDepth50 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "vtxo_range_" + randomString(24), VOut: 0}, + PubKey: pubkey, + Amount: 2000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 50, + } + vtxoDepth100 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "vtxo_range_" + randomString(24), VOut: 0}, + PubKey: pubkey, + Amount: 3000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 100, + } + vtxoDepth150 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "vtxo_range_" + randomString(24), VOut: 0}, + PubKey: pubkey, + Amount: 4000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 150, + } + + err = svc.Vtxos().AddVtxos(ctx, []domain.Vtxo{vtxoDepth0, vtxoDepth50, vtxoDepth100, vtxoDepth150}) + require.NoError(t, err) + + // Test GetVtxosByDepthRange(25, 125) - should return VTXOs at 50 and 100 + vtxosInRange, err := svc.Markers().GetVtxosByDepthRange(ctx, 25, 125) + require.NoError(t, err) + + // Filter to only our test vtxos + testVtxoTxids := map[string]bool{ + vtxoDepth0.Txid: true, + vtxoDepth50.Txid: true, + vtxoDepth100.Txid: true, + vtxoDepth150.Txid: true, + } + var ourVtxos []domain.Vtxo + for _, v := range vtxosInRange { + if testVtxoTxids[v.Txid] { + ourVtxos = append(ourVtxos, v) + } + } + require.Len(t, ourVtxos, 2) + foundVtxoDepths := []uint32{ourVtxos[0].Depth, ourVtxos[1].Depth} + require.ElementsMatch(t, []uint32{50, 100}, foundVtxoDepths) + + // Test range that includes all test vtxos + vtxosInRange, err = svc.Markers().GetVtxosByDepthRange(ctx, 0, 150) + require.NoError(t, err) + ourVtxos = nil + for _, v := range vtxosInRange { + if testVtxoTxids[v.Txid] { + ourVtxos = append(ourVtxos, v) + } + } + require.Len(t, ourVtxos, 4) + + // Test range that includes none + vtxosInRange, err = svc.Markers().GetVtxosByDepthRange(ctx, 200, 300) + require.NoError(t, err) + ourVtxos = nil + for _, v := range vtxosInRange { + if testVtxoTxids[v.Txid] { + ourVtxos = append(ourVtxos, v) + } + } + require.Empty(t, ourVtxos) + }) +} + +func testMarkerChainTraversal(t *testing.T, svc ports.RepoManager) { + t.Run("test_marker_chain_traversal", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + commitmentTxid := randomString(32) + + // Create markers for the chain + marker1 := domain.Marker{ + ID: "chain_marker_" + randomString(16), + Depth: 0, + ParentMarkerIDs: nil, + } + marker2 := domain.Marker{ + ID: "chain_marker_" + randomString(16), + Depth: 100, + ParentMarkerIDs: []string{marker1.ID}, + } + + err := svc.Markers().AddMarker(ctx, marker1) + require.NoError(t, err) + err = svc.Markers().AddMarker(ctx, marker2) + require.NoError(t, err) + + // Create an ark_txid that links vtxos together + arkTxid := "ark_chain_" + randomString(24) + + // Add VTXOs with ark_txid (marker_ids will be set via UpdateVtxoMarker) + vtxo1 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "chain_vtxo_" + randomString(20), VOut: 0}, + PubKey: pubkey, + Amount: 1000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 0, + ArkTxid: arkTxid, + } + vtxo2 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: arkTxid, VOut: 0}, // Created by arkTxid + PubKey: pubkey, + Amount: 900, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 1, + } + vtxo3 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "chain_vtxo_" + randomString(20), VOut: 0}, + PubKey: pubkey, + Amount: 800, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 100, + } + + err = svc.Vtxos().AddVtxos(ctx, []domain.Vtxo{vtxo1, vtxo2, vtxo3}) + require.NoError(t, err) + + // Associate VTXOs with their markers using UpdateVtxoMarker + err = svc.Markers().UpdateVtxoMarker(ctx, vtxo1.Outpoint, marker1.ID) + require.NoError(t, err) + err = svc.Markers().UpdateVtxoMarker(ctx, vtxo2.Outpoint, marker1.ID) + require.NoError(t, err) + err = svc.Markers().UpdateVtxoMarker(ctx, vtxo3.Outpoint, marker2.ID) + require.NoError(t, err) + + // Test GetVtxoChainByMarkers - returns VTXOs for given marker list + vtxosByMarkers, err := svc.Markers().GetVtxoChainByMarkers(ctx, []string{marker1.ID}) + require.NoError(t, err) + require.Len(t, vtxosByMarkers, 2) // vtxo1 and vtxo2 have marker1.ID + foundTxids := make(map[string]bool) + for _, v := range vtxosByMarkers { + foundTxids[v.Txid] = true + } + require.True(t, foundTxids[vtxo1.Txid]) + require.True(t, foundTxids[vtxo2.Txid]) + + // Test with both markers + vtxosByMarkers, err = svc.Markers().GetVtxoChainByMarkers(ctx, []string{marker1.ID, marker2.ID}) + require.NoError(t, err) + require.Len(t, vtxosByMarkers, 3) + + // Test with empty marker list + vtxosByMarkers, err = svc.Markers().GetVtxoChainByMarkers(ctx, []string{}) + require.NoError(t, err) + require.Nil(t, vtxosByMarkers) + + // Test with non-existent marker + vtxosByMarkers, err = svc.Markers().GetVtxoChainByMarkers(ctx, []string{"nonexistent"}) + require.NoError(t, err) + require.Empty(t, vtxosByMarkers) + + // Test GetVtxosByArkTxid - returns VTXOs created by specific ark tx + vtxosByArkTxid, err := svc.Markers().GetVtxosByArkTxid(ctx, arkTxid) + require.NoError(t, err) + require.Len(t, vtxosByArkTxid, 1) // Only vtxo2 has Txid == arkTxid + require.Equal(t, vtxo2.Txid, vtxosByArkTxid[0].Txid) + + // Test GetVtxosByArkTxid with non-existent ark txid + vtxosByArkTxid, err = svc.Markers().GetVtxosByArkTxid(ctx, "nonexistent") + require.NoError(t, err) + require.Empty(t, vtxosByArkTxid) + }) } func testScheduledSessionRepository(t *testing.T, svc ports.RepoManager) { @@ -1813,6 +2526,7 @@ func checkVtxos(t *testing.T, expectedVtxos sortVtxos, gotVtxos sortVtxos) { require.Exactly(t, expected.Spent, v.Spent) require.Exactly(t, expected.SpentBy, v.SpentBy) require.Exactly(t, expected.Swept, v.Swept) + require.Exactly(t, expected.Depth, v.Depth) require.ElementsMatch(t, expected.CommitmentTxids, v.CommitmentTxids) } } diff --git a/internal/infrastructure/db/sqlite/marker_repo.go b/internal/infrastructure/db/sqlite/marker_repo.go new file mode 100644 index 000000000..fc94fbca8 --- /dev/null +++ b/internal/infrastructure/db/sqlite/marker_repo.go @@ -0,0 +1,362 @@ +package sqlitedb + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "strings" + + "github.com/arkade-os/arkd/internal/core/domain" + "github.com/arkade-os/arkd/internal/infrastructure/db/sqlite/sqlc/queries" +) + +type markerRepository struct { + db *sql.DB + querier *queries.Queries +} + +func NewMarkerRepository(config ...interface{}) (domain.MarkerRepository, error) { + if len(config) != 1 { + return nil, fmt.Errorf("invalid config") + } + db, ok := config[0].(*sql.DB) + if !ok { + return nil, fmt.Errorf("cannot open marker repository: invalid config") + } + + return &markerRepository{ + db: db, + querier: queries.New(db), + }, nil +} + +func (m *markerRepository) Close() { + _ = m.db.Close() +} + +func (m *markerRepository) AddMarker(ctx context.Context, marker domain.Marker) error { + parentMarkersJSON, err := json.Marshal(marker.ParentMarkerIDs) + if err != nil { + return fmt.Errorf("failed to marshal parent markers: %w", err) + } + + return m.querier.UpsertMarker(ctx, queries.UpsertMarkerParams{ + ID: marker.ID, + Depth: int64(marker.Depth), + ParentMarkers: sql.NullString{String: string(parentMarkersJSON), Valid: true}, + }) +} + +func (m *markerRepository) GetMarker(ctx context.Context, id string) (*domain.Marker, error) { + row, err := m.querier.SelectMarker(ctx, id) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, err + } + + marker, err := rowToMarker(row) + if err != nil { + return nil, err + } + return &marker, nil +} + +func (m *markerRepository) GetMarkersByDepth(ctx context.Context, depth uint32) ([]domain.Marker, error) { + rows, err := m.querier.SelectMarkersByDepth(ctx, int64(depth)) + if err != nil { + return nil, err + } + + markers := make([]domain.Marker, 0, len(rows)) + for _, row := range rows { + marker, err := rowToMarker(row) + if err != nil { + return nil, err + } + markers = append(markers, marker) + } + return markers, nil +} + +func (m *markerRepository) GetMarkersByDepthRange(ctx context.Context, minDepth, maxDepth uint32) ([]domain.Marker, error) { + rows, err := m.querier.SelectMarkersByDepthRange(ctx, queries.SelectMarkersByDepthRangeParams{ + MinDepth: int64(minDepth), + MaxDepth: int64(maxDepth), + }) + if err != nil { + return nil, err + } + + markers := make([]domain.Marker, 0, len(rows)) + for _, row := range rows { + marker, err := rowToMarker(row) + if err != nil { + return nil, err + } + markers = append(markers, marker) + } + return markers, nil +} + +func (m *markerRepository) GetMarkersByIds(ctx context.Context, ids []string) ([]domain.Marker, error) { + if len(ids) == 0 { + return nil, nil + } + + rows, err := m.querier.SelectMarkersByIds(ctx, ids) + if err != nil { + return nil, err + } + + markers := make([]domain.Marker, 0, len(rows)) + for _, row := range rows { + marker, err := rowToMarker(row) + if err != nil { + return nil, err + } + markers = append(markers, marker) + } + return markers, nil +} + +func (m *markerRepository) SweepMarker(ctx context.Context, markerID string, sweptAt int64) error { + return m.querier.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ + MarkerID: markerID, + SweptAt: sweptAt, + }) +} + +func (m *markerRepository) IsMarkerSwept(ctx context.Context, markerID string) (bool, error) { + result, err := m.querier.IsMarkerSwept(ctx, markerID) + if err != nil { + return false, err + } + return result == 1, nil +} + +func (m *markerRepository) GetSweptMarkers(ctx context.Context, markerIDs []string) ([]domain.SweptMarker, error) { + if len(markerIDs) == 0 { + return nil, nil + } + + rows, err := m.querier.SelectSweptMarkersByIds(ctx, markerIDs) + if err != nil { + return nil, err + } + + sweptMarkers := make([]domain.SweptMarker, 0, len(rows)) + for _, row := range rows { + sweptMarkers = append(sweptMarkers, domain.SweptMarker{ + MarkerID: row.MarkerID, + SweptAt: row.SweptAt, + }) + } + return sweptMarkers, nil +} + +func (m *markerRepository) UpdateVtxoMarker(ctx context.Context, outpoint domain.Outpoint, markerID string) error { + return m.querier.UpdateVtxoMarkerId(ctx, queries.UpdateVtxoMarkerIdParams{ + MarkerID: sql.NullString{String: markerID, Valid: len(markerID) > 0}, + Txid: outpoint.Txid, + Vout: int64(outpoint.VOut), + }) +} + +func (m *markerRepository) GetVtxosByMarker(ctx context.Context, markerID string) ([]domain.Vtxo, error) { + rows, err := m.querier.SelectVtxosByMarkerId(ctx, sql.NullString{String: markerID, Valid: len(markerID) > 0}) + if err != nil { + return nil, err + } + + vtxos := make([]domain.Vtxo, 0, len(rows)) + for _, row := range rows { + vtxos = append(vtxos, rowToVtxoFromMarkerQuery(row)) + } + return vtxos, nil +} + +func (m *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID string) (int64, error) { + return m.querier.SweepVtxosByMarkerId(ctx, sql.NullString{String: markerID, Valid: len(markerID) > 0}) +} + +func (m *markerRepository) GetVtxosByDepthRange(ctx context.Context, minDepth, maxDepth uint32) ([]domain.Vtxo, error) { + rows, err := m.querier.SelectVtxosByDepthRange(ctx, queries.SelectVtxosByDepthRangeParams{ + MinDepth: int64(minDepth), + MaxDepth: int64(maxDepth), + }) + if err != nil { + return nil, err + } + + vtxos := make([]domain.Vtxo, 0, len(rows)) + for _, row := range rows { + vtxos = append(vtxos, rowToVtxoFromDepthRangeQuery(row)) + } + return vtxos, nil +} + +func (m *markerRepository) GetVtxosByArkTxid(ctx context.Context, arkTxid string) ([]domain.Vtxo, error) { + rows, err := m.querier.SelectVtxosByArkTxid(ctx, arkTxid) + if err != nil { + return nil, err + } + + vtxos := make([]domain.Vtxo, 0, len(rows)) + for _, row := range rows { + vtxos = append(vtxos, rowToVtxoFromArkTxidQuery(row)) + } + return vtxos, nil +} + +func (m *markerRepository) GetVtxoChainByMarkers(ctx context.Context, markerIDs []string) ([]domain.Vtxo, error) { + if len(markerIDs) == 0 { + return nil, nil + } + + // Convert string slice to sql.NullString slice + nullStringIDs := make([]sql.NullString, len(markerIDs)) + for i, id := range markerIDs { + nullStringIDs[i] = sql.NullString{String: id, Valid: true} + } + + rows, err := m.querier.SelectVtxoChainByMarker(ctx, nullStringIDs) + if err != nil { + return nil, err + } + + vtxos := make([]domain.Vtxo, 0, len(rows)) + for _, row := range rows { + vtxos = append(vtxos, rowToVtxoFromChainQuery(row)) + } + return vtxos, nil +} + +func rowToMarker(row queries.Marker) (domain.Marker, error) { + var parentMarkerIDs []string + if row.ParentMarkers.Valid && row.ParentMarkers.String != "" { + if err := json.Unmarshal([]byte(row.ParentMarkers.String), &parentMarkerIDs); err != nil { + return domain.Marker{}, fmt.Errorf("failed to unmarshal parent markers: %w", err) + } + } + + return domain.Marker{ + ID: row.ID, + Depth: uint32(row.Depth), + ParentMarkerIDs: parentMarkerIDs, + }, nil +} + +func rowToVtxoFromMarkerQuery(row queries.SelectVtxosByMarkerIdRow) domain.Vtxo { + var commitmentTxids []string + if commitments, ok := row.VtxoVw.Commitments.(string); ok && commitments != "" { + commitmentTxids = strings.Split(commitments, ",") + } + return domain.Vtxo{ + Outpoint: domain.Outpoint{ + Txid: row.VtxoVw.Txid, + VOut: uint32(row.VtxoVw.Vout), + }, + Amount: uint64(row.VtxoVw.Amount), + PubKey: row.VtxoVw.Pubkey, + RootCommitmentTxid: row.VtxoVw.CommitmentTxid, + CommitmentTxids: commitmentTxids, + SettledBy: row.VtxoVw.SettledBy.String, + ArkTxid: row.VtxoVw.ArkTxid.String, + SpentBy: row.VtxoVw.SpentBy.String, + Spent: row.VtxoVw.Spent, + Unrolled: row.VtxoVw.Unrolled, + Swept: row.VtxoVw.Swept, + Preconfirmed: row.VtxoVw.Preconfirmed, + ExpiresAt: row.VtxoVw.ExpiresAt, + CreatedAt: row.VtxoVw.CreatedAt, + Depth: uint32(row.VtxoVw.Depth), + MarkerID: row.VtxoVw.MarkerID.String, + } +} + +func rowToVtxoFromDepthRangeQuery(row queries.SelectVtxosByDepthRangeRow) domain.Vtxo { + var commitmentTxids []string + if commitments, ok := row.VtxoVw.Commitments.(string); ok && commitments != "" { + commitmentTxids = strings.Split(commitments, ",") + } + return domain.Vtxo{ + Outpoint: domain.Outpoint{ + Txid: row.VtxoVw.Txid, + VOut: uint32(row.VtxoVw.Vout), + }, + Amount: uint64(row.VtxoVw.Amount), + PubKey: row.VtxoVw.Pubkey, + RootCommitmentTxid: row.VtxoVw.CommitmentTxid, + CommitmentTxids: commitmentTxids, + SettledBy: row.VtxoVw.SettledBy.String, + ArkTxid: row.VtxoVw.ArkTxid.String, + SpentBy: row.VtxoVw.SpentBy.String, + Spent: row.VtxoVw.Spent, + Unrolled: row.VtxoVw.Unrolled, + Swept: row.VtxoVw.Swept, + Preconfirmed: row.VtxoVw.Preconfirmed, + ExpiresAt: row.VtxoVw.ExpiresAt, + CreatedAt: row.VtxoVw.CreatedAt, + Depth: uint32(row.VtxoVw.Depth), + MarkerID: row.VtxoVw.MarkerID.String, + } +} + +func rowToVtxoFromArkTxidQuery(row queries.SelectVtxosByArkTxidRow) domain.Vtxo { + var commitmentTxids []string + if commitments, ok := row.VtxoVw.Commitments.(string); ok && commitments != "" { + commitmentTxids = strings.Split(commitments, ",") + } + return domain.Vtxo{ + Outpoint: domain.Outpoint{ + Txid: row.VtxoVw.Txid, + VOut: uint32(row.VtxoVw.Vout), + }, + Amount: uint64(row.VtxoVw.Amount), + PubKey: row.VtxoVw.Pubkey, + RootCommitmentTxid: row.VtxoVw.CommitmentTxid, + CommitmentTxids: commitmentTxids, + SettledBy: row.VtxoVw.SettledBy.String, + ArkTxid: row.VtxoVw.ArkTxid.String, + SpentBy: row.VtxoVw.SpentBy.String, + Spent: row.VtxoVw.Spent, + Unrolled: row.VtxoVw.Unrolled, + Swept: row.VtxoVw.Swept, + Preconfirmed: row.VtxoVw.Preconfirmed, + ExpiresAt: row.VtxoVw.ExpiresAt, + CreatedAt: row.VtxoVw.CreatedAt, + Depth: uint32(row.VtxoVw.Depth), + MarkerID: row.VtxoVw.MarkerID.String, + } +} + +func rowToVtxoFromChainQuery(row queries.SelectVtxoChainByMarkerRow) domain.Vtxo { + var commitmentTxids []string + if commitments, ok := row.VtxoVw.Commitments.(string); ok && commitments != "" { + commitmentTxids = strings.Split(commitments, ",") + } + return domain.Vtxo{ + Outpoint: domain.Outpoint{ + Txid: row.VtxoVw.Txid, + VOut: uint32(row.VtxoVw.Vout), + }, + Amount: uint64(row.VtxoVw.Amount), + PubKey: row.VtxoVw.Pubkey, + RootCommitmentTxid: row.VtxoVw.CommitmentTxid, + CommitmentTxids: commitmentTxids, + SettledBy: row.VtxoVw.SettledBy.String, + ArkTxid: row.VtxoVw.ArkTxid.String, + SpentBy: row.VtxoVw.SpentBy.String, + Spent: row.VtxoVw.Spent, + Unrolled: row.VtxoVw.Unrolled, + Swept: row.VtxoVw.Swept, + Preconfirmed: row.VtxoVw.Preconfirmed, + ExpiresAt: row.VtxoVw.ExpiresAt, + CreatedAt: row.VtxoVw.CreatedAt, + Depth: uint32(row.VtxoVw.Depth), + MarkerID: row.VtxoVw.MarkerID.String, + } +} diff --git a/internal/infrastructure/db/sqlite/migration/20260210000000_add_vtxo_depth.down.sql b/internal/infrastructure/db/sqlite/migration/20260210000000_add_vtxo_depth.down.sql new file mode 100644 index 000000000..085a0d609 --- /dev/null +++ b/internal/infrastructure/db/sqlite/migration/20260210000000_add_vtxo_depth.down.sql @@ -0,0 +1,54 @@ +-- SQLite doesn't support DROP COLUMN directly, so we need to recreate the table +-- This migration creates a new table without the depth column and copies data + +DROP VIEW IF EXISTS intent_with_inputs_vw; +DROP VIEW IF EXISTS vtxo_vw; + +CREATE TABLE vtxo_new ( + txid TEXT NOT NULL, + vout INTEGER NOT NULL, + pubkey TEXT NOT NULL, + amount INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + created_at INTEGER NOT NULL, + commitment_txid TEXT NOT NULL, + spent_by TEXT, + spent BOOLEAN NOT NULL DEFAULT FALSE, + unrolled BOOLEAN NOT NULL DEFAULT FALSE, + swept BOOLEAN NOT NULL DEFAULT FALSE, + preconfirmed BOOLEAN NOT NULL DEFAULT FALSE, + settled_by TEXT, + ark_txid TEXT, + intent_id TEXT, + updated_at BIGINT, + PRIMARY KEY (txid, vout), + FOREIGN KEY (intent_id) REFERENCES intent(id) +); + +INSERT INTO vtxo_new (txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, intent_id, updated_at) +SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, intent_id, updated_at +FROM vtxo; + +DROP TABLE vtxo; +ALTER TABLE vtxo_new RENAME TO vtxo; + +-- Recreate foreign key index +CREATE INDEX IF NOT EXISTS fk_vtxo_intent_id ON vtxo(intent_id); + +-- Recreate views without depth column +CREATE VIEW vtxo_vw AS +SELECT v.*, COALESCE(group_concat(vc.commitment_txid), '') AS commitments +FROM vtxo v +LEFT JOIN vtxo_commitment_txid vc +ON v.txid = vc.vtxo_txid AND v.vout = vc.vtxo_vout +GROUP BY v.txid, v.vout; + +CREATE VIEW intent_with_inputs_vw AS +SELECT vtxo_vw.*, + intent.id, + intent.round_id, + intent.proof, + intent.message +FROM intent +LEFT OUTER JOIN vtxo_vw +ON intent.id = vtxo_vw.intent_id; diff --git a/internal/infrastructure/db/sqlite/migration/20260210000000_add_vtxo_depth.up.sql b/internal/infrastructure/db/sqlite/migration/20260210000000_add_vtxo_depth.up.sql new file mode 100644 index 000000000..1bf880bcb --- /dev/null +++ b/internal/infrastructure/db/sqlite/migration/20260210000000_add_vtxo_depth.up.sql @@ -0,0 +1,22 @@ +ALTER TABLE vtxo ADD COLUMN depth INTEGER NOT NULL DEFAULT 0; + +-- Recreate views to include the new depth column +DROP VIEW IF EXISTS intent_with_inputs_vw; +DROP VIEW IF EXISTS vtxo_vw; + +CREATE VIEW vtxo_vw AS +SELECT v.*, COALESCE(group_concat(vc.commitment_txid), '') AS commitments +FROM vtxo v +LEFT JOIN vtxo_commitment_txid vc +ON v.txid = vc.vtxo_txid AND v.vout = vc.vtxo_vout +GROUP BY v.txid, v.vout; + +CREATE VIEW intent_with_inputs_vw AS +SELECT vtxo_vw.*, + intent.id, + intent.round_id, + intent.proof, + intent.message +FROM intent +LEFT OUTER JOIN vtxo_vw +ON intent.id = vtxo_vw.intent_id; diff --git a/internal/infrastructure/db/sqlite/migration/20260211000000_add_markers.down.sql b/internal/infrastructure/db/sqlite/migration/20260211000000_add_markers.down.sql new file mode 100644 index 000000000..c58d58f05 --- /dev/null +++ b/internal/infrastructure/db/sqlite/migration/20260211000000_add_markers.down.sql @@ -0,0 +1,65 @@ +-- Remove marker_id column from vtxo +-- SQLite doesn't support DROP COLUMN directly, so we need to recreate the table + +-- Recreate views to remove marker_id column +DROP VIEW IF EXISTS intent_with_inputs_vw; +DROP VIEW IF EXISTS vtxo_vw; + +-- Create temp table without marker_id +CREATE TABLE vtxo_temp ( + txid TEXT NOT NULL, + vout INTEGER NOT NULL, + pubkey TEXT NOT NULL, + amount INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + created_at INTEGER NOT NULL, + commitment_txid TEXT NOT NULL, + spent_by TEXT, + spent BOOLEAN NOT NULL DEFAULT FALSE, + unrolled BOOLEAN NOT NULL DEFAULT FALSE, + swept BOOLEAN NOT NULL DEFAULT FALSE, + preconfirmed BOOLEAN NOT NULL DEFAULT FALSE, + settled_by TEXT, + ark_txid TEXT, + intent_id TEXT, + updated_at INTEGER, + depth INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (txid, vout), + FOREIGN KEY (intent_id) REFERENCES intent(id) +); + +-- Copy data +INSERT INTO vtxo_temp SELECT + txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, + spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, + intent_id, updated_at, depth +FROM vtxo; + +-- Drop old table and rename +DROP TABLE vtxo; +ALTER TABLE vtxo_temp RENAME TO vtxo; + +-- Recreate indexes +CREATE INDEX IF NOT EXISTS fk_vtxo_intent_id ON vtxo(intent_id); + +-- Drop marker tables +DROP TABLE IF EXISTS swept_marker; +DROP TABLE IF EXISTS marker; + +-- Recreate views +CREATE VIEW vtxo_vw AS +SELECT v.*, COALESCE(group_concat(vc.commitment_txid), '') AS commitments +FROM vtxo v +LEFT JOIN vtxo_commitment_txid vc +ON v.txid = vc.vtxo_txid AND v.vout = vc.vtxo_vout +GROUP BY v.txid, v.vout; + +CREATE VIEW intent_with_inputs_vw AS +SELECT vtxo_vw.*, + intent.id, + intent.round_id, + intent.proof, + intent.message +FROM intent +LEFT OUTER JOIN vtxo_vw +ON intent.id = vtxo_vw.intent_id; diff --git a/internal/infrastructure/db/sqlite/migration/20260211000000_add_markers.up.sql b/internal/infrastructure/db/sqlite/migration/20260211000000_add_markers.up.sql new file mode 100644 index 000000000..7d0b216f8 --- /dev/null +++ b/internal/infrastructure/db/sqlite/migration/20260211000000_add_markers.up.sql @@ -0,0 +1,56 @@ +-- Create markers table +CREATE TABLE IF NOT EXISTS marker ( + id TEXT PRIMARY KEY, + depth INTEGER NOT NULL, + parent_markers TEXT -- JSON array of parent marker IDs +); +CREATE INDEX IF NOT EXISTS idx_marker_depth ON marker(depth); + +-- Create swept_markers table (append-only) +CREATE TABLE IF NOT EXISTS swept_marker ( + marker_id TEXT PRIMARY KEY REFERENCES marker(id), + swept_at INTEGER NOT NULL +); + +-- Add marker_id column to vtxo table +ALTER TABLE vtxo ADD COLUMN marker_id TEXT REFERENCES marker(id); +CREATE INDEX IF NOT EXISTS idx_vtxo_marker_id ON vtxo(marker_id); + +-- Recreate views to include the new marker_id column +DROP VIEW IF EXISTS intent_with_inputs_vw; +DROP VIEW IF EXISTS vtxo_vw; + +CREATE VIEW vtxo_vw AS +SELECT v.*, COALESCE(group_concat(vc.commitment_txid), '') AS commitments +FROM vtxo v +LEFT JOIN vtxo_commitment_txid vc +ON v.txid = vc.vtxo_txid AND v.vout = vc.vtxo_vout +GROUP BY v.txid, v.vout; + +CREATE VIEW intent_with_inputs_vw AS +SELECT vtxo_vw.*, + intent.id, + intent.round_id, + intent.proof, + intent.message +FROM intent +LEFT OUTER JOIN vtxo_vw +ON intent.id = vtxo_vw.intent_id; + +-- Backfill markers for existing VTXOs based on their depth +-- VTXOs at depth 0, 100, 200, ... get their own markers +-- Other VTXOs will have their marker_id set during PR 5 (marker assignment logic) + +-- First, create markers for all existing VTXOs at marker boundary depths (depth % 100 == 0) +INSERT INTO marker (id, depth, parent_markers) +SELECT + v.txid || ':' || v.vout, -- Use VTXO outpoint as marker ID + v.depth, + '[]' -- Empty parent markers for initial backfill +FROM vtxo v +WHERE v.depth % 100 = 0; + +-- Assign marker_id to VTXOs at boundary depths +UPDATE vtxo +SET marker_id = txid || ':' || vout +WHERE depth % 100 = 0; diff --git a/internal/infrastructure/db/sqlite/sqlc/queries/models.go b/internal/infrastructure/db/sqlite/sqlc/queries/models.go index 8c039a121..ff55213ba 100644 --- a/internal/infrastructure/db/sqlite/sqlc/queries/models.go +++ b/internal/infrastructure/db/sqlite/sqlc/queries/models.go @@ -62,6 +62,8 @@ type IntentWithInputsVw struct { ArkTxid sql.NullString IntentID sql.NullString UpdatedAt sql.NullInt64 + Depth sql.NullInt64 + MarkerID sql.NullString Commitments interface{} ID sql.NullString RoundID sql.NullString @@ -80,6 +82,12 @@ type IntentWithReceiversVw struct { Message sql.NullString } +type Marker struct { + ID string + Depth int64 + ParentMarkers sql.NullString +} + type OffchainTx struct { Txid string Tx string @@ -173,6 +181,11 @@ type ScheduledSession struct { UpdatedAt int64 } +type SweptMarker struct { + MarkerID string + SweptAt int64 +} + type Tx struct { Txid string Tx string @@ -198,7 +211,9 @@ type Vtxo struct { SettledBy sql.NullString ArkTxid sql.NullString IntentID sql.NullString - UpdatedAt int64 + UpdatedAt sql.NullInt64 + Depth int64 + MarkerID sql.NullString } type VtxoCommitmentTxid struct { @@ -223,6 +238,8 @@ type VtxoVw struct { SettledBy sql.NullString ArkTxid sql.NullString IntentID sql.NullString - UpdatedAt int64 + UpdatedAt sql.NullInt64 + Depth int64 + MarkerID sql.NullString Commitments interface{} } diff --git a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go index 68e07d029..b02828b64 100644 --- a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go @@ -84,6 +84,22 @@ func (q *Queries) ClearScheduledSession(ctx context.Context) error { return err } +const insertSweptMarker = `-- name: InsertSweptMarker :exec +INSERT INTO swept_marker (marker_id, swept_at) +VALUES (?1, ?2) +ON CONFLICT(marker_id) DO NOTHING +` + +type InsertSweptMarkerParams struct { + MarkerID string + SweptAt int64 +} + +func (q *Queries) InsertSweptMarker(ctx context.Context, arg InsertSweptMarkerParams) error { + _, err := q.db.ExecContext(ctx, insertSweptMarker, arg.MarkerID, arg.SweptAt) + return err +} + const insertVtxoCommitmentTxid = `-- name: InsertVtxoCommitmentTxid :exec INSERT INTO vtxo_commitment_txid (vtxo_txid, vtxo_vout, commitment_txid) VALUES (?1, ?2, ?3) @@ -100,6 +116,17 @@ func (q *Queries) InsertVtxoCommitmentTxid(ctx context.Context, arg InsertVtxoCo return err } +const isMarkerSwept = `-- name: IsMarkerSwept :one +SELECT EXISTS(SELECT 1 FROM swept_marker WHERE marker_id = ?1) AS is_swept +` + +func (q *Queries) IsMarkerSwept(ctx context.Context, markerID string) (int64, error) { + row := q.db.QueryRowContext(ctx, isMarkerSwept, markerID) + var is_swept int64 + err := row.Scan(&is_swept) + return is_swept, err +} + const selectActiveScriptConvictions = `-- name: SelectActiveScriptConvictions :many SELECT id, type, created_at, expires_at, crime_type, crime_round_id, crime_reason, pardoned, script FROM conviction WHERE script = ?1 @@ -174,7 +201,7 @@ func (q *Queries) SelectAllRoundIds(ctx context.Context) ([]string, error) { } const selectAllVtxos = `-- name: SelectAllVtxos :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.commitments FROM vtxo_vw +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw ` type SelectAllVtxosRow struct { @@ -207,6 +234,8 @@ func (q *Queries) SelectAllVtxos(ctx context.Context) ([]SelectAllVtxosRow, erro &i.VtxoVw.ArkTxid, &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -410,8 +439,115 @@ func (q *Queries) SelectLatestScheduledSession(ctx context.Context) (ScheduledSe return i, err } +const selectMarker = `-- name: SelectMarker :one +SELECT id, depth, parent_markers FROM marker WHERE id = ?1 +` + +func (q *Queries) SelectMarker(ctx context.Context, id string) (Marker, error) { + row := q.db.QueryRowContext(ctx, selectMarker, id) + var i Marker + err := row.Scan(&i.ID, &i.Depth, &i.ParentMarkers) + return i, err +} + +const selectMarkersByDepth = `-- name: SelectMarkersByDepth :many +SELECT id, depth, parent_markers FROM marker WHERE depth = ?1 +` + +func (q *Queries) SelectMarkersByDepth(ctx context.Context, depth int64) ([]Marker, error) { + rows, err := q.db.QueryContext(ctx, selectMarkersByDepth, depth) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Marker + for rows.Next() { + var i Marker + if err := rows.Scan(&i.ID, &i.Depth, &i.ParentMarkers); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const selectMarkersByDepthRange = `-- name: SelectMarkersByDepthRange :many +SELECT id, depth, parent_markers FROM marker WHERE depth >= ?1 AND depth <= ?2 ORDER BY depth +` + +type SelectMarkersByDepthRangeParams struct { + MinDepth int64 + MaxDepth int64 +} + +func (q *Queries) SelectMarkersByDepthRange(ctx context.Context, arg SelectMarkersByDepthRangeParams) ([]Marker, error) { + rows, err := q.db.QueryContext(ctx, selectMarkersByDepthRange, arg.MinDepth, arg.MaxDepth) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Marker + for rows.Next() { + var i Marker + if err := rows.Scan(&i.ID, &i.Depth, &i.ParentMarkers); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const selectMarkersByIds = `-- name: SelectMarkersByIds :many +SELECT id, depth, parent_markers FROM marker WHERE id IN (/*SLICE:ids*/?) +` + +func (q *Queries) SelectMarkersByIds(ctx context.Context, ids []string) ([]Marker, error) { + query := selectMarkersByIds + var queryParams []interface{} + if len(ids) > 0 { + for _, v := range ids { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:ids*/?", strings.Repeat(",?", len(ids))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:ids*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []Marker + for rows.Next() { + var i Marker + if err := rows.Scan(&i.ID, &i.Depth, &i.ParentMarkers); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const selectNotUnrolledVtxos = `-- name: SelectNotUnrolledVtxos :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false ` type SelectNotUnrolledVtxosRow struct { @@ -444,6 +580,8 @@ func (q *Queries) SelectNotUnrolledVtxos(ctx context.Context) ([]SelectNotUnroll &i.VtxoVw.ArkTxid, &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -460,7 +598,7 @@ func (q *Queries) SelectNotUnrolledVtxos(ctx context.Context) ([]SelectNotUnroll } const selectNotUnrolledVtxosWithPubkey = `-- name: SelectNotUnrolledVtxosWithPubkey :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false AND pubkey = ?1 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false AND pubkey = ?1 ` type SelectNotUnrolledVtxosWithPubkeyRow struct { @@ -493,6 +631,8 @@ func (q *Queries) SelectNotUnrolledVtxosWithPubkey(ctx context.Context, pubkey s &i.VtxoVw.ArkTxid, &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -553,7 +693,7 @@ func (q *Queries) SelectOffchainTx(ctx context.Context, txid string) ([]SelectOf } const selectPendingSpentVtxo = `-- name: SelectPendingSpentVtxo :one -SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.commitments +SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.marker_id, v.commitments FROM vtxo_vw v WHERE v.txid = ?1 AND v.vout = ?2 AND v.spent = TRUE AND v.unrolled = FALSE AND COALESCE(v.settled_by, '') = '' @@ -587,13 +727,15 @@ func (q *Queries) SelectPendingSpentVtxo(ctx context.Context, arg SelectPendingS &i.ArkTxid, &i.IntentID, &i.UpdatedAt, + &i.Depth, + &i.MarkerID, &i.Commitments, ) return i, err } const selectPendingSpentVtxosWithPubkeys = `-- name: SelectPendingSpentVtxosWithPubkeys :many -SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.commitments +SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.marker_id, v.commitments FROM vtxo_vw v WHERE v.spent = TRUE AND v.unrolled = FALSE AND COALESCE(v.settled_by, '') = '' AND v.pubkey IN (/*SLICE:pubkeys*/?) @@ -606,7 +748,7 @@ WHERE v.spent = TRUE AND v.unrolled = FALSE AND COALESCE(v.settled_by, '') = '' type SelectPendingSpentVtxosWithPubkeysParams struct { Pubkeys []string - After int64 + After sql.NullInt64 Before int64 } @@ -648,6 +790,8 @@ func (q *Queries) SelectPendingSpentVtxosWithPubkeys(ctx context.Context, arg Se &i.ArkTxid, &i.IntentID, &i.UpdatedAt, + &i.Depth, + &i.MarkerID, &i.Commitments, ); err != nil { return nil, err @@ -865,7 +1009,7 @@ SELECT r.ending_timestamp, ( SELECT COALESCE(SUM(amount), 0) FROM ( - SELECT DISTINCT v2.txid, v2.vout, v2.pubkey, v2.amount, v2.expires_at, v2.created_at, v2.commitment_txid, v2.spent_by, v2.spent, v2.unrolled, v2.swept, v2.preconfirmed, v2.settled_by, v2.ark_txid, v2.intent_id, v2.updated_at FROM vtxo v2 JOIN intent i2 ON i2.id = v2.intent_id WHERE i2.round_id = r.id + SELECT DISTINCT v2.txid, v2.vout, v2.pubkey, v2.amount, v2.expires_at, v2.created_at, v2.commitment_txid, v2.spent_by, v2.spent, v2.unrolled, v2.swept, v2.preconfirmed, v2.settled_by, v2.ark_txid, v2.intent_id, v2.updated_at, v2.depth, v2.marker_id FROM vtxo v2 JOIN intent i2 ON i2.id = v2.intent_id WHERE i2.round_id = r.id ) as intent_with_inputs_amount ) AS total_forfeit_amount, ( @@ -952,7 +1096,7 @@ func (q *Queries) SelectRoundVtxoTree(ctx context.Context, txid string) ([]Tx, e } const selectRoundVtxoTreeLeaves = `-- name: SelectRoundVtxoTreeLeaves :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.commitments FROM vtxo_vw WHERE commitment_txid = ?1 AND preconfirmed = false +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE commitment_txid = ?1 AND preconfirmed = false ` type SelectRoundVtxoTreeLeavesRow struct { @@ -985,6 +1129,8 @@ func (q *Queries) SelectRoundVtxoTreeLeaves(ctx context.Context, commitmentTxid &i.VtxoVw.ArkTxid, &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -1005,7 +1151,7 @@ SELECT round.id, round.starting_timestamp, round.ending_timestamp, round.ended, round_intents_vw.id, round_intents_vw.round_id, round_intents_vw.proof, round_intents_vw.message, round_txs_vw.txid, round_txs_vw.tx, round_txs_vw.round_id, round_txs_vw.type, round_txs_vw.position, round_txs_vw.children, intent_with_receivers_vw.intent_id, intent_with_receivers_vw.pubkey, intent_with_receivers_vw.onchain_address, intent_with_receivers_vw.amount, intent_with_receivers_vw.id, intent_with_receivers_vw.round_id, intent_with_receivers_vw.proof, intent_with_receivers_vw.message, - intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message + intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.marker_id, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message FROM round LEFT OUTER JOIN round_intents_vw ON round.id=round_intents_vw.round_id LEFT OUTER JOIN round_txs_vw ON round.id=round_txs_vw.round_id @@ -1077,6 +1223,8 @@ func (q *Queries) SelectRoundWithId(ctx context.Context, id string) ([]SelectRou &i.IntentWithInputsVw.ArkTxid, &i.IntentWithInputsVw.IntentID, &i.IntentWithInputsVw.UpdatedAt, + &i.IntentWithInputsVw.Depth, + &i.IntentWithInputsVw.MarkerID, &i.IntentWithInputsVw.Commitments, &i.IntentWithInputsVw.ID, &i.IntentWithInputsVw.RoundID, @@ -1101,7 +1249,7 @@ SELECT round.id, round.starting_timestamp, round.ending_timestamp, round.ended, round_intents_vw.id, round_intents_vw.round_id, round_intents_vw.proof, round_intents_vw.message, round_txs_vw.txid, round_txs_vw.tx, round_txs_vw.round_id, round_txs_vw.type, round_txs_vw.position, round_txs_vw.children, intent_with_receivers_vw.intent_id, intent_with_receivers_vw.pubkey, intent_with_receivers_vw.onchain_address, intent_with_receivers_vw.amount, intent_with_receivers_vw.id, intent_with_receivers_vw.round_id, intent_with_receivers_vw.proof, intent_with_receivers_vw.message, - intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message + intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.marker_id, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message FROM round LEFT OUTER JOIN round_intents_vw ON round.id=round_intents_vw.round_id LEFT OUTER JOIN round_txs_vw ON round.id=round_txs_vw.round_id @@ -1175,6 +1323,8 @@ func (q *Queries) SelectRoundWithTxid(ctx context.Context, txid string) ([]Selec &i.IntentWithInputsVw.ArkTxid, &i.IntentWithInputsVw.IntentID, &i.IntentWithInputsVw.UpdatedAt, + &i.IntentWithInputsVw.Depth, + &i.IntentWithInputsVw.MarkerID, &i.IntentWithInputsVw.Commitments, &i.IntentWithInputsVw.ID, &i.IntentWithInputsVw.RoundID, @@ -1264,7 +1414,7 @@ func (q *Queries) SelectSweepableRounds(ctx context.Context) ([]string, error) { } const selectSweepableUnrolledVtxos = `-- name: SelectSweepableUnrolledVtxos :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.commitments FROM vtxo_vw WHERE spent = true AND unrolled = true AND swept = false AND (COALESCE(settled_by, '') = '') +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE spent = true AND unrolled = true AND swept = false AND (COALESCE(settled_by, '') = '') ` type SelectSweepableUnrolledVtxosRow struct { @@ -1297,6 +1447,8 @@ func (q *Queries) SelectSweepableUnrolledVtxos(ctx context.Context) ([]SelectSwe &i.VtxoVw.ArkTxid, &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -1348,6 +1500,54 @@ func (q *Queries) SelectSweepableVtxoOutpointsByCommitmentTxid(ctx context.Conte return items, nil } +const selectSweptMarker = `-- name: SelectSweptMarker :one +SELECT marker_id, swept_at FROM swept_marker WHERE marker_id = ?1 +` + +func (q *Queries) SelectSweptMarker(ctx context.Context, markerID string) (SweptMarker, error) { + row := q.db.QueryRowContext(ctx, selectSweptMarker, markerID) + var i SweptMarker + err := row.Scan(&i.MarkerID, &i.SweptAt) + return i, err +} + +const selectSweptMarkersByIds = `-- name: SelectSweptMarkersByIds :many +SELECT marker_id, swept_at FROM swept_marker WHERE marker_id IN (/*SLICE:marker_ids*/?) +` + +func (q *Queries) SelectSweptMarkersByIds(ctx context.Context, markerIds []string) ([]SweptMarker, error) { + query := selectSweptMarkersByIds + var queryParams []interface{} + if len(markerIds) > 0 { + for _, v := range markerIds { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:marker_ids*/?", strings.Repeat(",?", len(markerIds))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:marker_ids*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SweptMarker + for rows.Next() { + var i SweptMarker + if err := rows.Scan(&i.MarkerID, &i.SweptAt); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const selectSweptRoundsConnectorAddress = `-- name: SelectSweptRoundsConnectorAddress :many SELECT round.connector_address FROM round WHERE round.swept = true AND round.failed = false AND round.ended = true AND round.connector_address <> '' @@ -1445,7 +1645,7 @@ func (q *Queries) SelectTxs(ctx context.Context, arg SelectTxsParams) ([]SelectT } const selectVtxo = `-- name: SelectVtxo :one -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.commitments FROM vtxo_vw WHERE txid = ?1 AND vout = ?2 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE txid = ?1 AND vout = ?2 ` type SelectVtxoParams struct { @@ -1477,11 +1677,77 @@ func (q *Queries) SelectVtxo(ctx context.Context, arg SelectVtxoParams) (SelectV &i.VtxoVw.ArkTxid, &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, &i.VtxoVw.Commitments, ) return i, err } +const selectVtxoChainByMarker = `-- name: SelectVtxoChainByMarker :many +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw +WHERE marker_id IN (/*SLICE:marker_ids*/?) +ORDER BY depth DESC +` + +type SelectVtxoChainByMarkerRow struct { + VtxoVw VtxoVw +} + +// Get VTXOs that share the same marker or have markers in the parent chain +func (q *Queries) SelectVtxoChainByMarker(ctx context.Context, markerIds []sql.NullString) ([]SelectVtxoChainByMarkerRow, error) { + query := selectVtxoChainByMarker + var queryParams []interface{} + if len(markerIds) > 0 { + for _, v := range markerIds { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:marker_ids*/?", strings.Repeat(",?", len(markerIds))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:marker_ids*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SelectVtxoChainByMarkerRow + for rows.Next() { + var i SelectVtxoChainByMarkerRow + if err := rows.Scan( + &i.VtxoVw.Txid, + &i.VtxoVw.Vout, + &i.VtxoVw.Pubkey, + &i.VtxoVw.Amount, + &i.VtxoVw.ExpiresAt, + &i.VtxoVw.CreatedAt, + &i.VtxoVw.CommitmentTxid, + &i.VtxoVw.SpentBy, + &i.VtxoVw.Spent, + &i.VtxoVw.Unrolled, + &i.VtxoVw.Swept, + &i.VtxoVw.Preconfirmed, + &i.VtxoVw.SettledBy, + &i.VtxoVw.ArkTxid, + &i.VtxoVw.IntentID, + &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, + &i.VtxoVw.Commitments, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const selectVtxoPubKeysByCommitmentTxid = `-- name: SelectVtxoPubKeysByCommitmentTxid :many SELECT DISTINCT v.pubkey FROM vtxo_vw v @@ -1518,6 +1784,170 @@ func (q *Queries) SelectVtxoPubKeysByCommitmentTxid(ctx context.Context, arg Sel return items, nil } +const selectVtxosByArkTxid = `-- name: SelectVtxosByArkTxid :many +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE txid = ?1 +` + +type SelectVtxosByArkTxidRow struct { + VtxoVw VtxoVw +} + +// Get all VTXOs created by a specific ark tx (offchain tx) +func (q *Queries) SelectVtxosByArkTxid(ctx context.Context, arkTxid string) ([]SelectVtxosByArkTxidRow, error) { + rows, err := q.db.QueryContext(ctx, selectVtxosByArkTxid, arkTxid) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SelectVtxosByArkTxidRow + for rows.Next() { + var i SelectVtxosByArkTxidRow + if err := rows.Scan( + &i.VtxoVw.Txid, + &i.VtxoVw.Vout, + &i.VtxoVw.Pubkey, + &i.VtxoVw.Amount, + &i.VtxoVw.ExpiresAt, + &i.VtxoVw.CreatedAt, + &i.VtxoVw.CommitmentTxid, + &i.VtxoVw.SpentBy, + &i.VtxoVw.Spent, + &i.VtxoVw.Unrolled, + &i.VtxoVw.Swept, + &i.VtxoVw.Preconfirmed, + &i.VtxoVw.SettledBy, + &i.VtxoVw.ArkTxid, + &i.VtxoVw.IntentID, + &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, + &i.VtxoVw.Commitments, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const selectVtxosByDepthRange = `-- name: SelectVtxosByDepthRange :many + +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw +WHERE depth >= ?1 AND depth <= ?2 +ORDER BY depth DESC +` + +type SelectVtxosByDepthRangeParams struct { + MinDepth int64 + MaxDepth int64 +} + +type SelectVtxosByDepthRangeRow struct { + VtxoVw VtxoVw +} + +// Chain traversal queries for GetVtxoChain optimization +// Get all VTXOs within a depth range, useful for filling gaps between markers +func (q *Queries) SelectVtxosByDepthRange(ctx context.Context, arg SelectVtxosByDepthRangeParams) ([]SelectVtxosByDepthRangeRow, error) { + rows, err := q.db.QueryContext(ctx, selectVtxosByDepthRange, arg.MinDepth, arg.MaxDepth) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SelectVtxosByDepthRangeRow + for rows.Next() { + var i SelectVtxosByDepthRangeRow + if err := rows.Scan( + &i.VtxoVw.Txid, + &i.VtxoVw.Vout, + &i.VtxoVw.Pubkey, + &i.VtxoVw.Amount, + &i.VtxoVw.ExpiresAt, + &i.VtxoVw.CreatedAt, + &i.VtxoVw.CommitmentTxid, + &i.VtxoVw.SpentBy, + &i.VtxoVw.Spent, + &i.VtxoVw.Unrolled, + &i.VtxoVw.Swept, + &i.VtxoVw.Preconfirmed, + &i.VtxoVw.SettledBy, + &i.VtxoVw.ArkTxid, + &i.VtxoVw.IntentID, + &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, + &i.VtxoVw.Commitments, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const selectVtxosByMarkerId = `-- name: SelectVtxosByMarkerId :many +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE marker_id = ?1 +` + +type SelectVtxosByMarkerIdRow struct { + VtxoVw VtxoVw +} + +func (q *Queries) SelectVtxosByMarkerId(ctx context.Context, markerID sql.NullString) ([]SelectVtxosByMarkerIdRow, error) { + rows, err := q.db.QueryContext(ctx, selectVtxosByMarkerId, markerID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SelectVtxosByMarkerIdRow + for rows.Next() { + var i SelectVtxosByMarkerIdRow + if err := rows.Scan( + &i.VtxoVw.Txid, + &i.VtxoVw.Vout, + &i.VtxoVw.Pubkey, + &i.VtxoVw.Amount, + &i.VtxoVw.ExpiresAt, + &i.VtxoVw.CreatedAt, + &i.VtxoVw.CommitmentTxid, + &i.VtxoVw.SpentBy, + &i.VtxoVw.Spent, + &i.VtxoVw.Unrolled, + &i.VtxoVw.Swept, + &i.VtxoVw.Preconfirmed, + &i.VtxoVw.SettledBy, + &i.VtxoVw.ArkTxid, + &i.VtxoVw.IntentID, + &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, + &i.VtxoVw.Commitments, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const selectVtxosOutpointsByArkTxidRecursive = `-- name: SelectVtxosOutpointsByArkTxidRecursive :many WITH RECURSIVE descendants_chain AS ( -- seed @@ -1579,14 +2009,14 @@ func (q *Queries) SelectVtxosOutpointsByArkTxidRecursive(ctx context.Context, tx } const selectVtxosWithPubkeys = `-- name: SelectVtxosWithPubkeys :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.commitments FROM vtxo_vw WHERE pubkey IN (/*SLICE:pubkeys*/?) +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE pubkey IN (/*SLICE:pubkeys*/?) AND updated_at >= ?2 AND (CAST(?3 AS INTEGER) = 0 OR updated_at <= CAST(?3 AS INTEGER)) ` type SelectVtxosWithPubkeysParams struct { Pubkeys []string - After int64 + After sql.NullInt64 Before int64 } @@ -1632,6 +2062,8 @@ func (q *Queries) SelectVtxosWithPubkeys(ctx context.Context, arg SelectVtxosWit &i.VtxoVw.ArkTxid, &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, + &i.VtxoVw.Depth, + &i.VtxoVw.MarkerID, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -1647,6 +2079,19 @@ func (q *Queries) SelectVtxosWithPubkeys(ctx context.Context, arg SelectVtxosWit return items, nil } +const sweepVtxosByMarkerId = `-- name: SweepVtxosByMarkerId :execrows +UPDATE vtxo SET swept = true, updated_at = (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)) +WHERE marker_id = ?1 AND swept = false +` + +func (q *Queries) SweepVtxosByMarkerId(ctx context.Context, markerID sql.NullString) (int64, error) { + result, err := q.db.ExecContext(ctx, sweepVtxosByMarkerId, markerID) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + const updateConvictionPardoned = `-- name: UpdateConvictionPardoned :exec UPDATE conviction SET pardoned = true WHERE id = ?1 ` @@ -1686,6 +2131,21 @@ func (q *Queries) UpdateVtxoIntentId(ctx context.Context, arg UpdateVtxoIntentId return err } +const updateVtxoMarkerId = `-- name: UpdateVtxoMarkerId :exec +UPDATE vtxo SET marker_id = ?1 WHERE txid = ?2 AND vout = ?3 +` + +type UpdateVtxoMarkerIdParams struct { + MarkerID sql.NullString + Txid string + Vout int64 +} + +func (q *Queries) UpdateVtxoMarkerId(ctx context.Context, arg UpdateVtxoMarkerIdParams) error { + _, err := q.db.ExecContext(ctx, updateVtxoMarkerId, arg.MarkerID, arg.Txid, arg.Vout) + return err +} + const updateVtxoSettled = `-- name: UpdateVtxoSettled :exec UPDATE vtxo SET spent = true, spent_by = ?1, settled_by = ?2, updated_at = (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)) WHERE txid = ?3 AND vout = ?4 @@ -1855,6 +2315,27 @@ func (q *Queries) UpsertIntent(ctx context.Context, arg UpsertIntentParams) erro return err } +const upsertMarker = `-- name: UpsertMarker :exec + +INSERT INTO marker (id, depth, parent_markers) +VALUES (?1, ?2, ?3) +ON CONFLICT(id) DO UPDATE SET + depth = EXCLUDED.depth, + parent_markers = EXCLUDED.parent_markers +` + +type UpsertMarkerParams struct { + ID string + Depth int64 + ParentMarkers sql.NullString +} + +// Marker queries +func (q *Queries) UpsertMarker(ctx context.Context, arg UpsertMarkerParams) error { + _, err := q.db.ExecContext(ctx, upsertMarker, arg.ID, arg.Depth, arg.ParentMarkers) + return err +} + const upsertOffchainTx = `-- name: UpsertOffchainTx :exec INSERT INTO offchain_tx (txid, tx, starting_timestamp, ending_timestamp, expiry_timestamp, fail_reason, stage_code) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) @@ -2042,11 +2523,11 @@ func (q *Queries) UpsertTx(ctx context.Context, arg UpsertTxParams) error { const upsertVtxo = `-- name: UpsertVtxo :exec INSERT INTO vtxo ( txid, vout, pubkey, amount, commitment_txid, settled_by, ark_txid, - spent_by, spent, unrolled, swept, preconfirmed, expires_at, created_at, updated_at + spent_by, spent, unrolled, swept, preconfirmed, expires_at, created_at, updated_at, depth ) VALUES ( ?1, ?2, ?3, ?4, ?5, ?6, ?7, - ?8, ?9, ?10, ?11, ?12, ?13, ?14, (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)) + ?8, ?9, ?10, ?11, ?12, ?13, ?14, (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)), ?15 ) ON CONFLICT(txid, vout) DO UPDATE SET pubkey = EXCLUDED.pubkey, amount = EXCLUDED.amount, @@ -2060,7 +2541,8 @@ VALUES ( preconfirmed = EXCLUDED.preconfirmed, expires_at = EXCLUDED.expires_at, created_at = EXCLUDED.created_at, - updated_at = (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)) + updated_at = (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)), + depth = EXCLUDED.depth ` type UpsertVtxoParams struct { @@ -2078,6 +2560,7 @@ type UpsertVtxoParams struct { Preconfirmed bool ExpiresAt int64 CreatedAt int64 + Depth int64 } func (q *Queries) UpsertVtxo(ctx context.Context, arg UpsertVtxoParams) error { @@ -2096,6 +2579,7 @@ func (q *Queries) UpsertVtxo(ctx context.Context, arg UpsertVtxoParams) error { arg.Preconfirmed, arg.ExpiresAt, arg.CreatedAt, + arg.Depth, ) return err } diff --git a/internal/infrastructure/db/sqlite/sqlc/query.sql b/internal/infrastructure/db/sqlite/sqlc/query.sql index 8232b18ab..760fbac9f 100644 --- a/internal/infrastructure/db/sqlite/sqlc/query.sql +++ b/internal/infrastructure/db/sqlite/sqlc/query.sql @@ -48,11 +48,11 @@ ON CONFLICT(intent_id, pubkey, onchain_address) DO UPDATE SET -- name: UpsertVtxo :exec INSERT INTO vtxo ( txid, vout, pubkey, amount, commitment_txid, settled_by, ark_txid, - spent_by, spent, unrolled, swept, preconfirmed, expires_at, created_at, updated_at + spent_by, spent, unrolled, swept, preconfirmed, expires_at, created_at, updated_at, depth ) VALUES ( @txid, @vout, @pubkey, @amount, @commitment_txid, @settled_by, @ark_txid, - @spent_by, @spent, @unrolled, @swept, @preconfirmed, @expires_at, @created_at, (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)) + @spent_by, @spent, @unrolled, @swept, @preconfirmed, @expires_at, @created_at, (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)), @depth ) ON CONFLICT(txid, vout) DO UPDATE SET pubkey = EXCLUDED.pubkey, amount = EXCLUDED.amount, @@ -66,7 +66,8 @@ VALUES ( preconfirmed = EXCLUDED.preconfirmed, expires_at = EXCLUDED.expires_at, created_at = EXCLUDED.created_at, - updated_at = (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)); + updated_at = (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)), + depth = EXCLUDED.depth; -- name: InsertVtxoCommitmentTxid :exec INSERT INTO vtxo_commitment_txid (vtxo_txid, vtxo_vout, commitment_txid) @@ -426,4 +427,67 @@ VALUES ('', '', '', ''); -- name: SelectIntentByTxid :one SELECT id, txid, proof, message FROM intent -WHERE txid = @txid; \ No newline at end of file +WHERE txid = @txid; + +-- Marker queries + +-- name: UpsertMarker :exec +INSERT INTO marker (id, depth, parent_markers) +VALUES (@id, @depth, @parent_markers) +ON CONFLICT(id) DO UPDATE SET + depth = EXCLUDED.depth, + parent_markers = EXCLUDED.parent_markers; + +-- name: SelectMarker :one +SELECT * FROM marker WHERE id = @id; + +-- name: SelectMarkersByDepth :many +SELECT * FROM marker WHERE depth = @depth; + +-- name: SelectMarkersByDepthRange :many +SELECT * FROM marker WHERE depth >= @min_depth AND depth <= @max_depth ORDER BY depth; + +-- name: SelectMarkersByIds :many +SELECT * FROM marker WHERE id IN (sqlc.slice('ids')); + +-- name: InsertSweptMarker :exec +INSERT INTO swept_marker (marker_id, swept_at) +VALUES (@marker_id, @swept_at) +ON CONFLICT(marker_id) DO NOTHING; + +-- name: SelectSweptMarker :one +SELECT * FROM swept_marker WHERE marker_id = @marker_id; + +-- name: SelectSweptMarkersByIds :many +SELECT * FROM swept_marker WHERE marker_id IN (sqlc.slice('marker_ids')); + +-- name: IsMarkerSwept :one +SELECT EXISTS(SELECT 1 FROM swept_marker WHERE marker_id = @marker_id) AS is_swept; + +-- name: UpdateVtxoMarkerId :exec +UPDATE vtxo SET marker_id = @marker_id WHERE txid = @txid AND vout = @vout; + +-- name: SelectVtxosByMarkerId :many +SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE marker_id = @marker_id; + +-- name: SweepVtxosByMarkerId :execrows +UPDATE vtxo SET swept = true, updated_at = (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)) +WHERE marker_id = @marker_id AND swept = false; + +-- Chain traversal queries for GetVtxoChain optimization + +-- name: SelectVtxosByDepthRange :many +-- Get all VTXOs within a depth range, useful for filling gaps between markers +SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw +WHERE depth >= @min_depth AND depth <= @max_depth +ORDER BY depth DESC; + +-- name: SelectVtxosByArkTxid :many +-- Get all VTXOs created by a specific ark tx (offchain tx) +SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE txid = @ark_txid; + +-- name: SelectVtxoChainByMarker :many +-- Get VTXOs that share the same marker or have markers in the parent chain +SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw +WHERE marker_id IN (sqlc.slice('marker_ids')) +ORDER BY depth DESC; \ No newline at end of file diff --git a/internal/infrastructure/db/sqlite/vtxo_repo.go b/internal/infrastructure/db/sqlite/vtxo_repo.go index 50ac74d9c..94ef37d7c 100644 --- a/internal/infrastructure/db/sqlite/vtxo_repo.go +++ b/internal/infrastructure/db/sqlite/vtxo_repo.go @@ -57,6 +57,7 @@ func (v *vtxoRepository) AddVtxos(ctx context.Context, vtxos []domain.Vtxo) erro CreatedAt: vtxo.CreatedAt, ArkTxid: sql.NullString{String: vtxo.ArkTxid, Valid: len(vtxo.ArkTxid) > 0}, SettledBy: sql.NullString{String: vtxo.SettledBy, Valid: len(vtxo.SettledBy) > 0}, + Depth: int64(vtxo.Depth), }, ); err != nil { return err @@ -363,7 +364,7 @@ func (v *vtxoRepository) GetAllVtxosWithPubKeys( } res, err := v.querier.SelectVtxosWithPubkeys(ctx, queries.SelectVtxosWithPubkeysParams{ Pubkeys: pubkeys, - After: after, + After: sql.NullInt64{Int64: after, Valid: true}, Before: before, }) if err != nil { @@ -458,7 +459,7 @@ func (v *vtxoRepository) GetPendingSpentVtxosWithPubKeys( ctx, queries.SelectPendingSpentVtxosWithPubkeysParams{ Pubkeys: pubkeys, - After: after, + After: sql.NullInt64{Int64: after, Valid: true}, Before: before, }, ) @@ -536,6 +537,8 @@ func rowToVtxo(row queries.VtxoVw) domain.Vtxo { Preconfirmed: row.Preconfirmed, ExpiresAt: row.ExpiresAt, CreatedAt: row.CreatedAt, + Depth: uint32(row.Depth), + MarkerID: row.MarkerID.String, } } diff --git a/internal/interface/grpc/handlers/indexer.go b/internal/interface/grpc/handlers/indexer.go index e2a74e8d9..ff3a1e1fb 100644 --- a/internal/interface/grpc/handlers/indexer.go +++ b/internal/interface/grpc/handlers/indexer.go @@ -713,5 +713,6 @@ func newIndexerVtxo(vtxo domain.Vtxo) *arkv1.IndexerVtxo { CommitmentTxids: vtxo.CommitmentTxids, SettledBy: vtxo.SettledBy, ArkTxid: vtxo.ArkTxid, + Depth: vtxo.Depth, } } diff --git a/internal/interface/grpc/handlers/parser.go b/internal/interface/grpc/handlers/parser.go index 937f7494f..5d686050f 100644 --- a/internal/interface/grpc/handlers/parser.go +++ b/internal/interface/grpc/handlers/parser.go @@ -190,6 +190,7 @@ func (v vtxoList) toProto() []*arkv1.Vtxo { CreatedAt: vv.CreatedAt, SettledBy: vv.SettledBy, ArkTxid: vv.ArkTxid, + Depth: vv.Depth, }) } From bed30a7fafce18fb80cab5975487c59e01d1fd7b Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Tue, 10 Feb 2026 14:01:01 -0500 Subject: [PATCH 02/54] recursive CTE query for descendant markers, sweep test, prefetchVtxosByMarkers, getVtxosFromCacheOrDB --- internal/core/application/indexer.go | 91 ++++++++++++++++++- internal/core/domain/marker_repo.go | 3 + .../infrastructure/db/postgres/marker_repo.go | 23 +++++ .../db/postgres/sqlc/queries/query.sql.go | 39 ++++++++ .../infrastructure/db/postgres/sqlc/query.sql | 15 +++ internal/infrastructure/db/service.go | 42 +++++---- internal/infrastructure/db/service_test.go | 81 +++++++++++++++++ .../infrastructure/db/sqlite/marker_repo.go | 23 +++++ .../db/sqlite/sqlc/queries/query.sql.go | 39 ++++++++ .../infrastructure/db/sqlite/sqlc/query.sql | 15 +++ 10 files changed, 352 insertions(+), 19 deletions(-) diff --git a/internal/core/application/indexer.go b/internal/core/application/indexer.go index daf1de9ab..4d52a03be 100644 --- a/internal/core/application/indexer.go +++ b/internal/core/application/indexer.go @@ -239,8 +239,11 @@ func (i *indexerService) GetVtxoChain( nextVtxos := []domain.Outpoint{vtxoKey} visited := make(map[string]bool) + // Pre-fetch VTXOs using markers for optimization (reduces DB calls for deep chains) + vtxoCache := i.prefetchVtxosByMarkers(ctx, vtxoKey) + for len(nextVtxos) > 0 { - vtxos, err := i.repoManager.Vtxos().GetVtxos(ctx, nextVtxos) + vtxos, err := i.getVtxosFromCacheOrDB(ctx, nextVtxos, vtxoCache) if err != nil { return nil, err } @@ -369,6 +372,92 @@ func (i *indexerService) GetVtxoChain( }, nil } +// prefetchVtxosByMarkers pre-fetches VTXOs using markers for optimization. +// This reduces the number of DB calls for deep chains by bulk fetching VTXOs +// associated with the marker chain instead of fetching one at a time. +func (i *indexerService) prefetchVtxosByMarkers( + ctx context.Context, startKey Outpoint, +) map[string]domain.Vtxo { + cache := make(map[string]domain.Vtxo) + + if i.repoManager.Markers() == nil { + return cache + } + + // Get starting VTXO to find its marker + startVtxos, err := i.repoManager.Vtxos().GetVtxos(ctx, []domain.Outpoint{startKey}) + if err != nil || len(startVtxos) == 0 { + return cache + } + + startVtxo := startVtxos[0] + // Add starting VTXO to cache + cache[startVtxo.Outpoint.String()] = startVtxo + + if startVtxo.MarkerID == "" { + return cache + } + + // Collect marker chain by following ParentMarkerIDs + markerIDs := []string{startVtxo.MarkerID} + marker, err := i.repoManager.Markers().GetMarker(ctx, startVtxo.MarkerID) + if err != nil { + return cache + } + + // Follow the marker chain up to the root (depth 0) + for marker != nil && len(marker.ParentMarkerIDs) > 0 { + markerIDs = append(markerIDs, marker.ParentMarkerIDs...) + // Follow first parent marker to continue chain + marker, _ = i.repoManager.Markers().GetMarker(ctx, marker.ParentMarkerIDs[0]) + } + + // Bulk fetch VTXOs for all markers in the chain + vtxos, err := i.repoManager.Markers().GetVtxoChainByMarkers(ctx, markerIDs) + if err != nil { + return cache + } + + for _, v := range vtxos { + cache[v.Outpoint.String()] = v + } + + return cache +} + +// getVtxosFromCacheOrDB retrieves VTXOs from cache first, falling back to DB for cache misses. +// This is used in conjunction with prefetchVtxosByMarkers to reduce DB calls. +func (i *indexerService) getVtxosFromCacheOrDB( + ctx context.Context, + outpoints []domain.Outpoint, + cache map[string]domain.Vtxo, +) ([]domain.Vtxo, error) { + result := make([]domain.Vtxo, 0, len(outpoints)) + missingOutpoints := make([]domain.Outpoint, 0) + + for _, op := range outpoints { + if v, ok := cache[op.String()]; ok { + result = append(result, v) + } else { + missingOutpoints = append(missingOutpoints, op) + } + } + + if len(missingOutpoints) > 0 { + dbVtxos, err := i.repoManager.Vtxos().GetVtxos(ctx, missingOutpoints) + if err != nil { + return nil, err + } + result = append(result, dbVtxos...) + // Add to cache for future lookups in this chain traversal + for _, v := range dbVtxos { + cache[v.Outpoint.String()] = v + } + } + + return result, nil +} + func (i *indexerService) GetVirtualTxs( ctx context.Context, txids []string, page *Page, ) (*VirtualTxsResp, error) { diff --git a/internal/core/domain/marker_repo.go b/internal/core/domain/marker_repo.go index b05b26be4..7ea035f14 100644 --- a/internal/core/domain/marker_repo.go +++ b/internal/core/domain/marker_repo.go @@ -16,6 +16,9 @@ type MarkerRepository interface { // SweepMarker marks a marker as swept at the given timestamp SweepMarker(ctx context.Context, markerID string, sweptAt int64) error + // SweepMarkerWithDescendants marks a marker and all its descendants as swept + // Returns the number of markers swept (including descendants) + SweepMarkerWithDescendants(ctx context.Context, markerID string, sweptAt int64) (int64, error) // IsMarkerSwept checks if a marker has been swept IsMarkerSwept(ctx context.Context, markerID string) (bool, error) // GetSweptMarkers retrieves swept marker records for the given marker IDs diff --git a/internal/infrastructure/db/postgres/marker_repo.go b/internal/infrastructure/db/postgres/marker_repo.go index d8c8da7ca..356ad0afb 100644 --- a/internal/infrastructure/db/postgres/marker_repo.go +++ b/internal/infrastructure/db/postgres/marker_repo.go @@ -133,6 +133,29 @@ func (m *markerRepository) SweepMarker(ctx context.Context, markerID string, swe }) } +func (m *markerRepository) SweepMarkerWithDescendants(ctx context.Context, markerID string, sweptAt int64) (int64, error) { + // Get all descendant marker IDs (including the root marker) that are not already swept + descendantIDs, err := m.querier.GetDescendantMarkerIds(ctx, markerID) + if err != nil { + return 0, fmt.Errorf("failed to get descendant markers: %w", err) + } + + // Insert each descendant into swept_marker + var count int64 + for _, id := range descendantIDs { + err := m.querier.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ + MarkerID: id, + SweptAt: sweptAt, + }) + if err != nil { + return count, fmt.Errorf("failed to sweep marker %s: %w", id, err) + } + count++ + } + + return count, nil +} + func (m *markerRepository) IsMarkerSwept(ctx context.Context, markerID string) (bool, error) { result, err := m.querier.IsMarkerSwept(ctx, markerID) if err != nil { diff --git a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go index 2aabead78..c4ccb8188 100644 --- a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go @@ -86,6 +86,45 @@ func (q *Queries) ClearScheduledSession(ctx context.Context) error { return err } +const getDescendantMarkerIds = `-- name: GetDescendantMarkerIds :many +WITH RECURSIVE descendant_markers(id) AS ( + -- Base case: the marker being swept + SELECT marker.id FROM marker WHERE marker.id = $1 + UNION ALL + -- Recursive case: find markers whose parent_markers jsonb array contains any descendant + SELECT m.id FROM marker m + INNER JOIN descendant_markers dm ON ( + m.parent_markers @> jsonb_build_array(dm.id) + ) +) +SELECT descendant_markers.id AS marker_id FROM descendant_markers +WHERE descendant_markers.id NOT IN (SELECT sm.marker_id FROM swept_marker sm) +` + +// Recursively get a marker and all its descendants (markers whose parent_markers contain it) +func (q *Queries) GetDescendantMarkerIds(ctx context.Context, rootMarkerID string) ([]string, error) { + rows, err := q.db.QueryContext(ctx, getDescendantMarkerIds, rootMarkerID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var marker_id string + if err := rows.Scan(&marker_id); err != nil { + return nil, err + } + items = append(items, marker_id) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const insertSweptMarker = `-- name: InsertSweptMarker :exec INSERT INTO swept_marker (marker_id, swept_at) VALUES ($1, $2) diff --git a/internal/infrastructure/db/postgres/sqlc/query.sql b/internal/infrastructure/db/postgres/sqlc/query.sql index cb90f6cf3..83b12187c 100644 --- a/internal/infrastructure/db/postgres/sqlc/query.sql +++ b/internal/infrastructure/db/postgres/sqlc/query.sql @@ -461,6 +461,21 @@ SELECT * FROM swept_marker WHERE marker_id = ANY(@marker_ids::text[]); -- name: IsMarkerSwept :one SELECT EXISTS(SELECT 1 FROM swept_marker WHERE marker_id = @marker_id) AS is_swept; +-- name: GetDescendantMarkerIds :many +-- Recursively get a marker and all its descendants (markers whose parent_markers contain it) +WITH RECURSIVE descendant_markers(id) AS ( + -- Base case: the marker being swept + SELECT marker.id FROM marker WHERE marker.id = @root_marker_id + UNION ALL + -- Recursive case: find markers whose parent_markers jsonb array contains any descendant + SELECT m.id FROM marker m + INNER JOIN descendant_markers dm ON ( + m.parent_markers @> jsonb_build_array(dm.id) + ) +) +SELECT descendant_markers.id AS marker_id FROM descendant_markers +WHERE descendant_markers.id NOT IN (SELECT sm.marker_id FROM swept_marker sm); + -- name: UpdateVtxoMarkerId :exec UPDATE vtxo SET marker_id = @marker_id WHERE txid = @txid AND vout = @vout; diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index e17bbe1b1..375d508ec 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -579,8 +579,9 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) maxDepth = v.Depth } // Collect parent marker IDs for marker linking (will be used at boundary) - // Note: We need to get marker_id from the VTXO, which requires the field to be added to domain.Vtxo - // For now, we'll create markers without parent links - this can be enhanced in a follow-up + if v.MarkerID != "" { + parentMarkerSet[v.MarkerID] = struct{}{} + } } newDepth = maxDepth + 1 for id := range parentMarkerSet { @@ -589,22 +590,27 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) } } - // Create marker if at boundary depth + // Create marker if at boundary depth, or inherit from parent var markerID string - if s.markerStore != nil && domain.IsAtMarkerBoundary(newDepth) { - // Create marker ID from the first output (the ark tx id + first vtxo vout) - markerID = fmt.Sprintf("%s:marker:%d", txid, newDepth) - marker := domain.Marker{ - ID: markerID, - Depth: newDepth, - ParentMarkerIDs: parentMarkerIDs, - } - if err := s.markerStore.AddMarker(ctx, marker); err != nil { - log.WithError(err).Warn("failed to create marker for chained vtxo") - // Continue without marker - non-fatal - markerID = "" - } else { - log.Debugf("created marker %s at depth %d", markerID, newDepth) + if s.markerStore != nil { + if domain.IsAtMarkerBoundary(newDepth) { + // Create marker ID from the first output (the ark tx id + first vtxo vout) + markerID = fmt.Sprintf("%s:marker:%d", txid, newDepth) + marker := domain.Marker{ + ID: markerID, + Depth: newDepth, + ParentMarkerIDs: parentMarkerIDs, + } + if err := s.markerStore.AddMarker(ctx, marker); err != nil { + log.WithError(err).Warn("failed to create marker for chained vtxo") + // Continue without marker - non-fatal + markerID = "" + } else { + log.Debugf("created marker %s at depth %d", markerID, newDepth) + } + } else if len(parentMarkerIDs) > 0 { + // Inherit marker from parent at non-boundary depth + markerID = parentMarkerIDs[0] } } @@ -645,7 +651,7 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) } log.Debugf("added %d vtxos at depth %d", len(newVtxos), newDepth) - // Update marker_id for VTXOs at boundary depth + // Update marker_id for VTXOs (new marker at boundary, inherited at non-boundary) if markerID != "" && s.markerStore != nil { for _, vtxo := range newVtxos { if err := s.markerStore.UpdateVtxoMarker(ctx, vtxo.Outpoint, markerID); err != nil { diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index 1003ba174..8c14924e0 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -1540,6 +1540,87 @@ func testMarkerSweep(t *testing.T, svc ports.RepoManager) { require.NoError(t, err) require.False(t, isSwept) }) + + t.Run("test_sweep_marker_with_descendants", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + + // Create a marker hierarchy: + // root -> child1 -> grandchild1 + // -> child2 + root := domain.Marker{ + ID: "sweep_desc_root_" + randomString(16), + Depth: 0, + ParentMarkerIDs: nil, + } + child1 := domain.Marker{ + ID: "sweep_desc_child1_" + randomString(16), + Depth: 100, + ParentMarkerIDs: []string{root.ID}, + } + child2 := domain.Marker{ + ID: "sweep_desc_child2_" + randomString(16), + Depth: 100, + ParentMarkerIDs: []string{root.ID}, + } + grandchild1 := domain.Marker{ + ID: "sweep_desc_grandchild1_" + randomString(16), + Depth: 200, + ParentMarkerIDs: []string{child1.ID}, + } + + err := svc.Markers().AddMarker(ctx, root) + require.NoError(t, err) + err = svc.Markers().AddMarker(ctx, child1) + require.NoError(t, err) + err = svc.Markers().AddMarker(ctx, child2) + require.NoError(t, err) + err = svc.Markers().AddMarker(ctx, grandchild1) + require.NoError(t, err) + + // Verify none are swept initially + isSwept, err := svc.Markers().IsMarkerSwept(ctx, root.ID) + require.NoError(t, err) + require.False(t, isSwept) + + // Sweep root with descendants + sweptAt := time.Now().UnixMilli() + count, err := svc.Markers().SweepMarkerWithDescendants(ctx, root.ID, sweptAt) + require.NoError(t, err) + require.Equal(t, int64(4), count) // root + child1 + child2 + grandchild1 + + // Verify all markers are now swept + for _, m := range []domain.Marker{root, child1, child2, grandchild1} { + isSwept, err := svc.Markers().IsMarkerSwept(ctx, m.ID) + require.NoError(t, err) + require.True(t, isSwept, "Marker %s should be swept", m.ID) + } + + // Test idempotency - calling again should return 0 + count, err = svc.Markers().SweepMarkerWithDescendants(ctx, root.ID, sweptAt+1000) + require.NoError(t, err) + require.Equal(t, int64(0), count) + + // Test sweeping a leaf node (no descendants) + leaf := domain.Marker{ + ID: "sweep_desc_leaf_" + randomString(16), + Depth: 300, + ParentMarkerIDs: []string{grandchild1.ID}, + } + err = svc.Markers().AddMarker(ctx, leaf) + require.NoError(t, err) + + count, err = svc.Markers().SweepMarkerWithDescendants(ctx, leaf.ID, sweptAt) + require.NoError(t, err) + require.Equal(t, int64(1), count) // Just the leaf itself + + // Test with non-existent marker (should return 0) + count, err = svc.Markers().SweepMarkerWithDescendants(ctx, "nonexistent", sweptAt) + require.NoError(t, err) + require.Equal(t, int64(0), count) + }) } func testVtxoMarkerAssociation(t *testing.T, svc ports.RepoManager) { diff --git a/internal/infrastructure/db/sqlite/marker_repo.go b/internal/infrastructure/db/sqlite/marker_repo.go index fc94fbca8..a8d7e97c9 100644 --- a/internal/infrastructure/db/sqlite/marker_repo.go +++ b/internal/infrastructure/db/sqlite/marker_repo.go @@ -129,6 +129,29 @@ func (m *markerRepository) SweepMarker(ctx context.Context, markerID string, swe }) } +func (m *markerRepository) SweepMarkerWithDescendants(ctx context.Context, markerID string, sweptAt int64) (int64, error) { + // Get all descendant marker IDs (including the root marker) that are not already swept + descendantIDs, err := m.querier.GetDescendantMarkerIds(ctx, markerID) + if err != nil { + return 0, fmt.Errorf("failed to get descendant markers: %w", err) + } + + // Insert each descendant into swept_marker + var count int64 + for _, id := range descendantIDs { + err := m.querier.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ + MarkerID: id, + SweptAt: sweptAt, + }) + if err != nil { + return count, fmt.Errorf("failed to sweep marker %s: %w", id, err) + } + count++ + } + + return count, nil +} + func (m *markerRepository) IsMarkerSwept(ctx context.Context, markerID string) (bool, error) { result, err := m.querier.IsMarkerSwept(ctx, markerID) if err != nil { diff --git a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go index b02828b64..a415641c8 100644 --- a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go @@ -84,6 +84,45 @@ func (q *Queries) ClearScheduledSession(ctx context.Context) error { return err } +const getDescendantMarkerIds = `-- name: GetDescendantMarkerIds :many +WITH RECURSIVE descendant_markers(id) AS ( + -- Base case: the marker being swept + SELECT marker.id FROM marker WHERE marker.id = ?1 + UNION ALL + -- Recursive case: find markers whose parent_markers JSON array contains any descendant + SELECT m.id FROM marker m + INNER JOIN descendant_markers dm ON ( + m.parent_markers LIKE '%"' || dm.id || '"%' + ) +) +SELECT descendant_markers.id AS marker_id FROM descendant_markers +WHERE descendant_markers.id NOT IN (SELECT sm.marker_id FROM swept_marker sm) +` + +// Recursively get a marker and all its descendants (markers whose parent_markers contain it) +func (q *Queries) GetDescendantMarkerIds(ctx context.Context, rootMarkerID string) ([]string, error) { + rows, err := q.db.QueryContext(ctx, getDescendantMarkerIds, rootMarkerID) + if err != nil { + return nil, err + } + defer rows.Close() + var items []string + for rows.Next() { + var marker_id string + if err := rows.Scan(&marker_id); err != nil { + return nil, err + } + items = append(items, marker_id) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const insertSweptMarker = `-- name: InsertSweptMarker :exec INSERT INTO swept_marker (marker_id, swept_at) VALUES (?1, ?2) diff --git a/internal/infrastructure/db/sqlite/sqlc/query.sql b/internal/infrastructure/db/sqlite/sqlc/query.sql index 760fbac9f..88bd31d87 100644 --- a/internal/infrastructure/db/sqlite/sqlc/query.sql +++ b/internal/infrastructure/db/sqlite/sqlc/query.sql @@ -464,6 +464,21 @@ SELECT * FROM swept_marker WHERE marker_id IN (sqlc.slice('marker_ids')); -- name: IsMarkerSwept :one SELECT EXISTS(SELECT 1 FROM swept_marker WHERE marker_id = @marker_id) AS is_swept; +-- name: GetDescendantMarkerIds :many +-- Recursively get a marker and all its descendants (markers whose parent_markers contain it) +WITH RECURSIVE descendant_markers(id) AS ( + -- Base case: the marker being swept + SELECT marker.id FROM marker WHERE marker.id = @root_marker_id + UNION ALL + -- Recursive case: find markers whose parent_markers JSON array contains any descendant + SELECT m.id FROM marker m + INNER JOIN descendant_markers dm ON ( + m.parent_markers LIKE '%"' || dm.id || '"%' + ) +) +SELECT descendant_markers.id AS marker_id FROM descendant_markers +WHERE descendant_markers.id NOT IN (SELECT sm.marker_id FROM swept_marker sm); + -- name: UpdateVtxoMarkerId :exec UPDATE vtxo SET marker_id = @marker_id WHERE txid = @txid AND vout = @vout; From e8fbb86a576667242640b17434d0da81e5fff754 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Tue, 10 Feb 2026 14:18:50 -0500 Subject: [PATCH 03/54] testGetVtxoChainWithMarkerOptimization, linting --- .../infrastructure/db/postgres/marker_repo.go | 76 ++++--- internal/infrastructure/db/service.go | 14 +- internal/infrastructure/db/service_test.go | 207 +++++++++++++++++- .../infrastructure/db/sqlite/marker_repo.go | 62 +++++- 4 files changed, 311 insertions(+), 48 deletions(-) diff --git a/internal/infrastructure/db/postgres/marker_repo.go b/internal/infrastructure/db/postgres/marker_repo.go index 356ad0afb..dc3fc39ac 100644 --- a/internal/infrastructure/db/postgres/marker_repo.go +++ b/internal/infrastructure/db/postgres/marker_repo.go @@ -1,7 +1,6 @@ package pgdb import ( - "bytes" "context" "database/sql" "encoding/json" @@ -68,7 +67,10 @@ func (m *markerRepository) GetMarker(ctx context.Context, id string) (*domain.Ma return &marker, nil } -func (m *markerRepository) GetMarkersByDepth(ctx context.Context, depth uint32) ([]domain.Marker, error) { +func (m *markerRepository) GetMarkersByDepth( + ctx context.Context, + depth uint32, +) ([]domain.Marker, error) { rows, err := m.querier.SelectMarkersByDepth(ctx, int32(depth)) if err != nil { return nil, err @@ -85,7 +87,10 @@ func (m *markerRepository) GetMarkersByDepth(ctx context.Context, depth uint32) return markers, nil } -func (m *markerRepository) GetMarkersByDepthRange(ctx context.Context, minDepth, maxDepth uint32) ([]domain.Marker, error) { +func (m *markerRepository) GetMarkersByDepthRange( + ctx context.Context, + minDepth, maxDepth uint32, +) ([]domain.Marker, error) { rows, err := m.querier.SelectMarkersByDepthRange(ctx, queries.SelectMarkersByDepthRangeParams{ MinDepth: int32(minDepth), MaxDepth: int32(maxDepth), @@ -105,7 +110,10 @@ func (m *markerRepository) GetMarkersByDepthRange(ctx context.Context, minDepth, return markers, nil } -func (m *markerRepository) GetMarkersByIds(ctx context.Context, ids []string) ([]domain.Marker, error) { +func (m *markerRepository) GetMarkersByIds( + ctx context.Context, + ids []string, +) ([]domain.Marker, error) { if len(ids) == 0 { return nil, nil } @@ -133,7 +141,11 @@ func (m *markerRepository) SweepMarker(ctx context.Context, markerID string, swe }) } -func (m *markerRepository) SweepMarkerWithDescendants(ctx context.Context, markerID string, sweptAt int64) (int64, error) { +func (m *markerRepository) SweepMarkerWithDescendants( + ctx context.Context, + markerID string, + sweptAt int64, +) (int64, error) { // Get all descendant marker IDs (including the root marker) that are not already swept descendantIDs, err := m.querier.GetDescendantMarkerIds(ctx, markerID) if err != nil { @@ -164,7 +176,10 @@ func (m *markerRepository) IsMarkerSwept(ctx context.Context, markerID string) ( return result, nil } -func (m *markerRepository) GetSweptMarkers(ctx context.Context, markerIDs []string) ([]domain.SweptMarker, error) { +func (m *markerRepository) GetSweptMarkers( + ctx context.Context, + markerIDs []string, +) ([]domain.SweptMarker, error) { if len(markerIDs) == 0 { return nil, nil } @@ -184,7 +199,11 @@ func (m *markerRepository) GetSweptMarkers(ctx context.Context, markerIDs []stri return sweptMarkers, nil } -func (m *markerRepository) UpdateVtxoMarker(ctx context.Context, outpoint domain.Outpoint, markerID string) error { +func (m *markerRepository) UpdateVtxoMarker( + ctx context.Context, + outpoint domain.Outpoint, + markerID string, +) error { return m.querier.UpdateVtxoMarkerId(ctx, queries.UpdateVtxoMarkerIdParams{ MarkerID: sql.NullString{String: markerID, Valid: len(markerID) > 0}, Txid: outpoint.Txid, @@ -192,8 +211,14 @@ func (m *markerRepository) UpdateVtxoMarker(ctx context.Context, outpoint domain }) } -func (m *markerRepository) GetVtxosByMarker(ctx context.Context, markerID string) ([]domain.Vtxo, error) { - rows, err := m.querier.SelectVtxosByMarkerId(ctx, sql.NullString{String: markerID, Valid: len(markerID) > 0}) +func (m *markerRepository) GetVtxosByMarker( + ctx context.Context, + markerID string, +) ([]domain.Vtxo, error) { + rows, err := m.querier.SelectVtxosByMarkerId( + ctx, + sql.NullString{String: markerID, Valid: len(markerID) > 0}, + ) if err != nil { return nil, err } @@ -206,10 +231,16 @@ func (m *markerRepository) GetVtxosByMarker(ctx context.Context, markerID string } func (m *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID string) (int64, error) { - return m.querier.SweepVtxosByMarkerId(ctx, sql.NullString{String: markerID, Valid: len(markerID) > 0}) + return m.querier.SweepVtxosByMarkerId( + ctx, + sql.NullString{String: markerID, Valid: len(markerID) > 0}, + ) } -func (m *markerRepository) GetVtxosByDepthRange(ctx context.Context, minDepth, maxDepth uint32) ([]domain.Vtxo, error) { +func (m *markerRepository) GetVtxosByDepthRange( + ctx context.Context, + minDepth, maxDepth uint32, +) ([]domain.Vtxo, error) { rows, err := m.querier.SelectVtxosByDepthRange(ctx, queries.SelectVtxosByDepthRangeParams{ MinDepth: int32(minDepth), MaxDepth: int32(maxDepth), @@ -225,7 +256,10 @@ func (m *markerRepository) GetVtxosByDepthRange(ctx context.Context, minDepth, m return vtxos, nil } -func (m *markerRepository) GetVtxosByArkTxid(ctx context.Context, arkTxid string) ([]domain.Vtxo, error) { +func (m *markerRepository) GetVtxosByArkTxid( + ctx context.Context, + arkTxid string, +) ([]domain.Vtxo, error) { rows, err := m.querier.SelectVtxosByArkTxid(ctx, arkTxid) if err != nil { return nil, err @@ -238,7 +272,10 @@ func (m *markerRepository) GetVtxosByArkTxid(ctx context.Context, arkTxid string return vtxos, nil } -func (m *markerRepository) GetVtxoChainByMarkers(ctx context.Context, markerIDs []string) ([]domain.Vtxo, error) { +func (m *markerRepository) GetVtxoChainByMarkers( + ctx context.Context, + markerIDs []string, +) ([]domain.Vtxo, error) { if len(markerIDs) == 0 { return nil, nil } @@ -318,16 +355,3 @@ func rowToVtxoFromMarkerQuery(row queries.SelectVtxosByMarkerIdRow) domain.Vtxo MarkerID: row.VtxoVw.MarkerID.String, } } - -// parseCommitmentsBytes is a local helper function that handles nil slices -func parseCommitmentsBytes(commitments []byte, separator []byte) []string { - if len(commitments) == 0 { - return nil - } - parts := bytes.Split(commitments, separator) - commitmentsStr := make([]string, 0, len(parts)) - for _, p := range parts { - commitmentsStr = append(commitmentsStr, string(p)) - } - return commitmentsStr -} diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index 375d508ec..edbc5e33e 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -502,11 +502,13 @@ func (s *service) updateProjectionsAfterRoundEvents(events []domain.Event) { ParentMarkerIDs: nil, // Root markers have no parents } if err := s.markerStore.AddMarker(ctx, marker); err != nil { - log.WithError(err).Warnf("failed to create root marker for vtxo %s", vtxo.Outpoint.String()) + log.WithError(err). + Warnf("failed to create root marker for vtxo %s", vtxo.Outpoint.String()) continue } if err := s.markerStore.UpdateVtxoMarker(ctx, vtxo.Outpoint, markerID); err != nil { - log.WithError(err).Warnf("failed to update marker_id for vtxo %s", vtxo.Outpoint.String()) + log.WithError(err). + Warnf("failed to update marker_id for vtxo %s", vtxo.Outpoint.String()) } } log.Debugf("created %d root markers for batch vtxos", len(newVtxos)) @@ -655,7 +657,8 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) if markerID != "" && s.markerStore != nil { for _, vtxo := range newVtxos { if err := s.markerStore.UpdateVtxoMarker(ctx, vtxo.Outpoint, markerID); err != nil { - log.WithError(err).Warnf("failed to update marker_id for vtxo %s", vtxo.Outpoint.String()) + log.WithError(err). + Warnf("failed to update marker_id for vtxo %s", vtxo.Outpoint.String()) } } } @@ -735,7 +738,10 @@ func getNewVtxosFromRound(round *domain.Round) []domain.Vtxo { // sweepVtxosWithMarkers performs marker-based sweeping for VTXOs. // It groups VTXOs by their marker, sweeps each marker, then bulk-updates all VTXOs. // Returns the total count of VTXOs swept. -func (s *service) sweepVtxosWithMarkers(ctx context.Context, vtxoOutpoints []domain.Outpoint) int64 { +func (s *service) sweepVtxosWithMarkers( + ctx context.Context, + vtxoOutpoints []domain.Outpoint, +) int64 { if len(vtxoOutpoints) == 0 { return 0 } diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index 8c14924e0..de9eb8450 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -192,6 +192,7 @@ func TestService(t *testing.T) { testSweepVtxosByMarker(t, svc) testMarkerDepthRangeQueries(t, svc) testMarkerChainTraversal(t, svc) + testGetVtxoChainWithMarkerOptimization(t, svc) testOffchainTxRepository(t, svc) testScheduledSessionRepository(t, svc) testConvictionRepository(t, svc) @@ -1443,7 +1444,8 @@ func testMarkerBasicOperations(t *testing.T, svc ports.RepoManager) { require.True(t, foundMarker4) // Test GetMarkersByIds - batch retrieve - markersById, err := svc.Markers().GetMarkersByIds(ctx, []string{marker1.ID, marker3.ID, marker4.ID}) + markersById, err := svc.Markers(). + GetMarkersByIds(ctx, []string{marker1.ID, marker3.ID, marker4.ID}) require.NoError(t, err) require.Len(t, markersById, 3) retrievedIds := make([]string, len(markersById)) @@ -1686,11 +1688,19 @@ func testVtxoMarkerAssociation(t *testing.T, svc ports.RepoManager) { vtxosByMarker, err := svc.Markers().GetVtxosByMarker(ctx, markerID) require.NoError(t, err) require.Len(t, vtxosByMarker, 2) - outpoints := []string{vtxosByMarker[0].Outpoint.String(), vtxosByMarker[1].Outpoint.String()} - require.ElementsMatch(t, []string{vtxo1.Outpoint.String(), vtxo2.Outpoint.String()}, outpoints) + outpoints := []string{ + vtxosByMarker[0].Outpoint.String(), + vtxosByMarker[1].Outpoint.String(), + } + require.ElementsMatch( + t, + []string{vtxo1.Outpoint.String(), vtxo2.Outpoint.String()}, + outpoints, + ) // Verify VTXO.MarkerID field is populated when retrieved via GetVtxos - retrievedVtxos, err = svc.Vtxos().GetVtxos(ctx, []domain.Outpoint{vtxo1.Outpoint, vtxo2.Outpoint}) + retrievedVtxos, err = svc.Vtxos(). + GetVtxos(ctx, []domain.Outpoint{vtxo1.Outpoint, vtxo2.Outpoint}) require.NoError(t, err) require.Len(t, retrievedVtxos, 2) for _, v := range retrievedVtxos { @@ -1905,7 +1915,8 @@ func testMarkerDepthRangeQueries(t *testing.T, svc ports.RepoManager) { Depth: 150, } - err = svc.Vtxos().AddVtxos(ctx, []domain.Vtxo{vtxoDepth0, vtxoDepth50, vtxoDepth100, vtxoDepth150}) + err = svc.Vtxos(). + AddVtxos(ctx, []domain.Vtxo{vtxoDepth0, vtxoDepth50, vtxoDepth100, vtxoDepth150}) require.NoError(t, err) // Test GetVtxosByDepthRange(25, 125) - should return VTXOs at 50 and 100 @@ -2031,7 +2042,8 @@ func testMarkerChainTraversal(t *testing.T, svc ports.RepoManager) { require.True(t, foundTxids[vtxo2.Txid]) // Test with both markers - vtxosByMarkers, err = svc.Markers().GetVtxoChainByMarkers(ctx, []string{marker1.ID, marker2.ID}) + vtxosByMarkers, err = svc.Markers(). + GetVtxoChainByMarkers(ctx, []string{marker1.ID, marker2.ID}) require.NoError(t, err) require.Len(t, vtxosByMarkers, 3) @@ -2058,6 +2070,189 @@ func testMarkerChainTraversal(t *testing.T, svc ports.RepoManager) { }) } +// testGetVtxoChainWithMarkerOptimization tests that GetVtxoChain correctly +// traverses a deep VTXO chain and uses marker-based prefetching. +// This verifies: +// 1. Markers are correctly created at depth boundaries (0, 100, 200) +// 2. VTXOs have correct marker assignments +// 3. GetVtxoChainByMarkers returns all VTXOs for the marker chain +func testGetVtxoChainWithMarkerOptimization(t *testing.T, svc ports.RepoManager) { + t.Run("test_get_vtxo_chain_with_marker_optimization", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + commitmentTxid := randomString(32) + + // Create markers at depths 0, 100, 200 (simulating a chain spanning 250 depths) + marker0 := domain.Marker{ + ID: "opt_marker_0_" + randomString(16), + Depth: 0, + ParentMarkerIDs: nil, + } + marker100 := domain.Marker{ + ID: "opt_marker_100_" + randomString(16), + Depth: 100, + ParentMarkerIDs: []string{marker0.ID}, + } + marker200 := domain.Marker{ + ID: "opt_marker_200_" + randomString(16), + Depth: 200, + ParentMarkerIDs: []string{marker100.ID}, + } + + err := svc.Markers().AddMarker(ctx, marker0) + require.NoError(t, err) + err = svc.Markers().AddMarker(ctx, marker100) + require.NoError(t, err) + err = svc.Markers().AddMarker(ctx, marker200) + require.NoError(t, err) + + // Create VTXOs at various depths across the marker boundaries: + // - VTXOs at depth 0-99 should have marker0.ID + // - VTXOs at depth 100-199 should have marker100.ID + // - VTXOs at depth 200-250 should have marker200.ID + vtxos := make([]domain.Vtxo, 0) + vtxoMarkerMap := make(map[string]string) // outpoint -> markerID + + // Helper to determine which marker a VTXO should have based on depth + getMarkerForDepth := func(depth uint32) string { + if depth >= 200 { + return marker200.ID + } else if depth >= 100 { + return marker100.ID + } + return marker0.ID + } + + // Create VTXOs at sample depths: 0, 50, 99, 100, 150, 199, 200, 225, 250 + sampleDepths := []uint32{0, 50, 99, 100, 150, 199, 200, 225, 250} + for i, depth := range sampleDepths { + vtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{ + Txid: "opt_chain_vtxo_" + randomString(16), + VOut: uint32(i), + }, + PubKey: pubkey, + Amount: uint64(1000 * (i + 1)), + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: depth, + } + vtxos = append(vtxos, vtxo) + vtxoMarkerMap[vtxo.Outpoint.String()] = getMarkerForDepth(depth) + } + + // Add all VTXOs + err = svc.Vtxos().AddVtxos(ctx, vtxos) + require.NoError(t, err) + + // Associate VTXOs with their markers + for _, v := range vtxos { + markerID := vtxoMarkerMap[v.Outpoint.String()] + err = svc.Markers().UpdateVtxoMarker(ctx, v.Outpoint, markerID) + require.NoError(t, err) + } + + // Verify each VTXO has the correct marker assigned + for _, v := range vtxos { + retrievedVtxos, err := svc.Vtxos().GetVtxos(ctx, []domain.Outpoint{v.Outpoint}) + require.NoError(t, err) + require.Len(t, retrievedVtxos, 1) + expectedMarker := vtxoMarkerMap[v.Outpoint.String()] + require.Equal(t, expectedMarker, retrievedVtxos[0].MarkerID, + "VTXO at depth %d should have marker %s", v.Depth, expectedMarker) + } + + // Test 1: Query VTXOs using the full marker chain (marker200 -> marker100 -> marker0) + // This simulates what prefetchVtxosByMarkers does + fullMarkerChain := []string{marker200.ID, marker100.ID, marker0.ID} + allChainVtxos, err := svc.Markers().GetVtxoChainByMarkers(ctx, fullMarkerChain) + require.NoError(t, err) + require.Len(t, allChainVtxos, len(vtxos), "Should return all VTXOs in the chain") + + // Verify all our VTXOs are in the result + resultOutpoints := make(map[string]bool) + for _, v := range allChainVtxos { + resultOutpoints[v.Outpoint.String()] = true + } + for _, v := range vtxos { + require.True(t, resultOutpoints[v.Outpoint.String()], + "VTXO %s at depth %d should be in result", v.Outpoint.String(), v.Depth) + } + + // Test 2: Query with just marker0 - should return only depth 0-99 VTXOs + marker0Vtxos, err := svc.Markers().GetVtxoChainByMarkers(ctx, []string{marker0.ID}) + require.NoError(t, err) + for _, v := range marker0Vtxos { + // Only check our test VTXOs (filter by prefix) + if len(v.Txid) > 0 && v.Txid[:13] == "opt_chain_vtx" { + require.True(t, v.Depth < 100, + "VTXOs with marker0 should have depth < 100, got depth %d", v.Depth) + } + } + + // Test 3: Query with marker200 only - should return only depth 200+ VTXOs + marker200Vtxos, err := svc.Markers().GetVtxoChainByMarkers(ctx, []string{marker200.ID}) + require.NoError(t, err) + for _, v := range marker200Vtxos { + if len(v.Txid) > 0 && v.Txid[:13] == "opt_chain_vtx" { + require.True(t, v.Depth >= 200, + "VTXOs with marker200 should have depth >= 200, got depth %d", v.Depth) + } + } + + // Test 4: Verify marker chain can be followed via ParentMarkerIDs + // Starting from marker200, should be able to traverse to marker0 + currentMarker, err := svc.Markers().GetMarker(ctx, marker200.ID) + require.NoError(t, err) + require.NotNil(t, currentMarker) + require.Equal(t, uint32(200), currentMarker.Depth) + require.Len(t, currentMarker.ParentMarkerIDs, 1) + require.Equal(t, marker100.ID, currentMarker.ParentMarkerIDs[0]) + + currentMarker, err = svc.Markers().GetMarker(ctx, currentMarker.ParentMarkerIDs[0]) + require.NoError(t, err) + require.NotNil(t, currentMarker) + require.Equal(t, uint32(100), currentMarker.Depth) + require.Len(t, currentMarker.ParentMarkerIDs, 1) + require.Equal(t, marker0.ID, currentMarker.ParentMarkerIDs[0]) + + currentMarker, err = svc.Markers().GetMarker(ctx, currentMarker.ParentMarkerIDs[0]) + require.NoError(t, err) + require.NotNil(t, currentMarker) + require.Equal(t, uint32(0), currentMarker.Depth) + require.Nil(t, currentMarker.ParentMarkerIDs) // Root marker has no parents + + // Test 5: Test GetMarkersByIds with the full chain + markers, err := svc.Markers().GetMarkersByIds(ctx, fullMarkerChain) + require.NoError(t, err) + require.Len(t, markers, 3) + markerDepths := make(map[uint32]bool) + for _, m := range markers { + markerDepths[m.Depth] = true + } + require.True(t, markerDepths[0]) + require.True(t, markerDepths[100]) + require.True(t, markerDepths[200]) + + // Test 6: Verify VTXOs can be retrieved by depth range + vtxosDepth50to150, err := svc.Markers().GetVtxosByDepthRange(ctx, 50, 150) + require.NoError(t, err) + // Filter to our test VTXOs + ourVtxosInRange := 0 + for _, v := range vtxosDepth50to150 { + if len(v.Txid) > 13 && v.Txid[:13] == "opt_chain_vtx" { + ourVtxosInRange++ + require.True(t, v.Depth >= 50 && v.Depth <= 150, + "VTXO depth %d should be in range [50, 150]", v.Depth) + } + } + // We expect VTXOs at depths 50, 99, 100, 150 to be in range + require.Equal(t, 4, ourVtxosInRange, "Expected 4 VTXOs in depth range 50-150") + }) +} + func testScheduledSessionRepository(t *testing.T, svc ports.RepoManager) { t.Run("test_scheduled_session_repository", func(t *testing.T) { ctx := context.Background() diff --git a/internal/infrastructure/db/sqlite/marker_repo.go b/internal/infrastructure/db/sqlite/marker_repo.go index a8d7e97c9..9307d2336 100644 --- a/internal/infrastructure/db/sqlite/marker_repo.go +++ b/internal/infrastructure/db/sqlite/marker_repo.go @@ -64,7 +64,10 @@ func (m *markerRepository) GetMarker(ctx context.Context, id string) (*domain.Ma return &marker, nil } -func (m *markerRepository) GetMarkersByDepth(ctx context.Context, depth uint32) ([]domain.Marker, error) { +func (m *markerRepository) GetMarkersByDepth( + ctx context.Context, + depth uint32, +) ([]domain.Marker, error) { rows, err := m.querier.SelectMarkersByDepth(ctx, int64(depth)) if err != nil { return nil, err @@ -81,7 +84,10 @@ func (m *markerRepository) GetMarkersByDepth(ctx context.Context, depth uint32) return markers, nil } -func (m *markerRepository) GetMarkersByDepthRange(ctx context.Context, minDepth, maxDepth uint32) ([]domain.Marker, error) { +func (m *markerRepository) GetMarkersByDepthRange( + ctx context.Context, + minDepth, maxDepth uint32, +) ([]domain.Marker, error) { rows, err := m.querier.SelectMarkersByDepthRange(ctx, queries.SelectMarkersByDepthRangeParams{ MinDepth: int64(minDepth), MaxDepth: int64(maxDepth), @@ -101,7 +107,10 @@ func (m *markerRepository) GetMarkersByDepthRange(ctx context.Context, minDepth, return markers, nil } -func (m *markerRepository) GetMarkersByIds(ctx context.Context, ids []string) ([]domain.Marker, error) { +func (m *markerRepository) GetMarkersByIds( + ctx context.Context, + ids []string, +) ([]domain.Marker, error) { if len(ids) == 0 { return nil, nil } @@ -129,7 +138,11 @@ func (m *markerRepository) SweepMarker(ctx context.Context, markerID string, swe }) } -func (m *markerRepository) SweepMarkerWithDescendants(ctx context.Context, markerID string, sweptAt int64) (int64, error) { +func (m *markerRepository) SweepMarkerWithDescendants( + ctx context.Context, + markerID string, + sweptAt int64, +) (int64, error) { // Get all descendant marker IDs (including the root marker) that are not already swept descendantIDs, err := m.querier.GetDescendantMarkerIds(ctx, markerID) if err != nil { @@ -160,7 +173,10 @@ func (m *markerRepository) IsMarkerSwept(ctx context.Context, markerID string) ( return result == 1, nil } -func (m *markerRepository) GetSweptMarkers(ctx context.Context, markerIDs []string) ([]domain.SweptMarker, error) { +func (m *markerRepository) GetSweptMarkers( + ctx context.Context, + markerIDs []string, +) ([]domain.SweptMarker, error) { if len(markerIDs) == 0 { return nil, nil } @@ -180,7 +196,11 @@ func (m *markerRepository) GetSweptMarkers(ctx context.Context, markerIDs []stri return sweptMarkers, nil } -func (m *markerRepository) UpdateVtxoMarker(ctx context.Context, outpoint domain.Outpoint, markerID string) error { +func (m *markerRepository) UpdateVtxoMarker( + ctx context.Context, + outpoint domain.Outpoint, + markerID string, +) error { return m.querier.UpdateVtxoMarkerId(ctx, queries.UpdateVtxoMarkerIdParams{ MarkerID: sql.NullString{String: markerID, Valid: len(markerID) > 0}, Txid: outpoint.Txid, @@ -188,8 +208,14 @@ func (m *markerRepository) UpdateVtxoMarker(ctx context.Context, outpoint domain }) } -func (m *markerRepository) GetVtxosByMarker(ctx context.Context, markerID string) ([]domain.Vtxo, error) { - rows, err := m.querier.SelectVtxosByMarkerId(ctx, sql.NullString{String: markerID, Valid: len(markerID) > 0}) +func (m *markerRepository) GetVtxosByMarker( + ctx context.Context, + markerID string, +) ([]domain.Vtxo, error) { + rows, err := m.querier.SelectVtxosByMarkerId( + ctx, + sql.NullString{String: markerID, Valid: len(markerID) > 0}, + ) if err != nil { return nil, err } @@ -202,10 +228,16 @@ func (m *markerRepository) GetVtxosByMarker(ctx context.Context, markerID string } func (m *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID string) (int64, error) { - return m.querier.SweepVtxosByMarkerId(ctx, sql.NullString{String: markerID, Valid: len(markerID) > 0}) + return m.querier.SweepVtxosByMarkerId( + ctx, + sql.NullString{String: markerID, Valid: len(markerID) > 0}, + ) } -func (m *markerRepository) GetVtxosByDepthRange(ctx context.Context, minDepth, maxDepth uint32) ([]domain.Vtxo, error) { +func (m *markerRepository) GetVtxosByDepthRange( + ctx context.Context, + minDepth, maxDepth uint32, +) ([]domain.Vtxo, error) { rows, err := m.querier.SelectVtxosByDepthRange(ctx, queries.SelectVtxosByDepthRangeParams{ MinDepth: int64(minDepth), MaxDepth: int64(maxDepth), @@ -221,7 +253,10 @@ func (m *markerRepository) GetVtxosByDepthRange(ctx context.Context, minDepth, m return vtxos, nil } -func (m *markerRepository) GetVtxosByArkTxid(ctx context.Context, arkTxid string) ([]domain.Vtxo, error) { +func (m *markerRepository) GetVtxosByArkTxid( + ctx context.Context, + arkTxid string, +) ([]domain.Vtxo, error) { rows, err := m.querier.SelectVtxosByArkTxid(ctx, arkTxid) if err != nil { return nil, err @@ -234,7 +269,10 @@ func (m *markerRepository) GetVtxosByArkTxid(ctx context.Context, arkTxid string return vtxos, nil } -func (m *markerRepository) GetVtxoChainByMarkers(ctx context.Context, markerIDs []string) ([]domain.Vtxo, error) { +func (m *markerRepository) GetVtxoChainByMarkers( + ctx context.Context, + markerIDs []string, +) ([]domain.Vtxo, error) { if len(markerIDs) == 0 { return nil, nil } From e39e4d23fbc12e8ff75e42f8196ee8d8d19ef2b3 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Tue, 10 Feb 2026 15:10:45 -0500 Subject: [PATCH 04/54] badger impl, migration file renames, marker_test.go --- internal/core/domain/marker_test.go | 78 +++ .../infrastructure/db/badger/marker_repo.go | 515 ++++++++++++++++++ .../infrastructure/db/badger/vtxo_repo.go | 63 ++- ...=> 20260210100000_add_vtxo_depth.down.sql} | 0 ...l => 20260210100000_add_vtxo_depth.up.sql} | 0 ...ql => 20260211020000_add_markers.down.sql} | 0 ....sql => 20260211020000_add_markers.up.sql} | 0 internal/infrastructure/db/service.go | 16 +- internal/test/e2e/e2e_test.go | 4 + internal/test/e2e/utils_test.go | 3 + 10 files changed, 649 insertions(+), 30 deletions(-) create mode 100644 internal/core/domain/marker_test.go create mode 100644 internal/infrastructure/db/badger/marker_repo.go rename internal/infrastructure/db/postgres/migration/{20260210000000_add_vtxo_depth.down.sql => 20260210100000_add_vtxo_depth.down.sql} (100%) rename internal/infrastructure/db/postgres/migration/{20260210000000_add_vtxo_depth.up.sql => 20260210100000_add_vtxo_depth.up.sql} (100%) rename internal/infrastructure/db/postgres/migration/{20260211000000_add_markers.down.sql => 20260211020000_add_markers.down.sql} (100%) rename internal/infrastructure/db/postgres/migration/{20260211000000_add_markers.up.sql => 20260211020000_add_markers.up.sql} (100%) diff --git a/internal/core/domain/marker_test.go b/internal/core/domain/marker_test.go new file mode 100644 index 000000000..2834e4938 --- /dev/null +++ b/internal/core/domain/marker_test.go @@ -0,0 +1,78 @@ +package domain + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestIsAtMarkerBoundary(t *testing.T) { + tests := []struct { + depth uint32 + expected bool + }{ + {0, true}, // First marker boundary + {1, false}, + {50, false}, + {99, false}, + {100, true}, // Second marker boundary + {101, false}, + {150, false}, + {199, false}, + {200, true}, // Third marker boundary + {201, false}, + {300, true}, + {1000, true}, + {1001, false}, + {10000, true}, + } + + for _, tt := range tests { + result := IsAtMarkerBoundary(tt.depth) + require.Equal(t, tt.expected, result, + "IsAtMarkerBoundary(%d) should be %v", tt.depth, tt.expected) + } +} + +func TestMarkerInterval(t *testing.T) { + // Verify the constant is set correctly + require.Equal(t, uint32(100), uint32(MarkerInterval)) +} + +func TestMarkerStruct(t *testing.T) { + // Test Marker struct creation + marker := Marker{ + ID: "test-marker-id", + Depth: 100, + ParentMarkerIDs: []string{"parent-marker-1", "parent-marker-2"}, + } + + require.Equal(t, "test-marker-id", marker.ID) + require.Equal(t, uint32(100), marker.Depth) + require.Len(t, marker.ParentMarkerIDs, 2) + require.Contains(t, marker.ParentMarkerIDs, "parent-marker-1") + require.Contains(t, marker.ParentMarkerIDs, "parent-marker-2") +} + +func TestSweptMarkerStruct(t *testing.T) { + // Test SweptMarker struct creation + sweptMarker := SweptMarker{ + MarkerID: "swept-marker-id", + SweptAt: 1234567890, + } + + require.Equal(t, "swept-marker-id", sweptMarker.MarkerID) + require.Equal(t, int64(1234567890), sweptMarker.SweptAt) +} + +func TestRootMarkerHasNoParents(t *testing.T) { + // Root markers (depth 0) should have no parent markers + rootMarker := Marker{ + ID: "root-marker", + Depth: 0, + ParentMarkerIDs: nil, + } + + require.True(t, IsAtMarkerBoundary(rootMarker.Depth)) + require.Nil(t, rootMarker.ParentMarkerIDs) +} diff --git a/internal/infrastructure/db/badger/marker_repo.go b/internal/infrastructure/db/badger/marker_repo.go new file mode 100644 index 000000000..133f1c6ec --- /dev/null +++ b/internal/infrastructure/db/badger/marker_repo.go @@ -0,0 +1,515 @@ +package badgerdb + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "time" + + "github.com/arkade-os/arkd/internal/core/domain" + "github.com/dgraph-io/badger/v4" + "github.com/timshannon/badgerhold/v4" +) + +const ( + markerStoreDir = "markers" + sweptMarkerStoreDir = "swept_markers" +) + +type markerRepository struct { + markerStore *badgerhold.Store + sweptMarkerStore *badgerhold.Store + vtxoStore *badgerhold.Store + ownsVtxoStore bool // whether this repo owns the vtxo store (for Close()) +} + +type markerDTO struct { + ID string + Depth uint32 + ParentMarkerIDs []string +} + +type sweptMarkerDTO struct { + MarkerID string + SweptAt int64 +} + +// NewMarkerRepository creates a new marker repository. +// Config can be: +// - [baseDir string, logger badger.Logger] - creates its own vtxo store +// - [baseDir string, logger badger.Logger, vtxoStore *badgerhold.Store] - uses shared vtxo store +func NewMarkerRepository(config ...interface{}) (domain.MarkerRepository, error) { + if len(config) < 2 { + return nil, fmt.Errorf("invalid config: need at least baseDir and logger") + } + baseDir, ok := config[0].(string) + if !ok { + return nil, fmt.Errorf("invalid base directory") + } + var logger badger.Logger + if config[1] != nil { + logger, ok = config[1].(badger.Logger) + if !ok { + return nil, fmt.Errorf("invalid logger") + } + } + + var markerDir, sweptMarkerDir string + if len(baseDir) > 0 { + markerDir = filepath.Join(baseDir, markerStoreDir) + sweptMarkerDir = filepath.Join(baseDir, sweptMarkerStoreDir) + } + + markerStore, err := createDB(markerDir, logger) + if err != nil { + return nil, fmt.Errorf("failed to open marker store: %s", err) + } + + sweptMarkerStore, err := createDB(sweptMarkerDir, logger) + if err != nil { + _ = markerStore.Close() + return nil, fmt.Errorf("failed to open swept marker store: %s", err) + } + + // Check if a shared vtxo store was provided + var vtxoStore *badgerhold.Store + ownsVtxoStore := false + if len(config) >= 3 && config[2] != nil { + vtxoStore, ok = config[2].(*badgerhold.Store) + if !ok { + _ = markerStore.Close() + _ = sweptMarkerStore.Close() + return nil, fmt.Errorf("invalid vtxo store") + } + } else { + // Create our own vtxo store + var vtxoDir string + if len(baseDir) > 0 { + vtxoDir = filepath.Join(baseDir, vtxoStoreDir) + } + vtxoStore, err = createDB(vtxoDir, logger) + if err != nil { + _ = markerStore.Close() + _ = sweptMarkerStore.Close() + return nil, fmt.Errorf("failed to open vtxo store for marker repo: %s", err) + } + ownsVtxoStore = true + } + + return &markerRepository{ + markerStore: markerStore, + sweptMarkerStore: sweptMarkerStore, + vtxoStore: vtxoStore, + ownsVtxoStore: ownsVtxoStore, + }, nil +} + +func (r *markerRepository) Close() { + _ = r.markerStore.Close() + _ = r.sweptMarkerStore.Close() + if r.ownsVtxoStore { + _ = r.vtxoStore.Close() + } +} + +func (r *markerRepository) AddMarker(ctx context.Context, marker domain.Marker) error { + dto := markerDTO{ + ID: marker.ID, + Depth: marker.Depth, + ParentMarkerIDs: marker.ParentMarkerIDs, + } + + err := r.markerStore.Upsert(marker.ID, dto) + if err != nil { + if errors.Is(err, badger.ErrConflict) { + for attempts := 1; attempts <= maxRetries; attempts++ { + time.Sleep(100 * time.Millisecond) + err = r.markerStore.Upsert(marker.ID, dto) + if err == nil { + break + } + } + } + } + return err +} + +func (r *markerRepository) GetMarker(ctx context.Context, id string) (*domain.Marker, error) { + var dto markerDTO + err := r.markerStore.Get(id, &dto) + if err != nil { + if err == badgerhold.ErrNotFound { + return nil, nil + } + return nil, err + } + + return &domain.Marker{ + ID: dto.ID, + Depth: dto.Depth, + ParentMarkerIDs: dto.ParentMarkerIDs, + }, nil +} + +func (r *markerRepository) GetMarkersByDepth( + ctx context.Context, + depth uint32, +) ([]domain.Marker, error) { + var dtos []markerDTO + err := r.markerStore.Find(&dtos, badgerhold.Where("Depth").Eq(depth)) + if err != nil { + return nil, err + } + + markers := make([]domain.Marker, 0, len(dtos)) + for _, dto := range dtos { + markers = append(markers, domain.Marker{ + ID: dto.ID, + Depth: dto.Depth, + ParentMarkerIDs: dto.ParentMarkerIDs, + }) + } + return markers, nil +} + +func (r *markerRepository) GetMarkersByDepthRange( + ctx context.Context, + minDepth, maxDepth uint32, +) ([]domain.Marker, error) { + var dtos []markerDTO + err := r.markerStore.Find(&dtos, + badgerhold.Where("Depth").Ge(minDepth).And("Depth").Le(maxDepth)) + if err != nil { + return nil, err + } + + markers := make([]domain.Marker, 0, len(dtos)) + for _, dto := range dtos { + markers = append(markers, domain.Marker{ + ID: dto.ID, + Depth: dto.Depth, + ParentMarkerIDs: dto.ParentMarkerIDs, + }) + } + return markers, nil +} + +func (r *markerRepository) GetMarkersByIds( + ctx context.Context, + ids []string, +) ([]domain.Marker, error) { + if len(ids) == 0 { + return nil, nil + } + + markers := make([]domain.Marker, 0, len(ids)) + for _, id := range ids { + marker, err := r.GetMarker(ctx, id) + if err != nil { + return nil, err + } + if marker != nil { + markers = append(markers, *marker) + } + } + return markers, nil +} + +func (r *markerRepository) SweepMarker(ctx context.Context, markerID string, sweptAt int64) error { + // Check if already swept - if so, preserve original swept_at (ON CONFLICT DO NOTHING behavior) + var existing sweptMarkerDTO + err := r.sweptMarkerStore.Get(markerID, &existing) + if err == nil { + // Already swept, don't update + return nil + } + if err != badgerhold.ErrNotFound { + return err + } + + dto := sweptMarkerDTO{ + MarkerID: markerID, + SweptAt: sweptAt, + } + + err = r.sweptMarkerStore.Insert(markerID, dto) + if err != nil { + if errors.Is(err, badgerhold.ErrKeyExists) { + // Already exists (race condition), that's fine + return nil + } + if errors.Is(err, badger.ErrConflict) { + for attempts := 1; attempts <= maxRetries; attempts++ { + time.Sleep(100 * time.Millisecond) + err = r.sweptMarkerStore.Insert(markerID, dto) + if err == nil || errors.Is(err, badgerhold.ErrKeyExists) { + return nil + } + } + } + return err + } + return nil +} + +func (r *markerRepository) SweepMarkerWithDescendants( + ctx context.Context, + markerID string, + sweptAt int64, +) (int64, error) { + // Find all descendant markers using BFS + descendantIDs, err := r.getDescendantMarkerIds(ctx, markerID) + if err != nil { + return 0, fmt.Errorf("failed to get descendant markers: %w", err) + } + + var count int64 + for _, id := range descendantIDs { + // Check if already swept + isSwept, err := r.IsMarkerSwept(ctx, id) + if err != nil { + return count, err + } + if isSwept { + continue + } + + if err := r.SweepMarker(ctx, id, sweptAt); err != nil { + return count, fmt.Errorf("failed to sweep marker %s: %w", id, err) + } + count++ + } + + return count, nil +} + +// getDescendantMarkerIds finds all markers that descend from the given marker ID +// using BFS traversal of the parent_marker_ids relationship. +// Returns empty slice if the root marker doesn't exist. +func (r *markerRepository) getDescendantMarkerIds( + ctx context.Context, + rootMarkerID string, +) ([]string, error) { + // First check if the root marker exists + var rootDTO markerDTO + err := r.markerStore.Get(rootMarkerID, &rootDTO) + if err != nil { + if err == badgerhold.ErrNotFound { + return []string{}, nil // Root doesn't exist, return empty + } + return nil, err + } + + descendantIDs := []string{rootMarkerID} + visited := map[string]bool{rootMarkerID: true} + queue := []string{rootMarkerID} + + for len(queue) > 0 { + currentID := queue[0] + queue = queue[1:] + + // Find all markers that have currentID in their ParentMarkerIDs + var dtos []markerDTO + err := r.markerStore.Find(&dtos, + badgerhold.Where("ParentMarkerIDs").Contains(currentID)) + if err != nil { + return nil, err + } + + for _, dto := range dtos { + if !visited[dto.ID] { + visited[dto.ID] = true + descendantIDs = append(descendantIDs, dto.ID) + queue = append(queue, dto.ID) + } + } + } + + return descendantIDs, nil +} + +func (r *markerRepository) IsMarkerSwept(ctx context.Context, markerID string) (bool, error) { + var dto sweptMarkerDTO + err := r.sweptMarkerStore.Get(markerID, &dto) + if err != nil { + if err == badgerhold.ErrNotFound { + return false, nil + } + return false, err + } + return true, nil +} + +func (r *markerRepository) GetSweptMarkers( + ctx context.Context, + markerIDs []string, +) ([]domain.SweptMarker, error) { + if len(markerIDs) == 0 { + return nil, nil + } + + sweptMarkers := make([]domain.SweptMarker, 0, len(markerIDs)) + for _, id := range markerIDs { + var dto sweptMarkerDTO + err := r.sweptMarkerStore.Get(id, &dto) + if err != nil { + if err == badgerhold.ErrNotFound { + continue + } + return nil, err + } + sweptMarkers = append(sweptMarkers, domain.SweptMarker{ + MarkerID: dto.MarkerID, + SweptAt: dto.SweptAt, + }) + } + return sweptMarkers, nil +} + +func (r *markerRepository) UpdateVtxoMarker( + ctx context.Context, + outpoint domain.Outpoint, + markerID string, +) error { + var dto vtxoDTO + err := r.vtxoStore.Get(outpoint.String(), &dto) + if err != nil { + if err == badgerhold.ErrNotFound { + return nil // VTXO not found, nothing to update + } + return err + } + + dto.MarkerID = markerID + dto.UpdatedAt = time.Now().UnixMilli() + + err = r.vtxoStore.Update(outpoint.String(), dto) + if err != nil { + if errors.Is(err, badger.ErrConflict) { + for attempts := 1; attempts <= maxRetries; attempts++ { + time.Sleep(100 * time.Millisecond) + err = r.vtxoStore.Update(outpoint.String(), dto) + if err == nil { + break + } + } + } + } + return err +} + +func (r *markerRepository) GetVtxosByMarker( + ctx context.Context, + markerID string, +) ([]domain.Vtxo, error) { + var dtos []vtxoDTO + err := r.vtxoStore.Find(&dtos, badgerhold.Where("MarkerID").Eq(markerID)) + if err != nil { + return nil, err + } + + vtxos := make([]domain.Vtxo, 0, len(dtos)) + for _, dto := range dtos { + vtxos = append(vtxos, dto.Vtxo) + } + return vtxos, nil +} + +func (r *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID string) (int64, error) { + var dtos []vtxoDTO + err := r.vtxoStore.Find(&dtos, + badgerhold.Where("MarkerID").Eq(markerID).And("Swept").Eq(false)) + if err != nil { + return 0, err + } + + var count int64 + for _, dto := range dtos { + dto.Swept = true + dto.UpdatedAt = time.Now().UnixMilli() + + err := r.vtxoStore.Update(dto.Outpoint.String(), dto) + if err != nil { + if errors.Is(err, badger.ErrConflict) { + for attempts := 1; attempts <= maxRetries; attempts++ { + time.Sleep(100 * time.Millisecond) + err = r.vtxoStore.Update(dto.Outpoint.String(), dto) + if err == nil { + break + } + } + } + if err != nil { + return count, err + } + } + count++ + } + return count, nil +} + +func (r *markerRepository) GetVtxosByDepthRange( + ctx context.Context, + minDepth, maxDepth uint32, +) ([]domain.Vtxo, error) { + var dtos []vtxoDTO + err := r.vtxoStore.Find(&dtos, + badgerhold.Where("Depth").Ge(minDepth).And("Depth").Le(maxDepth)) + if err != nil { + return nil, err + } + + vtxos := make([]domain.Vtxo, 0, len(dtos)) + for _, dto := range dtos { + vtxos = append(vtxos, dto.Vtxo) + } + return vtxos, nil +} + +func (r *markerRepository) GetVtxosByArkTxid( + ctx context.Context, + arkTxid string, +) ([]domain.Vtxo, error) { + var dtos []vtxoDTO + err := r.vtxoStore.Find(&dtos, badgerhold.Where("Txid").Eq(arkTxid)) + if err != nil { + return nil, err + } + + vtxos := make([]domain.Vtxo, 0, len(dtos)) + for _, dto := range dtos { + vtxos = append(vtxos, dto.Vtxo) + } + return vtxos, nil +} + +func (r *markerRepository) GetVtxoChainByMarkers( + ctx context.Context, + markerIDs []string, +) ([]domain.Vtxo, error) { + if len(markerIDs) == 0 { + return nil, nil + } + + // Build a set of marker IDs for efficient lookup + markerIDSet := make(map[string]bool) + for _, id := range markerIDs { + markerIDSet[id] = true + } + + // Find all VTXOs that have a marker_id in our set + var dtos []vtxoDTO + err := r.vtxoStore.Find(&dtos, &badgerhold.Query{}) + if err != nil { + return nil, err + } + + vtxos := make([]domain.Vtxo, 0) + for _, dto := range dtos { + if dto.MarkerID != "" && markerIDSet[dto.MarkerID] { + vtxos = append(vtxos, dto.Vtxo) + } + } + return vtxos, nil +} diff --git a/internal/infrastructure/db/badger/vtxo_repo.go b/internal/infrastructure/db/badger/vtxo_repo.go index 31b3cc1b6..69c6d81f1 100644 --- a/internal/infrastructure/db/badger/vtxo_repo.go +++ b/internal/infrastructure/db/badger/vtxo_repo.go @@ -16,7 +16,7 @@ import ( const vtxoStoreDir = "vtxos" -type vtxoRepository struct { +type VtxoRepository struct { store *badgerhold.Store } @@ -50,16 +50,16 @@ func NewVtxoRepository(config ...interface{}) (domain.VtxoRepository, error) { return nil, fmt.Errorf("failed to open round events store: %s", err) } - return &vtxoRepository{store}, nil + return &VtxoRepository{store}, nil } -func (r *vtxoRepository) AddVtxos( +func (r *VtxoRepository) AddVtxos( ctx context.Context, vtxos []domain.Vtxo, ) error { return r.addVtxos(ctx, vtxos) } -func (r *vtxoRepository) SettleVtxos( +func (r *VtxoRepository) SettleVtxos( ctx context.Context, spentVtxos map[domain.Outpoint]string, commitmentTxid string, ) error { for outpoint, spentBy := range spentVtxos { @@ -70,7 +70,7 @@ func (r *vtxoRepository) SettleVtxos( return nil } -func (r *vtxoRepository) SpendVtxos( +func (r *VtxoRepository) SpendVtxos( ctx context.Context, spentVtxos map[domain.Outpoint]string, arkTxid string, ) error { for outpoint, spentBy := range spentVtxos { @@ -81,7 +81,7 @@ func (r *vtxoRepository) SpendVtxos( return nil } -func (r *vtxoRepository) UnrollVtxos( +func (r *VtxoRepository) UnrollVtxos( ctx context.Context, outpoints []domain.Outpoint, ) error { for _, outpoint := range outpoints { @@ -93,7 +93,7 @@ func (r *vtxoRepository) UnrollVtxos( return nil } -func (r *vtxoRepository) GetVtxos( +func (r *VtxoRepository) GetVtxos( ctx context.Context, outpoints []domain.Outpoint, ) ([]domain.Vtxo, error) { vtxos := make([]domain.Vtxo, 0, len(outpoints)) @@ -113,14 +113,14 @@ func (r *vtxoRepository) GetVtxos( return vtxos, nil } -func (r *vtxoRepository) GetLeafVtxosForBatch( +func (r *VtxoRepository) GetLeafVtxosForBatch( ctx context.Context, txid string, ) ([]domain.Vtxo, error) { query := badgerhold.Where("RootCommitmentTxid").Eq(txid).And("Preconfirmed").Eq(false) return r.findVtxos(ctx, query) } -func (r *vtxoRepository) GetAllNonUnrolledVtxos( +func (r *VtxoRepository) GetAllNonUnrolledVtxos( ctx context.Context, pubkey string, ) ([]domain.Vtxo, []domain.Vtxo, error) { query := badgerhold.Where("Unrolled").Eq(false) @@ -144,7 +144,7 @@ func (r *vtxoRepository) GetAllNonUnrolledVtxos( return unspentVtxos, spentVtxos, nil } -func (r *vtxoRepository) GetAllSweepableUnrolledVtxos( +func (r *VtxoRepository) GetAllSweepableUnrolledVtxos( ctx context.Context, ) ([]domain.Vtxo, error) { query := badgerhold.Where("Unrolled"). @@ -158,11 +158,11 @@ func (r *vtxoRepository) GetAllSweepableUnrolledVtxos( return r.findVtxos(ctx, query) } -func (r *vtxoRepository) GetAllVtxos(ctx context.Context) ([]domain.Vtxo, error) { +func (r *VtxoRepository) GetAllVtxos(ctx context.Context) ([]domain.Vtxo, error) { return r.findVtxos(ctx, &badgerhold.Query{}) } -func (r *vtxoRepository) SweepVtxos( +func (r *VtxoRepository) SweepVtxos( ctx context.Context, outpoints []domain.Outpoint, ) (int, error) { sweptCount := 0 @@ -185,7 +185,7 @@ func (r *vtxoRepository) SweepVtxos( return sweptCount, nil } -func (r *vtxoRepository) UpdateVtxosExpiration( +func (r *VtxoRepository) UpdateVtxosExpiration( ctx context.Context, vtxos []domain.Outpoint, expiresAt int64, ) error { var err error @@ -229,7 +229,7 @@ func (r *vtxoRepository) UpdateVtxosExpiration( return err } -func (r *vtxoRepository) GetAllVtxosWithPubKeys( +func (r *VtxoRepository) GetAllVtxosWithPubKeys( ctx context.Context, pubkeys []string, after, before int64, ) ([]domain.Vtxo, error) { if err := validateTimeRange(after, before); err != nil { @@ -254,7 +254,7 @@ func (r *vtxoRepository) GetAllVtxosWithPubKeys( return allVtxos, nil } -func (r *vtxoRepository) GetExpiringLiquidity( +func (r *VtxoRepository) GetExpiringLiquidity( ctx context.Context, after, before int64, ) (uint64, error) { query := badgerhold.Where("Swept").Eq(false). @@ -278,7 +278,7 @@ func (r *vtxoRepository) GetExpiringLiquidity( return sum, nil } -func (r *vtxoRepository) GetRecoverableLiquidity(ctx context.Context) (uint64, error) { +func (r *VtxoRepository) GetRecoverableLiquidity(ctx context.Context) (uint64, error) { query := badgerhold.Where("Swept").Eq(true).And("Spent").Eq(false) vtxos, err := r.findVtxos(ctx, query) if err != nil { @@ -292,7 +292,7 @@ func (r *vtxoRepository) GetRecoverableLiquidity(ctx context.Context) (uint64, e return sum, nil } -func (r *vtxoRepository) GetVtxoPubKeysByCommitmentTxid( +func (r *VtxoRepository) GetVtxoPubKeysByCommitmentTxid( ctx context.Context, commitmentTxid string, amountFilter uint64, ) ([]string, error) { if commitmentTxid == "" { @@ -339,7 +339,7 @@ func (r *vtxoRepository) GetVtxoPubKeysByCommitmentTxid( return taprootKeys, nil } -func (r *vtxoRepository) GetPendingSpentVtxosWithPubKeys( +func (r *VtxoRepository) GetPendingSpentVtxosWithPubKeys( ctx context.Context, pubkeys []string, after, before int64, ) ([]domain.Vtxo, error) { if err := validateTimeRange(after, before); err != nil { @@ -389,7 +389,7 @@ func (r *vtxoRepository) GetPendingSpentVtxosWithPubKeys( return vtxos, nil } -func (r *vtxoRepository) GetPendingSpentVtxosWithOutpoints( +func (r *VtxoRepository) GetPendingSpentVtxosWithOutpoints( ctx context.Context, outpoints []domain.Outpoint, ) ([]domain.Vtxo, error) { // Get all candidates @@ -431,12 +431,17 @@ func (r *vtxoRepository) GetPendingSpentVtxosWithOutpoints( return vtxos, nil } -func (r *vtxoRepository) Close() { +func (r *VtxoRepository) Close() { // nolint:all r.store.Close() } -func (r *vtxoRepository) addVtxos( +// Store returns the underlying badgerhold store for sharing with other repositories. +func (r *VtxoRepository) Store() *badgerhold.Store { + return r.store +} + +func (r *VtxoRepository) addVtxos( ctx context.Context, vtxos []domain.Vtxo, ) error { for _, vtxo := range vtxos { @@ -474,7 +479,7 @@ func (r *vtxoRepository) addVtxos( return nil } -func (r *vtxoRepository) getVtxo( +func (r *VtxoRepository) getVtxo( ctx context.Context, outpoint domain.Outpoint, ) (*domain.Vtxo, error) { var dto vtxoDTO @@ -495,7 +500,7 @@ func (r *vtxoRepository) getVtxo( return &dto.Vtxo, nil } -func (r *vtxoRepository) settleVtxo( +func (r *VtxoRepository) settleVtxo( ctx context.Context, outpoint domain.Outpoint, spentBy, settledBy string, ) error { vtxo, err := r.getVtxo(ctx, outpoint) @@ -516,7 +521,7 @@ func (r *vtxoRepository) settleVtxo( return r.updateVtxo(ctx, vtxo) } -func (r *vtxoRepository) spendVtxo( +func (r *VtxoRepository) spendVtxo( ctx context.Context, outpoint domain.Outpoint, spentBy, arkTxid string, ) error { vtxo, err := r.getVtxo(ctx, outpoint) @@ -537,7 +542,7 @@ func (r *vtxoRepository) spendVtxo( return r.updateVtxo(ctx, vtxo) } -func (r *vtxoRepository) unrollVtxo( +func (r *VtxoRepository) unrollVtxo( ctx context.Context, outpoint domain.Outpoint, ) (*domain.Vtxo, error) { vtxo, err := r.getVtxo(ctx, outpoint) @@ -559,7 +564,7 @@ func (r *vtxoRepository) unrollVtxo( return vtxo, nil } -func (r *vtxoRepository) findVtxos( +func (r *VtxoRepository) findVtxos( ctx context.Context, query *badgerhold.Query, ) ([]domain.Vtxo, error) { vtxos := make([]domain.Vtxo, 0) @@ -579,7 +584,7 @@ func (r *vtxoRepository) findVtxos( return vtxos, err } -func (r *vtxoRepository) updateVtxo(ctx context.Context, vtxo *domain.Vtxo) error { +func (r *VtxoRepository) updateVtxo(ctx context.Context, vtxo *domain.Vtxo) error { dto := vtxoDTO{ Vtxo: *vtxo, UpdatedAt: time.Now().UnixMilli(), @@ -610,7 +615,7 @@ func (r *vtxoRepository) updateVtxo(ctx context.Context, vtxo *domain.Vtxo) erro return nil } -func (r *vtxoRepository) GetSweepableVtxosByCommitmentTxid( +func (r *VtxoRepository) GetSweepableVtxosByCommitmentTxid( ctx context.Context, txid string, ) ([]domain.Outpoint, error) { @@ -653,7 +658,7 @@ func (r *vtxoRepository) GetSweepableVtxosByCommitmentTxid( return outpoints, nil } -func (r *vtxoRepository) GetAllChildrenVtxos( +func (r *VtxoRepository) GetAllChildrenVtxos( ctx context.Context, txid string, ) ([]domain.Outpoint, error) { diff --git a/internal/infrastructure/db/postgres/migration/20260210000000_add_vtxo_depth.down.sql b/internal/infrastructure/db/postgres/migration/20260210100000_add_vtxo_depth.down.sql similarity index 100% rename from internal/infrastructure/db/postgres/migration/20260210000000_add_vtxo_depth.down.sql rename to internal/infrastructure/db/postgres/migration/20260210100000_add_vtxo_depth.down.sql diff --git a/internal/infrastructure/db/postgres/migration/20260210000000_add_vtxo_depth.up.sql b/internal/infrastructure/db/postgres/migration/20260210100000_add_vtxo_depth.up.sql similarity index 100% rename from internal/infrastructure/db/postgres/migration/20260210000000_add_vtxo_depth.up.sql rename to internal/infrastructure/db/postgres/migration/20260210100000_add_vtxo_depth.up.sql diff --git a/internal/infrastructure/db/postgres/migration/20260211000000_add_markers.down.sql b/internal/infrastructure/db/postgres/migration/20260211020000_add_markers.down.sql similarity index 100% rename from internal/infrastructure/db/postgres/migration/20260211000000_add_markers.down.sql rename to internal/infrastructure/db/postgres/migration/20260211020000_add_markers.down.sql diff --git a/internal/infrastructure/db/postgres/migration/20260211000000_add_markers.up.sql b/internal/infrastructure/db/postgres/migration/20260211020000_add_markers.up.sql similarity index 100% rename from internal/infrastructure/db/postgres/migration/20260211000000_add_markers.up.sql rename to internal/infrastructure/db/postgres/migration/20260211020000_add_markers.up.sql diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index edbc5e33e..6b61915a0 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -74,6 +74,7 @@ var ( "postgres": pgdb.NewIntentFeesRepository, } markerStoreTypes = map[string]func(...interface{}) (domain.MarkerRepository, error){ + "badger": badgerdb.NewMarkerRepository, "sqlite": sqlitedb.NewMarkerRepository, "postgres": pgdb.NewMarkerRepository, } @@ -132,7 +133,7 @@ func NewService(config ServiceConfig, txDecoder ports.TxDecoder) (ports.RepoMana if !ok { return nil, fmt.Errorf("invalid data store type: %s", config.DataStoreType) } - markerStoreFactory := markerStoreTypes[config.DataStoreType] // optional, may be nil for badger + markerStoreFactory := markerStoreTypes[config.DataStoreType] var eventStore domain.EventRepository var roundStore domain.RoundRepository @@ -204,6 +205,19 @@ func NewService(config ServiceConfig, txDecoder ports.TxDecoder) (ports.RepoMana if err != nil { return nil, fmt.Errorf("failed to create intent fees store: %w", err) } + if markerStoreFactory != nil { + // For badger, pass the vtxo store to marker repo to share the same database + badgerVtxoRepo, ok := vtxoStore.(*badgerdb.VtxoRepository) + if ok { + markerConfig := append(config.DataStoreConfig, badgerVtxoRepo.Store()) + markerStore, err = markerStoreFactory(markerConfig...) + } else { + markerStore, err = markerStoreFactory(config.DataStoreConfig...) + } + if err != nil { + return nil, fmt.Errorf("failed to create marker store: %w", err) + } + } case "postgres": if len(config.DataStoreConfig) != 2 { return nil, fmt.Errorf("invalid data store config for postgres") diff --git a/internal/test/e2e/e2e_test.go b/internal/test/e2e/e2e_test.go index dc62d3052..dfe4b1b5b 100644 --- a/internal/test/e2e/e2e_test.go +++ b/internal/test/e2e/e2e_test.go @@ -4084,3 +4084,7 @@ func TestFee(t *testing.T) { require.Zero(t, int(bobBalance.OnchainBalance.SpendableAmount)) require.Empty(t, bobBalance.OnchainBalance.LockedAmount) } + +// TODO: TestVtxoDepth is commented out until the SDK proto package includes the Depth field. +// Once github.com/arkade-os/go-sdk/api-spec/protobuf/gen/ark/v1 has GetDepth() on IndexerVtxo, +// this test can be re-enabled to verify that VTXO depth increments correctly during offchain transactions. diff --git a/internal/test/e2e/utils_test.go b/internal/test/e2e/utils_test.go index 432360cef..a624dc269 100644 --- a/internal/test/e2e/utils_test.go +++ b/internal/test/e2e/utils_test.go @@ -739,3 +739,6 @@ func refill(httpClient *http.Client) error { } return nil } + +// TODO: setupRawIndexerClient and getVtxoDepthByOutpoint are commented out until +// the SDK proto package includes the Depth field on IndexerVtxo. From 94adccbbeff99b50bd5677978fc5a8d2df178630 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Wed, 11 Feb 2026 15:18:54 -0500 Subject: [PATCH 05/54] change vtxo table marker column to be JSONB to hold >=1 marker --- internal/core/application/indexer.go | 39 +++++-- internal/core/domain/marker_repo.go | 4 +- internal/core/domain/vtxo.go | 4 +- .../infrastructure/db/badger/marker_repo.go | 48 +++++--- .../infrastructure/db/postgres/marker_repo.go | 42 ++++--- ...0210100000_add_depth_and_markers.down.sql} | 11 +- ...260210100000_add_depth_and_markers.up.sql} | 28 +++-- .../20260210100000_add_vtxo_depth.down.sql | 22 ---- .../20260210100000_add_vtxo_depth.up.sql | 23 ---- .../db/postgres/sqlc/queries/models.go | 6 +- .../db/postgres/sqlc/queries/query.sql.go | 89 +++++++-------- .../infrastructure/db/postgres/sqlc/query.sql | 14 ++- .../infrastructure/db/postgres/vtxo_repo.go | 16 ++- internal/infrastructure/db/service.go | 44 ++++---- internal/infrastructure/db/service_test.go | 32 +++--- .../infrastructure/db/sqlite/marker_repo.go | 64 +++++++---- ...0210000000_add_depth_and_markers.down.sql} | 9 +- ...260210000000_add_depth_and_markers.up.sql} | 21 ++-- .../20260210000000_add_vtxo_depth.down.sql | 54 --------- .../20260210000000_add_vtxo_depth.up.sql | 22 ---- .../db/sqlite/sqlc/queries/models.go | 6 +- .../db/sqlite/sqlc/queries/query.sql.go | 103 ++++++++---------- .../infrastructure/db/sqlite/sqlc/query.sql | 17 +-- .../infrastructure/db/sqlite/vtxo_repo.go | 15 ++- 24 files changed, 356 insertions(+), 377 deletions(-) rename internal/infrastructure/db/postgres/migration/{20260211020000_add_markers.down.sql => 20260210100000_add_depth_and_markers.down.sql} (69%) rename internal/infrastructure/db/postgres/migration/{20260211020000_add_markers.up.sql => 20260210100000_add_depth_and_markers.up.sql} (57%) delete mode 100644 internal/infrastructure/db/postgres/migration/20260210100000_add_vtxo_depth.down.sql delete mode 100644 internal/infrastructure/db/postgres/migration/20260210100000_add_vtxo_depth.up.sql rename internal/infrastructure/db/sqlite/migration/{20260211000000_add_markers.down.sql => 20260210000000_add_depth_and_markers.down.sql} (88%) rename internal/infrastructure/db/sqlite/migration/{20260211000000_add_markers.up.sql => 20260210000000_add_depth_and_markers.up.sql} (73%) delete mode 100644 internal/infrastructure/db/sqlite/migration/20260210000000_add_vtxo_depth.down.sql delete mode 100644 internal/infrastructure/db/sqlite/migration/20260210000000_add_vtxo_depth.up.sql diff --git a/internal/core/application/indexer.go b/internal/core/application/indexer.go index 4d52a03be..19ee90c73 100644 --- a/internal/core/application/indexer.go +++ b/internal/core/application/indexer.go @@ -394,22 +394,39 @@ func (i *indexerService) prefetchVtxosByMarkers( // Add starting VTXO to cache cache[startVtxo.Outpoint.String()] = startVtxo - if startVtxo.MarkerID == "" { + if len(startVtxo.MarkerIDs) == 0 { return cache } - // Collect marker chain by following ParentMarkerIDs - markerIDs := []string{startVtxo.MarkerID} - marker, err := i.repoManager.Markers().GetMarker(ctx, startVtxo.MarkerID) - if err != nil { - return cache + // Collect marker chain by following ParentMarkerIDs from all markers + markerIDs := make([]string, 0, len(startVtxo.MarkerIDs)) + markerIDs = append(markerIDs, startVtxo.MarkerIDs...) + + // BFS to follow all parent markers + visited := make(map[string]bool) + for _, id := range startVtxo.MarkerIDs { + visited[id] = true } - // Follow the marker chain up to the root (depth 0) - for marker != nil && len(marker.ParentMarkerIDs) > 0 { - markerIDs = append(markerIDs, marker.ParentMarkerIDs...) - // Follow first parent marker to continue chain - marker, _ = i.repoManager.Markers().GetMarker(ctx, marker.ParentMarkerIDs[0]) + queue := make([]string, 0, len(startVtxo.MarkerIDs)) + queue = append(queue, startVtxo.MarkerIDs...) + + for len(queue) > 0 { + currentID := queue[0] + queue = queue[1:] + + marker, err := i.repoManager.Markers().GetMarker(ctx, currentID) + if err != nil || marker == nil { + continue + } + + for _, parentID := range marker.ParentMarkerIDs { + if !visited[parentID] { + visited[parentID] = true + markerIDs = append(markerIDs, parentID) + queue = append(queue, parentID) + } + } } // Bulk fetch VTXOs for all markers in the chain diff --git a/internal/core/domain/marker_repo.go b/internal/core/domain/marker_repo.go index 7ea035f14..c8426c322 100644 --- a/internal/core/domain/marker_repo.go +++ b/internal/core/domain/marker_repo.go @@ -24,8 +24,8 @@ type MarkerRepository interface { // GetSweptMarkers retrieves swept marker records for the given marker IDs GetSweptMarkers(ctx context.Context, markerIDs []string) ([]SweptMarker, error) - // UpdateVtxoMarker updates the marker_id for a VTXO - UpdateVtxoMarker(ctx context.Context, outpoint Outpoint, markerID string) error + // UpdateVtxoMarkers updates the markers array for a VTXO + UpdateVtxoMarkers(ctx context.Context, outpoint Outpoint, markerIDs []string) error // GetVtxosByMarker retrieves all VTXOs associated with a marker GetVtxosByMarker(ctx context.Context, markerID string) ([]Vtxo, error) // SweepVtxosByMarker marks all VTXOs with the given marker_id as swept diff --git a/internal/core/domain/vtxo.go b/internal/core/domain/vtxo.go index 02641fa5b..6bc866787 100644 --- a/internal/core/domain/vtxo.go +++ b/internal/core/domain/vtxo.go @@ -50,8 +50,8 @@ type Vtxo struct { Preconfirmed bool ExpiresAt int64 CreatedAt int64 - Depth uint32 // chain depth: 0 for vtxos from batch, increments on each chain - MarkerID string // marker ID for DAG traversal optimization + Depth uint32 // chain depth: 0 for vtxos from batch, increments on each chain + MarkerIDs []string // marker IDs for DAG traversal optimization (supports multiple parent markers) } func (v Vtxo) String() string { diff --git a/internal/infrastructure/db/badger/marker_repo.go b/internal/infrastructure/db/badger/marker_repo.go index 133f1c6ec..828b0f0a5 100644 --- a/internal/infrastructure/db/badger/marker_repo.go +++ b/internal/infrastructure/db/badger/marker_repo.go @@ -367,10 +367,10 @@ func (r *markerRepository) GetSweptMarkers( return sweptMarkers, nil } -func (r *markerRepository) UpdateVtxoMarker( +func (r *markerRepository) UpdateVtxoMarkers( ctx context.Context, outpoint domain.Outpoint, - markerID string, + markerIDs []string, ) error { var dto vtxoDTO err := r.vtxoStore.Get(outpoint.String(), &dto) @@ -381,7 +381,7 @@ func (r *markerRepository) UpdateVtxoMarker( return err } - dto.MarkerID = markerID + dto.MarkerIDs = markerIDs dto.UpdatedAt = time.Now().UnixMilli() err = r.vtxoStore.Update(outpoint.String(), dto) @@ -403,29 +403,47 @@ func (r *markerRepository) GetVtxosByMarker( ctx context.Context, markerID string, ) ([]domain.Vtxo, error) { + // For badger, we need to scan all VTXOs and filter by MarkerIDs slice membership var dtos []vtxoDTO - err := r.vtxoStore.Find(&dtos, badgerhold.Where("MarkerID").Eq(markerID)) + err := r.vtxoStore.Find(&dtos, &badgerhold.Query{}) if err != nil { return nil, err } - vtxos := make([]domain.Vtxo, 0, len(dtos)) + vtxos := make([]domain.Vtxo, 0) for _, dto := range dtos { - vtxos = append(vtxos, dto.Vtxo) + for _, id := range dto.MarkerIDs { + if id == markerID { + vtxos = append(vtxos, dto.Vtxo) + break + } + } } return vtxos, nil } func (r *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID string) (int64, error) { - var dtos []vtxoDTO - err := r.vtxoStore.Find(&dtos, - badgerhold.Where("MarkerID").Eq(markerID).And("Swept").Eq(false)) + // Find all VTXOs whose MarkerIDs contains markerID and are not swept + var allDtos []vtxoDTO + err := r.vtxoStore.Find(&allDtos, badgerhold.Where("Swept").Eq(false)) if err != nil { return 0, err } var count int64 - for _, dto := range dtos { + for _, dto := range allDtos { + // Check if this VTXO has the markerID + hasMarker := false + for _, id := range dto.MarkerIDs { + if id == markerID { + hasMarker = true + break + } + } + if !hasMarker { + continue + } + dto.Swept = true dto.UpdatedAt = time.Now().UnixMilli() @@ -498,7 +516,7 @@ func (r *markerRepository) GetVtxoChainByMarkers( markerIDSet[id] = true } - // Find all VTXOs that have a marker_id in our set + // Find all VTXOs that have any marker_id in our set var dtos []vtxoDTO err := r.vtxoStore.Find(&dtos, &badgerhold.Query{}) if err != nil { @@ -507,8 +525,12 @@ func (r *markerRepository) GetVtxoChainByMarkers( vtxos := make([]domain.Vtxo, 0) for _, dto := range dtos { - if dto.MarkerID != "" && markerIDSet[dto.MarkerID] { - vtxos = append(vtxos, dto.Vtxo) + // Check if any of the VTXO's markers are in our set + for _, markerID := range dto.MarkerIDs { + if markerIDSet[markerID] { + vtxos = append(vtxos, dto.Vtxo) + break + } } } return vtxos, nil diff --git a/internal/infrastructure/db/postgres/marker_repo.go b/internal/infrastructure/db/postgres/marker_repo.go index dc3fc39ac..e0a2eccb9 100644 --- a/internal/infrastructure/db/postgres/marker_repo.go +++ b/internal/infrastructure/db/postgres/marker_repo.go @@ -199,15 +199,19 @@ func (m *markerRepository) GetSweptMarkers( return sweptMarkers, nil } -func (m *markerRepository) UpdateVtxoMarker( +func (m *markerRepository) UpdateVtxoMarkers( ctx context.Context, outpoint domain.Outpoint, - markerID string, + markerIDs []string, ) error { - return m.querier.UpdateVtxoMarkerId(ctx, queries.UpdateVtxoMarkerIdParams{ - MarkerID: sql.NullString{String: markerID, Valid: len(markerID) > 0}, - Txid: outpoint.Txid, - Vout: int32(outpoint.VOut), + markersJSON, err := json.Marshal(markerIDs) + if err != nil { + return fmt.Errorf("failed to marshal markers: %w", err) + } + return m.querier.UpdateVtxoMarkers(ctx, queries.UpdateVtxoMarkersParams{ + Markers: markersJSON, + Txid: outpoint.Txid, + Vout: int32(outpoint.VOut), }) } @@ -215,10 +219,7 @@ func (m *markerRepository) GetVtxosByMarker( ctx context.Context, markerID string, ) ([]domain.Vtxo, error) { - rows, err := m.querier.SelectVtxosByMarkerId( - ctx, - sql.NullString{String: markerID, Valid: len(markerID) > 0}, - ) + rows, err := m.querier.SelectVtxosByMarkerId(ctx, markerID) if err != nil { return nil, err } @@ -231,10 +232,7 @@ func (m *markerRepository) GetVtxosByMarker( } func (m *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID string) (int64, error) { - return m.querier.SweepVtxosByMarkerId( - ctx, - sql.NullString{String: markerID, Valid: len(markerID) > 0}, - ) + return m.querier.SweepVtxosByMarkerId(ctx, markerID) } func (m *markerRepository) GetVtxosByDepthRange( @@ -313,7 +311,7 @@ func rowToVtxoFromVtxoVw(row queries.VtxoVw) domain.Vtxo { ExpiresAt: row.ExpiresAt, CreatedAt: row.CreatedAt, Depth: uint32(row.Depth), - MarkerID: row.MarkerID.String, + MarkerIDs: parseMarkersJSONB(row.Markers), } } @@ -352,6 +350,18 @@ func rowToVtxoFromMarkerQuery(row queries.SelectVtxosByMarkerIdRow) domain.Vtxo ExpiresAt: row.VtxoVw.ExpiresAt, CreatedAt: row.VtxoVw.CreatedAt, Depth: uint32(row.VtxoVw.Depth), - MarkerID: row.VtxoVw.MarkerID.String, + MarkerIDs: parseMarkersJSONB(row.VtxoVw.Markers), + } +} + +// parseMarkersJSONB parses a JSONB array into a slice of strings +func parseMarkersJSONB(markers pqtype.NullRawMessage) []string { + if !markers.Valid || len(markers.RawMessage) == 0 { + return nil + } + var markerIDs []string + if err := json.Unmarshal(markers.RawMessage, &markerIDs); err != nil { + return nil } + return markerIDs } diff --git a/internal/infrastructure/db/postgres/migration/20260211020000_add_markers.down.sql b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.down.sql similarity index 69% rename from internal/infrastructure/db/postgres/migration/20260211020000_add_markers.down.sql rename to internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.down.sql index 43602d209..fee52f3f6 100644 --- a/internal/infrastructure/db/postgres/migration/20260211020000_add_markers.down.sql +++ b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.down.sql @@ -1,12 +1,15 @@ --- Drop marker_id column from vtxo -DROP INDEX IF EXISTS idx_vtxo_marker_id; -ALTER TABLE vtxo DROP COLUMN IF EXISTS marker_id; +-- Drop markers column from vtxo +DROP INDEX IF EXISTS idx_vtxo_markers; +ALTER TABLE vtxo DROP COLUMN IF EXISTS markers; + +-- Drop depth column from vtxo +ALTER TABLE vtxo DROP COLUMN IF EXISTS depth; -- Drop marker tables DROP TABLE IF EXISTS swept_marker; DROP TABLE IF EXISTS marker; --- Recreate views without marker_id +-- Recreate views without depth and markers columns DROP VIEW IF EXISTS intent_with_inputs_vw; DROP VIEW IF EXISTS vtxo_vw; diff --git a/internal/infrastructure/db/postgres/migration/20260211020000_add_markers.up.sql b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql similarity index 57% rename from internal/infrastructure/db/postgres/migration/20260211020000_add_markers.up.sql rename to internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql index d1bda68b3..4503bc723 100644 --- a/internal/infrastructure/db/postgres/migration/20260211020000_add_markers.up.sql +++ b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql @@ -1,4 +1,7 @@ --- Create markers table +-- Add depth column +ALTER TABLE vtxo ADD COLUMN IF NOT EXISTS depth INTEGER NOT NULL DEFAULT 0; + +-- Create marker table CREATE TABLE IF NOT EXISTS marker ( id TEXT PRIMARY KEY, depth INTEGER NOT NULL, @@ -6,17 +9,17 @@ CREATE TABLE IF NOT EXISTS marker ( ); CREATE INDEX IF NOT EXISTS idx_marker_depth ON marker(depth); --- Create swept_markers table (append-only) +-- Create swept_marker table (append-only) CREATE TABLE IF NOT EXISTS swept_marker ( marker_id TEXT PRIMARY KEY REFERENCES marker(id), swept_at BIGINT NOT NULL ); --- Add marker_id column to vtxo table -ALTER TABLE vtxo ADD COLUMN marker_id TEXT REFERENCES marker(id); -CREATE INDEX IF NOT EXISTS idx_vtxo_marker_id ON vtxo(marker_id); +-- Add markers column (JSONB array) +ALTER TABLE vtxo ADD COLUMN IF NOT EXISTS markers JSONB; +CREATE INDEX IF NOT EXISTS idx_vtxo_markers ON vtxo USING GIN (markers); --- Recreate views to include the new marker_id column +-- Recreate views to include the new columns DROP VIEW IF EXISTS intent_with_inputs_vw; DROP VIEW IF EXISTS vtxo_vw; @@ -38,19 +41,14 @@ LEFT OUTER JOIN vtxo_vw ON intent.id = vtxo_vw.intent_id; -- Backfill markers for existing VTXOs based on their depth --- VTXOs at depth 0, 100, 200, ... get their own markers --- Other VTXOs will have their marker_id set during PR 5 (marker assignment logic) - --- First, create markers for all existing VTXOs at marker boundary depths (depth % 100 == 0) INSERT INTO marker (id, depth, parent_markers) SELECT - v.txid || ':' || v.vout, -- Use VTXO outpoint as marker ID + v.txid || ':' || v.vout, v.depth, - '[]'::jsonb -- Empty parent markers for initial backfill + '[]'::jsonb FROM vtxo v WHERE v.depth % 100 = 0; --- Assign marker_id to VTXOs at boundary depths -UPDATE vtxo -SET marker_id = txid || ':' || vout +-- Assign markers array to VTXOs at boundary depths +UPDATE vtxo SET markers = jsonb_build_array(txid || ':' || vout) WHERE depth % 100 = 0; diff --git a/internal/infrastructure/db/postgres/migration/20260210100000_add_vtxo_depth.down.sql b/internal/infrastructure/db/postgres/migration/20260210100000_add_vtxo_depth.down.sql deleted file mode 100644 index 94fd10573..000000000 --- a/internal/infrastructure/db/postgres/migration/20260210100000_add_vtxo_depth.down.sql +++ /dev/null @@ -1,22 +0,0 @@ --- Recreate views without depth column -DROP VIEW IF EXISTS intent_with_inputs_vw; -DROP VIEW IF EXISTS vtxo_vw; - -ALTER TABLE vtxo DROP COLUMN IF EXISTS depth; - -CREATE VIEW vtxo_vw AS -SELECT v.*, string_agg(vc.commitment_txid, ',') AS commitments -FROM vtxo v -LEFT JOIN vtxo_commitment_txid vc -ON v.txid = vc.vtxo_txid AND v.vout = vc.vtxo_vout -GROUP BY v.txid, v.vout; - -CREATE VIEW intent_with_inputs_vw AS -SELECT vtxo_vw.*, - intent.id, - intent.round_id, - intent.proof, - intent.message -FROM intent -LEFT OUTER JOIN vtxo_vw -ON intent.id = vtxo_vw.intent_id; diff --git a/internal/infrastructure/db/postgres/migration/20260210100000_add_vtxo_depth.up.sql b/internal/infrastructure/db/postgres/migration/20260210100000_add_vtxo_depth.up.sql deleted file mode 100644 index 2585dfb5f..000000000 --- a/internal/infrastructure/db/postgres/migration/20260210100000_add_vtxo_depth.up.sql +++ /dev/null @@ -1,23 +0,0 @@ -ALTER TABLE vtxo -ADD COLUMN IF NOT EXISTS depth INTEGER NOT NULL DEFAULT 0; - --- Recreate views to include the new depth column -DROP VIEW IF EXISTS intent_with_inputs_vw; -DROP VIEW IF EXISTS vtxo_vw; - -CREATE VIEW vtxo_vw AS -SELECT v.*, string_agg(vc.commitment_txid, ',') AS commitments -FROM vtxo v -LEFT JOIN vtxo_commitment_txid vc -ON v.txid = vc.vtxo_txid AND v.vout = vc.vtxo_vout -GROUP BY v.txid, v.vout; - -CREATE VIEW intent_with_inputs_vw AS -SELECT vtxo_vw.*, - intent.id, - intent.round_id, - intent.proof, - intent.message -FROM intent -LEFT OUTER JOIN vtxo_vw -ON intent.id = vtxo_vw.intent_id; diff --git a/internal/infrastructure/db/postgres/sqlc/queries/models.go b/internal/infrastructure/db/postgres/sqlc/queries/models.go index d9e276c09..7eeebaac9 100644 --- a/internal/infrastructure/db/postgres/sqlc/queries/models.go +++ b/internal/infrastructure/db/postgres/sqlc/queries/models.go @@ -65,7 +65,7 @@ type IntentWithInputsVw struct { IntentID sql.NullString UpdatedAt sql.NullInt64 Depth sql.NullInt32 - MarkerID sql.NullString + Markers pqtype.NullRawMessage Commitments []byte ID sql.NullString RoundID sql.NullString @@ -226,7 +226,7 @@ type Vtxo struct { IntentID sql.NullString UpdatedAt int64 Depth int32 - MarkerID sql.NullString + Markers pqtype.NullRawMessage } type VtxoCommitmentTxid struct { @@ -253,6 +253,6 @@ type VtxoVw struct { IntentID sql.NullString UpdatedAt int64 Depth int32 - MarkerID sql.NullString + Markers pqtype.NullRawMessage Commitments []byte } diff --git a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go index c4ccb8188..5aa1fb9a0 100644 --- a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go @@ -8,6 +8,7 @@ package queries import ( "context" "database/sql" + "encoding/json" "github.com/lib/pq" "github.com/sqlc-dev/pqtype" @@ -242,7 +243,7 @@ func (q *Queries) SelectAllRoundIds(ctx context.Context) ([]string, error) { } const selectAllVtxos = `-- name: SelectAllVtxos :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw ` type SelectAllVtxosRow struct { @@ -276,7 +277,7 @@ func (q *Queries) SelectAllVtxos(ctx context.Context) ([]SelectAllVtxosRow, erro &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -578,7 +579,7 @@ func (q *Queries) SelectMarkersByIds(ctx context.Context, ids []string) ([]Marke } const selectNotUnrolledVtxos = `-- name: SelectNotUnrolledVtxos :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false ` type SelectNotUnrolledVtxosRow struct { @@ -612,7 +613,7 @@ func (q *Queries) SelectNotUnrolledVtxos(ctx context.Context) ([]SelectNotUnroll &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -629,7 +630,7 @@ func (q *Queries) SelectNotUnrolledVtxos(ctx context.Context) ([]SelectNotUnroll } const selectNotUnrolledVtxosWithPubkey = `-- name: SelectNotUnrolledVtxosWithPubkey :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false AND pubkey = $1 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false AND pubkey = $1 ` type SelectNotUnrolledVtxosWithPubkeyRow struct { @@ -663,7 +664,7 @@ func (q *Queries) SelectNotUnrolledVtxosWithPubkey(ctx context.Context, pubkey s &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -724,7 +725,7 @@ func (q *Queries) SelectOffchainTx(ctx context.Context, txid string) ([]SelectOf } const selectPendingSpentVtxo = `-- name: SelectPendingSpentVtxo :one -SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.marker_id, v.commitments +SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.markers, v.commitments FROM vtxo_vw v WHERE v.txid = $1 AND v.vout = $2 AND v.spent = TRUE AND v.unrolled = FALSE and COALESCE(v.settled_by, '') = '' @@ -759,14 +760,14 @@ func (q *Queries) SelectPendingSpentVtxo(ctx context.Context, arg SelectPendingS &i.IntentID, &i.UpdatedAt, &i.Depth, - &i.MarkerID, + &i.Markers, &i.Commitments, ) return i, err } const selectPendingSpentVtxosWithPubkeys = `-- name: SelectPendingSpentVtxosWithPubkeys :many -SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.marker_id, v.commitments +SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.markers, v.commitments FROM vtxo_vw v WHERE v.spent = TRUE AND v.unrolled = FALSE and COALESCE(v.settled_by, '') = '' AND v.pubkey = ANY($1::varchar[]) @@ -810,7 +811,7 @@ func (q *Queries) SelectPendingSpentVtxosWithPubkeys(ctx context.Context, arg Se &i.IntentID, &i.UpdatedAt, &i.Depth, - &i.MarkerID, + &i.Markers, &i.Commitments, ); err != nil { return nil, err @@ -1110,7 +1111,7 @@ func (q *Queries) SelectRoundVtxoTree(ctx context.Context, txid string) ([]Tx, e } const selectRoundVtxoTreeLeaves = `-- name: SelectRoundVtxoTreeLeaves :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE commitment_txid = $1 AND preconfirmed = false +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE commitment_txid = $1 AND preconfirmed = false ` type SelectRoundVtxoTreeLeavesRow struct { @@ -1144,7 +1145,7 @@ func (q *Queries) SelectRoundVtxoTreeLeaves(ctx context.Context, commitmentTxid &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -1165,7 +1166,7 @@ SELECT round.id, round.starting_timestamp, round.ending_timestamp, round.ended, round_intents_vw.id, round_intents_vw.round_id, round_intents_vw.proof, round_intents_vw.message, round_txs_vw.txid, round_txs_vw.tx, round_txs_vw.round_id, round_txs_vw.type, round_txs_vw.position, round_txs_vw.children, intent_with_receivers_vw.intent_id, intent_with_receivers_vw.pubkey, intent_with_receivers_vw.onchain_address, intent_with_receivers_vw.amount, intent_with_receivers_vw.id, intent_with_receivers_vw.round_id, intent_with_receivers_vw.proof, intent_with_receivers_vw.message, - intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.marker_id, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message + intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.markers, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message FROM round LEFT OUTER JOIN round_intents_vw ON round.id=round_intents_vw.round_id LEFT OUTER JOIN round_txs_vw ON round.id=round_txs_vw.round_id @@ -1238,7 +1239,7 @@ func (q *Queries) SelectRoundWithId(ctx context.Context, id string) ([]SelectRou &i.IntentWithInputsVw.IntentID, &i.IntentWithInputsVw.UpdatedAt, &i.IntentWithInputsVw.Depth, - &i.IntentWithInputsVw.MarkerID, + &i.IntentWithInputsVw.Markers, &i.IntentWithInputsVw.Commitments, &i.IntentWithInputsVw.ID, &i.IntentWithInputsVw.RoundID, @@ -1263,7 +1264,7 @@ SELECT round.id, round.starting_timestamp, round.ending_timestamp, round.ended, round_intents_vw.id, round_intents_vw.round_id, round_intents_vw.proof, round_intents_vw.message, round_txs_vw.txid, round_txs_vw.tx, round_txs_vw.round_id, round_txs_vw.type, round_txs_vw.position, round_txs_vw.children, intent_with_receivers_vw.intent_id, intent_with_receivers_vw.pubkey, intent_with_receivers_vw.onchain_address, intent_with_receivers_vw.amount, intent_with_receivers_vw.id, intent_with_receivers_vw.round_id, intent_with_receivers_vw.proof, intent_with_receivers_vw.message, - intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.marker_id, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message + intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.markers, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message FROM round LEFT OUTER JOIN round_intents_vw ON round.id=round_intents_vw.round_id LEFT OUTER JOIN round_txs_vw ON round.id=round_txs_vw.round_id @@ -1338,7 +1339,7 @@ func (q *Queries) SelectRoundWithTxid(ctx context.Context, txid string) ([]Selec &i.IntentWithInputsVw.IntentID, &i.IntentWithInputsVw.UpdatedAt, &i.IntentWithInputsVw.Depth, - &i.IntentWithInputsVw.MarkerID, + &i.IntentWithInputsVw.Markers, &i.IntentWithInputsVw.Commitments, &i.IntentWithInputsVw.ID, &i.IntentWithInputsVw.RoundID, @@ -1418,7 +1419,7 @@ func (q *Queries) SelectSweepableRounds(ctx context.Context) ([]string, error) { } const selectSweepableUnrolledVtxos = `-- name: SelectSweepableUnrolledVtxos :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE spent = true AND unrolled = true AND swept = false AND COALESCE(settled_by, '') = '' +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE spent = true AND unrolled = true AND swept = false AND COALESCE(settled_by, '') = '' ` type SelectSweepableUnrolledVtxosRow struct { @@ -1452,7 +1453,7 @@ func (q *Queries) SelectSweepableUnrolledVtxos(ctx context.Context) ([]SelectSwe &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -1607,7 +1608,7 @@ func (q *Queries) SelectTxs(ctx context.Context, dollar_1 []string) ([]SelectTxs } const selectVtxo = `-- name: SelectVtxo :one -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE txid = $1 AND vout = $2 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE txid = $1 AND vout = $2 ` type SelectVtxoParams struct { @@ -1640,19 +1641,19 @@ func (q *Queries) SelectVtxo(ctx context.Context, arg SelectVtxoParams) (SelectV &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ) return i, err } const selectVtxoChainByMarker = `-- name: SelectVtxoChainByMarker :many -SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, marker_id, commitments FROM vtxo_vw -WHERE marker_id = ANY($1::TEXT[]) +SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, markers, commitments FROM vtxo_vw +WHERE markers ?| $1::TEXT[] ORDER BY depth DESC ` -// Get VTXOs that share the same marker or have markers in the parent chain +// Get VTXOs whose markers JSONB array contains any of the given marker IDs func (q *Queries) SelectVtxoChainByMarker(ctx context.Context, markerIds []string) ([]VtxoVw, error) { rows, err := q.db.QueryContext(ctx, selectVtxoChainByMarker, pq.Array(markerIds)) if err != nil { @@ -1680,7 +1681,7 @@ func (q *Queries) SelectVtxoChainByMarker(ctx context.Context, markerIds []strin &i.IntentID, &i.UpdatedAt, &i.Depth, - &i.MarkerID, + &i.Markers, &i.Commitments, ); err != nil { return nil, err @@ -1733,7 +1734,7 @@ func (q *Queries) SelectVtxoPubKeysByCommitmentTxid(ctx context.Context, arg Sel } const selectVtxosByArkTxid = `-- name: SelectVtxosByArkTxid :many -SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, marker_id, commitments FROM vtxo_vw WHERE txid = $1 +SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, markers, commitments FROM vtxo_vw WHERE txid = $1 ` // Get all VTXOs created by a specific ark tx (offchain tx) @@ -1764,7 +1765,7 @@ func (q *Queries) SelectVtxosByArkTxid(ctx context.Context, arkTxid string) ([]V &i.IntentID, &i.UpdatedAt, &i.Depth, - &i.MarkerID, + &i.Markers, &i.Commitments, ); err != nil { return nil, err @@ -1782,7 +1783,7 @@ func (q *Queries) SelectVtxosByArkTxid(ctx context.Context, arkTxid string) ([]V const selectVtxosByDepthRange = `-- name: SelectVtxosByDepthRange :many -SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, marker_id, commitments FROM vtxo_vw +SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, markers, commitments FROM vtxo_vw WHERE depth >= $1 AND depth <= $2 ORDER BY depth DESC ` @@ -1821,7 +1822,7 @@ func (q *Queries) SelectVtxosByDepthRange(ctx context.Context, arg SelectVtxosBy &i.IntentID, &i.UpdatedAt, &i.Depth, - &i.MarkerID, + &i.Markers, &i.Commitments, ); err != nil { return nil, err @@ -1838,14 +1839,15 @@ func (q *Queries) SelectVtxosByDepthRange(ctx context.Context, arg SelectVtxosBy } const selectVtxosByMarkerId = `-- name: SelectVtxosByMarkerId :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE marker_id = $1 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE markers @> jsonb_build_array($1::TEXT) ` type SelectVtxosByMarkerIdRow struct { VtxoVw VtxoVw } -func (q *Queries) SelectVtxosByMarkerId(ctx context.Context, markerID sql.NullString) ([]SelectVtxosByMarkerIdRow, error) { +// Find VTXOs whose markers JSONB array contains the given marker_id +func (q *Queries) SelectVtxosByMarkerId(ctx context.Context, markerID string) ([]SelectVtxosByMarkerIdRow, error) { rows, err := q.db.QueryContext(ctx, selectVtxosByMarkerId, markerID) if err != nil { return nil, err @@ -1872,7 +1874,7 @@ func (q *Queries) SelectVtxosByMarkerId(ctx context.Context, markerID sql.NullSt &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -1950,7 +1952,7 @@ func (q *Queries) SelectVtxosOutpointsByArkTxidRecursive(ctx context.Context, tx } const selectVtxosWithPubkeys = `-- name: SelectVtxosWithPubkeys :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE vtxo_vw.pubkey = ANY($1::varchar[]) AND vtxo_vw.updated_at >= $2::bigint AND ($3::bigint = 0 OR vtxo_vw.updated_at <= $3::bigint) @@ -1993,7 +1995,7 @@ func (q *Queries) SelectVtxosWithPubkeys(ctx context.Context, arg SelectVtxosWit &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -2011,10 +2013,11 @@ func (q *Queries) SelectVtxosWithPubkeys(ctx context.Context, arg SelectVtxosWit const sweepVtxosByMarkerId = `-- name: SweepVtxosByMarkerId :execrows UPDATE vtxo SET swept = true, updated_at = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT -WHERE marker_id = $1 AND swept = false +WHERE markers @> jsonb_build_array($1::TEXT) AND swept = false ` -func (q *Queries) SweepVtxosByMarkerId(ctx context.Context, markerID sql.NullString) (int64, error) { +// Sweep VTXOs whose markers JSONB array contains the given marker_id +func (q *Queries) SweepVtxosByMarkerId(ctx context.Context, markerID string) (int64, error) { result, err := q.db.ExecContext(ctx, sweepVtxosByMarkerId, markerID) if err != nil { return 0, err @@ -2061,18 +2064,18 @@ func (q *Queries) UpdateVtxoIntentId(ctx context.Context, arg UpdateVtxoIntentId return err } -const updateVtxoMarkerId = `-- name: UpdateVtxoMarkerId :exec -UPDATE vtxo SET marker_id = $1 WHERE txid = $2 AND vout = $3 +const updateVtxoMarkers = `-- name: UpdateVtxoMarkers :exec +UPDATE vtxo SET markers = $1::jsonb WHERE txid = $2 AND vout = $3 ` -type UpdateVtxoMarkerIdParams struct { - MarkerID sql.NullString - Txid string - Vout int32 +type UpdateVtxoMarkersParams struct { + Markers json.RawMessage + Txid string + Vout int32 } -func (q *Queries) UpdateVtxoMarkerId(ctx context.Context, arg UpdateVtxoMarkerIdParams) error { - _, err := q.db.ExecContext(ctx, updateVtxoMarkerId, arg.MarkerID, arg.Txid, arg.Vout) +func (q *Queries) UpdateVtxoMarkers(ctx context.Context, arg UpdateVtxoMarkersParams) error { + _, err := q.db.ExecContext(ctx, updateVtxoMarkers, arg.Markers, arg.Txid, arg.Vout) return err } diff --git a/internal/infrastructure/db/postgres/sqlc/query.sql b/internal/infrastructure/db/postgres/sqlc/query.sql index 83b12187c..a959852c2 100644 --- a/internal/infrastructure/db/postgres/sqlc/query.sql +++ b/internal/infrastructure/db/postgres/sqlc/query.sql @@ -476,15 +476,17 @@ WITH RECURSIVE descendant_markers(id) AS ( SELECT descendant_markers.id AS marker_id FROM descendant_markers WHERE descendant_markers.id NOT IN (SELECT sm.marker_id FROM swept_marker sm); --- name: UpdateVtxoMarkerId :exec -UPDATE vtxo SET marker_id = @marker_id WHERE txid = @txid AND vout = @vout; +-- name: UpdateVtxoMarkers :exec +UPDATE vtxo SET markers = @markers::jsonb WHERE txid = @txid AND vout = @vout; -- name: SelectVtxosByMarkerId :many -SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE marker_id = @marker_id; +-- Find VTXOs whose markers JSONB array contains the given marker_id +SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE markers @> jsonb_build_array(@marker_id::TEXT); -- name: SweepVtxosByMarkerId :execrows +-- Sweep VTXOs whose markers JSONB array contains the given marker_id UPDATE vtxo SET swept = true, updated_at = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT -WHERE marker_id = @marker_id AND swept = false; +WHERE markers @> jsonb_build_array(@marker_id::TEXT) AND swept = false; -- Chain traversal queries for GetVtxoChain optimization @@ -499,7 +501,7 @@ ORDER BY depth DESC; SELECT * FROM vtxo_vw WHERE txid = @ark_txid; -- name: SelectVtxoChainByMarker :many --- Get VTXOs that share the same marker or have markers in the parent chain +-- Get VTXOs whose markers JSONB array contains any of the given marker IDs SELECT * FROM vtxo_vw -WHERE marker_id = ANY(@marker_ids::TEXT[]) +WHERE markers ?| @marker_ids::TEXT[] ORDER BY depth DESC; \ No newline at end of file diff --git a/internal/infrastructure/db/postgres/vtxo_repo.go b/internal/infrastructure/db/postgres/vtxo_repo.go index bfd090527..3f539a667 100644 --- a/internal/infrastructure/db/postgres/vtxo_repo.go +++ b/internal/infrastructure/db/postgres/vtxo_repo.go @@ -3,12 +3,14 @@ package pgdb import ( "context" "database/sql" + "encoding/json" "errors" "fmt" "sort" "github.com/arkade-os/arkd/internal/core/domain" "github.com/arkade-os/arkd/internal/infrastructure/db/postgres/sqlc/queries" + "github.com/sqlc-dev/pqtype" ) type vtxoRepository struct { @@ -526,10 +528,22 @@ func rowToVtxo(row queries.VtxoVw) domain.Vtxo { ExpiresAt: row.ExpiresAt, CreatedAt: row.CreatedAt, Depth: uint32(row.Depth), - MarkerID: row.MarkerID.String, + MarkerIDs: parseMarkersJSONBFromVtxo(row.Markers), } } +// parseMarkersJSONBFromVtxo parses a JSONB array into a slice of strings for vtxo repo +func parseMarkersJSONBFromVtxo(markers pqtype.NullRawMessage) []string { + if !markers.Valid || len(markers.RawMessage) == 0 { + return nil + } + var markerIDs []string + if err := json.Unmarshal(markers.RawMessage, &markerIDs); err != nil { + return nil + } + return markerIDs +} + func readRows(rows []queries.VtxoVw) ([]domain.Vtxo, error) { vtxos := make([]domain.Vtxo, 0, len(rows)) for _, vtxo := range rows { diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index 6b61915a0..02af6ce8b 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -520,9 +520,9 @@ func (s *service) updateProjectionsAfterRoundEvents(events []domain.Event) { Warnf("failed to create root marker for vtxo %s", vtxo.Outpoint.String()) continue } - if err := s.markerStore.UpdateVtxoMarker(ctx, vtxo.Outpoint, markerID); err != nil { + if err := s.markerStore.UpdateVtxoMarkers(ctx, vtxo.Outpoint, []string{markerID}); err != nil { log.WithError(err). - Warnf("failed to update marker_id for vtxo %s", vtxo.Outpoint.String()) + Warnf("failed to update markers for vtxo %s", vtxo.Outpoint.String()) } } log.Debugf("created %d root markers for batch vtxos", len(newVtxos)) @@ -594,9 +594,11 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) if v.Depth > maxDepth { maxDepth = v.Depth } - // Collect parent marker IDs for marker linking (will be used at boundary) - if v.MarkerID != "" { - parentMarkerSet[v.MarkerID] = struct{}{} + // Collect ALL parent marker IDs for marker linking + for _, markerID := range v.MarkerIDs { + if markerID != "" { + parentMarkerSet[markerID] = struct{}{} + } } } newDepth = maxDepth + 1 @@ -606,27 +608,27 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) } } - // Create marker if at boundary depth, or inherit from parent - var markerID string + // Create marker if at boundary depth, or inherit ALL parent markers + var markerIDs []string if s.markerStore != nil { if domain.IsAtMarkerBoundary(newDepth) { // Create marker ID from the first output (the ark tx id + first vtxo vout) - markerID = fmt.Sprintf("%s:marker:%d", txid, newDepth) + newMarkerID := fmt.Sprintf("%s:marker:%d", txid, newDepth) marker := domain.Marker{ - ID: markerID, + ID: newMarkerID, Depth: newDepth, ParentMarkerIDs: parentMarkerIDs, } if err := s.markerStore.AddMarker(ctx, marker); err != nil { log.WithError(err).Warn("failed to create marker for chained vtxo") // Continue without marker - non-fatal - markerID = "" } else { - log.Debugf("created marker %s at depth %d", markerID, newDepth) + log.Debugf("created marker %s at depth %d", newMarkerID, newDepth) + markerIDs = []string{newMarkerID} } } else if len(parentMarkerIDs) > 0 { - // Inherit marker from parent at non-boundary depth - markerID = parentMarkerIDs[0] + // Inherit ALL markers from parents at non-boundary depth + markerIDs = parentMarkerIDs } } @@ -667,12 +669,12 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) } log.Debugf("added %d vtxos at depth %d", len(newVtxos), newDepth) - // Update marker_id for VTXOs (new marker at boundary, inherited at non-boundary) - if markerID != "" && s.markerStore != nil { + // Update markers for VTXOs (new marker at boundary, inherited at non-boundary) + if len(markerIDs) > 0 && s.markerStore != nil { for _, vtxo := range newVtxos { - if err := s.markerStore.UpdateVtxoMarker(ctx, vtxo.Outpoint, markerID); err != nil { + if err := s.markerStore.UpdateVtxoMarkers(ctx, vtxo.Outpoint, markerIDs); err != nil { log.WithError(err). - Warnf("failed to update marker_id for vtxo %s", vtxo.Outpoint.String()) + Warnf("failed to update markers for vtxo %s", vtxo.Outpoint.String()) } } } @@ -769,13 +771,15 @@ func (s *service) sweepVtxosWithMarkers( return int64(count) } - // Group VTXOs by marker ID + // Group VTXOs by their first marker ID (for sweep optimization) + // We use first marker to avoid duplicate sweeps when vtxo has multiple markers markerVtxos := make(map[string][]domain.Outpoint) noMarkerVtxos := make([]domain.Outpoint, 0) for _, vtxo := range vtxos { - if vtxo.MarkerID != "" { - markerVtxos[vtxo.MarkerID] = append(markerVtxos[vtxo.MarkerID], vtxo.Outpoint) + if len(vtxo.MarkerIDs) > 0 { + // Use first marker ID for grouping + markerVtxos[vtxo.MarkerIDs[0]] = append(markerVtxos[vtxo.MarkerIDs[0]], vtxo.Outpoint) } else { noMarkerVtxos = append(noMarkerVtxos, vtxo.Outpoint) } diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index de9eb8450..2d94b687d 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -1672,16 +1672,16 @@ func testVtxoMarkerAssociation(t *testing.T, svc ports.RepoManager) { err = svc.Vtxos().AddVtxos(ctx, []domain.Vtxo{vtxo1, vtxo2, vtxo3}) require.NoError(t, err) - // Verify VTXOs initially have no marker_id + // Verify VTXOs initially have no markers retrievedVtxos, err := svc.Vtxos().GetVtxos(ctx, []domain.Outpoint{vtxo1.Outpoint}) require.NoError(t, err) require.Len(t, retrievedVtxos, 1) - require.Empty(t, retrievedVtxos[0].MarkerID) + require.Empty(t, retrievedVtxos[0].MarkerIDs) - // Call UpdateVtxoMarker to associate VTXOs with marker - err = svc.Markers().UpdateVtxoMarker(ctx, vtxo1.Outpoint, markerID) + // Call UpdateVtxoMarkers to associate VTXOs with marker + err = svc.Markers().UpdateVtxoMarkers(ctx, vtxo1.Outpoint, []string{markerID}) require.NoError(t, err) - err = svc.Markers().UpdateVtxoMarker(ctx, vtxo2.Outpoint, markerID) + err = svc.Markers().UpdateVtxoMarkers(ctx, vtxo2.Outpoint, []string{markerID}) require.NoError(t, err) // Verify GetVtxosByMarker returns the associated VTXOs @@ -1698,20 +1698,20 @@ func testVtxoMarkerAssociation(t *testing.T, svc ports.RepoManager) { outpoints, ) - // Verify VTXO.MarkerID field is populated when retrieved via GetVtxos + // Verify VTXO.MarkerIDs field is populated when retrieved via GetVtxos retrievedVtxos, err = svc.Vtxos(). GetVtxos(ctx, []domain.Outpoint{vtxo1.Outpoint, vtxo2.Outpoint}) require.NoError(t, err) require.Len(t, retrievedVtxos, 2) for _, v := range retrievedVtxos { - require.Equal(t, markerID, v.MarkerID) + require.Contains(t, v.MarkerIDs, markerID) } - // Verify vtxo3 still has no marker + // Verify vtxo3 still has no markers retrievedVtxos, err = svc.Vtxos().GetVtxos(ctx, []domain.Outpoint{vtxo3.Outpoint}) require.NoError(t, err) require.Len(t, retrievedVtxos, 1) - require.Empty(t, retrievedVtxos[0].MarkerID) + require.Empty(t, retrievedVtxos[0].MarkerIDs) // Test GetVtxosByMarker with non-existent marker vtxosByNonExistent, err := svc.Markers().GetVtxosByMarker(ctx, "nonexistent") @@ -1757,7 +1757,7 @@ func testSweepVtxosByMarker(t *testing.T, svc ports.RepoManager) { // Associate all VTXOs with the marker for _, v := range vtxos { - err = svc.Markers().UpdateVtxoMarker(ctx, v.Outpoint, markerID) + err = svc.Markers().UpdateVtxoMarkers(ctx, v.Outpoint, []string{markerID}) require.NoError(t, err) } @@ -2022,12 +2022,12 @@ func testMarkerChainTraversal(t *testing.T, svc ports.RepoManager) { err = svc.Vtxos().AddVtxos(ctx, []domain.Vtxo{vtxo1, vtxo2, vtxo3}) require.NoError(t, err) - // Associate VTXOs with their markers using UpdateVtxoMarker - err = svc.Markers().UpdateVtxoMarker(ctx, vtxo1.Outpoint, marker1.ID) + // Associate VTXOs with their markers using UpdateVtxoMarkers + err = svc.Markers().UpdateVtxoMarkers(ctx, vtxo1.Outpoint, []string{marker1.ID}) require.NoError(t, err) - err = svc.Markers().UpdateVtxoMarker(ctx, vtxo2.Outpoint, marker1.ID) + err = svc.Markers().UpdateVtxoMarkers(ctx, vtxo2.Outpoint, []string{marker1.ID}) require.NoError(t, err) - err = svc.Markers().UpdateVtxoMarker(ctx, vtxo3.Outpoint, marker2.ID) + err = svc.Markers().UpdateVtxoMarkers(ctx, vtxo3.Outpoint, []string{marker2.ID}) require.NoError(t, err) // Test GetVtxoChainByMarkers - returns VTXOs for given marker list @@ -2150,7 +2150,7 @@ func testGetVtxoChainWithMarkerOptimization(t *testing.T, svc ports.RepoManager) // Associate VTXOs with their markers for _, v := range vtxos { markerID := vtxoMarkerMap[v.Outpoint.String()] - err = svc.Markers().UpdateVtxoMarker(ctx, v.Outpoint, markerID) + err = svc.Markers().UpdateVtxoMarkers(ctx, v.Outpoint, []string{markerID}) require.NoError(t, err) } @@ -2160,7 +2160,7 @@ func testGetVtxoChainWithMarkerOptimization(t *testing.T, svc ports.RepoManager) require.NoError(t, err) require.Len(t, retrievedVtxos, 1) expectedMarker := vtxoMarkerMap[v.Outpoint.String()] - require.Equal(t, expectedMarker, retrievedVtxos[0].MarkerID, + require.Contains(t, retrievedVtxos[0].MarkerIDs, expectedMarker, "VTXO at depth %d should have marker %s", v.Depth, expectedMarker) } diff --git a/internal/infrastructure/db/sqlite/marker_repo.go b/internal/infrastructure/db/sqlite/marker_repo.go index 9307d2336..ee7c7cd63 100644 --- a/internal/infrastructure/db/sqlite/marker_repo.go +++ b/internal/infrastructure/db/sqlite/marker_repo.go @@ -196,15 +196,19 @@ func (m *markerRepository) GetSweptMarkers( return sweptMarkers, nil } -func (m *markerRepository) UpdateVtxoMarker( +func (m *markerRepository) UpdateVtxoMarkers( ctx context.Context, outpoint domain.Outpoint, - markerID string, + markerIDs []string, ) error { - return m.querier.UpdateVtxoMarkerId(ctx, queries.UpdateVtxoMarkerIdParams{ - MarkerID: sql.NullString{String: markerID, Valid: len(markerID) > 0}, - Txid: outpoint.Txid, - Vout: int64(outpoint.VOut), + markersJSON, err := json.Marshal(markerIDs) + if err != nil { + return fmt.Errorf("failed to marshal markers: %w", err) + } + return m.querier.UpdateVtxoMarkers(ctx, queries.UpdateVtxoMarkersParams{ + Markers: sql.NullString{String: string(markersJSON), Valid: len(markerIDs) > 0}, + Txid: outpoint.Txid, + Vout: int64(outpoint.VOut), }) } @@ -277,20 +281,24 @@ func (m *markerRepository) GetVtxoChainByMarkers( return nil, nil } - // Convert string slice to sql.NullString slice - nullStringIDs := make([]sql.NullString, len(markerIDs)) - for i, id := range markerIDs { - nullStringIDs[i] = sql.NullString{String: id, Valid: true} - } + // Since SQLite query handles one marker at a time, we need to query for each marker + // and deduplicate results + seen := make(map[string]bool) + vtxos := make([]domain.Vtxo, 0) - rows, err := m.querier.SelectVtxoChainByMarker(ctx, nullStringIDs) - if err != nil { - return nil, err - } + for _, markerID := range markerIDs { + rows, err := m.querier.SelectVtxoChainByMarker(ctx, sql.NullString{String: markerID, Valid: true}) + if err != nil { + return nil, err + } - vtxos := make([]domain.Vtxo, 0, len(rows)) - for _, row := range rows { - vtxos = append(vtxos, rowToVtxoFromChainQuery(row)) + for _, row := range rows { + key := row.VtxoVw.Txid + ":" + fmt.Sprintf("%d", row.VtxoVw.Vout) + if !seen[key] { + seen[key] = true + vtxos = append(vtxos, rowToVtxoFromChainQuery(row)) + } + } } return vtxos, nil } @@ -334,7 +342,7 @@ func rowToVtxoFromMarkerQuery(row queries.SelectVtxosByMarkerIdRow) domain.Vtxo ExpiresAt: row.VtxoVw.ExpiresAt, CreatedAt: row.VtxoVw.CreatedAt, Depth: uint32(row.VtxoVw.Depth), - MarkerID: row.VtxoVw.MarkerID.String, + MarkerIDs: parseMarkersJSON(row.VtxoVw.Markers.String), } } @@ -362,7 +370,7 @@ func rowToVtxoFromDepthRangeQuery(row queries.SelectVtxosByDepthRangeRow) domain ExpiresAt: row.VtxoVw.ExpiresAt, CreatedAt: row.VtxoVw.CreatedAt, Depth: uint32(row.VtxoVw.Depth), - MarkerID: row.VtxoVw.MarkerID.String, + MarkerIDs: parseMarkersJSON(row.VtxoVw.Markers.String), } } @@ -390,7 +398,7 @@ func rowToVtxoFromArkTxidQuery(row queries.SelectVtxosByArkTxidRow) domain.Vtxo ExpiresAt: row.VtxoVw.ExpiresAt, CreatedAt: row.VtxoVw.CreatedAt, Depth: uint32(row.VtxoVw.Depth), - MarkerID: row.VtxoVw.MarkerID.String, + MarkerIDs: parseMarkersJSON(row.VtxoVw.Markers.String), } } @@ -418,6 +426,18 @@ func rowToVtxoFromChainQuery(row queries.SelectVtxoChainByMarkerRow) domain.Vtxo ExpiresAt: row.VtxoVw.ExpiresAt, CreatedAt: row.VtxoVw.CreatedAt, Depth: uint32(row.VtxoVw.Depth), - MarkerID: row.VtxoVw.MarkerID.String, + MarkerIDs: parseMarkersJSON(row.VtxoVw.Markers.String), + } +} + +// parseMarkersJSON parses a JSON array string into a slice of strings +func parseMarkersJSON(markersJSON string) []string { + if markersJSON == "" { + return nil + } + var markerIDs []string + if err := json.Unmarshal([]byte(markersJSON), &markerIDs); err != nil { + return nil } + return markerIDs } diff --git a/internal/infrastructure/db/sqlite/migration/20260211000000_add_markers.down.sql b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.down.sql similarity index 88% rename from internal/infrastructure/db/sqlite/migration/20260211000000_add_markers.down.sql rename to internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.down.sql index c58d58f05..7f9fefbeb 100644 --- a/internal/infrastructure/db/sqlite/migration/20260211000000_add_markers.down.sql +++ b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.down.sql @@ -1,11 +1,9 @@ --- Remove marker_id column from vtxo -- SQLite doesn't support DROP COLUMN directly, so we need to recreate the table --- Recreate views to remove marker_id column DROP VIEW IF EXISTS intent_with_inputs_vw; DROP VIEW IF EXISTS vtxo_vw; --- Create temp table without marker_id +-- Create temp table without depth and markers columns CREATE TABLE vtxo_temp ( txid TEXT NOT NULL, vout INTEGER NOT NULL, @@ -23,7 +21,6 @@ CREATE TABLE vtxo_temp ( ark_txid TEXT, intent_id TEXT, updated_at INTEGER, - depth INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (txid, vout), FOREIGN KEY (intent_id) REFERENCES intent(id) ); @@ -32,7 +29,7 @@ CREATE TABLE vtxo_temp ( INSERT INTO vtxo_temp SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, - intent_id, updated_at, depth + intent_id, updated_at FROM vtxo; -- Drop old table and rename @@ -46,7 +43,7 @@ CREATE INDEX IF NOT EXISTS fk_vtxo_intent_id ON vtxo(intent_id); DROP TABLE IF EXISTS swept_marker; DROP TABLE IF EXISTS marker; --- Recreate views +-- Recreate views without depth and markers columns CREATE VIEW vtxo_vw AS SELECT v.*, COALESCE(group_concat(vc.commitment_txid), '') AS commitments FROM vtxo v diff --git a/internal/infrastructure/db/sqlite/migration/20260211000000_add_markers.up.sql b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql similarity index 73% rename from internal/infrastructure/db/sqlite/migration/20260211000000_add_markers.up.sql rename to internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql index 7d0b216f8..c1daab1f9 100644 --- a/internal/infrastructure/db/sqlite/migration/20260211000000_add_markers.up.sql +++ b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql @@ -1,4 +1,7 @@ --- Create markers table +-- Add depth column +ALTER TABLE vtxo ADD COLUMN depth INTEGER NOT NULL DEFAULT 0; + +-- Create marker table CREATE TABLE IF NOT EXISTS marker ( id TEXT PRIMARY KEY, depth INTEGER NOT NULL, @@ -6,17 +9,17 @@ CREATE TABLE IF NOT EXISTS marker ( ); CREATE INDEX IF NOT EXISTS idx_marker_depth ON marker(depth); --- Create swept_markers table (append-only) +-- Create swept_marker table (append-only) CREATE TABLE IF NOT EXISTS swept_marker ( marker_id TEXT PRIMARY KEY REFERENCES marker(id), swept_at INTEGER NOT NULL ); --- Add marker_id column to vtxo table -ALTER TABLE vtxo ADD COLUMN marker_id TEXT REFERENCES marker(id); -CREATE INDEX IF NOT EXISTS idx_vtxo_marker_id ON vtxo(marker_id); +-- Add markers column (JSON array, not single marker_id) +ALTER TABLE vtxo ADD COLUMN markers TEXT; +CREATE INDEX IF NOT EXISTS idx_vtxo_markers ON vtxo(markers); --- Recreate views to include the new marker_id column +-- Recreate views to include the new columns DROP VIEW IF EXISTS intent_with_inputs_vw; DROP VIEW IF EXISTS vtxo_vw; @@ -39,7 +42,6 @@ ON intent.id = vtxo_vw.intent_id; -- Backfill markers for existing VTXOs based on their depth -- VTXOs at depth 0, 100, 200, ... get their own markers --- Other VTXOs will have their marker_id set during PR 5 (marker assignment logic) -- First, create markers for all existing VTXOs at marker boundary depths (depth % 100 == 0) INSERT INTO marker (id, depth, parent_markers) @@ -50,7 +52,6 @@ SELECT FROM vtxo v WHERE v.depth % 100 = 0; --- Assign marker_id to VTXOs at boundary depths -UPDATE vtxo -SET marker_id = txid || ':' || vout +-- Assign markers array to VTXOs at boundary depths +UPDATE vtxo SET markers = '["' || txid || ':' || vout || '"]' WHERE depth % 100 = 0; diff --git a/internal/infrastructure/db/sqlite/migration/20260210000000_add_vtxo_depth.down.sql b/internal/infrastructure/db/sqlite/migration/20260210000000_add_vtxo_depth.down.sql deleted file mode 100644 index 085a0d609..000000000 --- a/internal/infrastructure/db/sqlite/migration/20260210000000_add_vtxo_depth.down.sql +++ /dev/null @@ -1,54 +0,0 @@ --- SQLite doesn't support DROP COLUMN directly, so we need to recreate the table --- This migration creates a new table without the depth column and copies data - -DROP VIEW IF EXISTS intent_with_inputs_vw; -DROP VIEW IF EXISTS vtxo_vw; - -CREATE TABLE vtxo_new ( - txid TEXT NOT NULL, - vout INTEGER NOT NULL, - pubkey TEXT NOT NULL, - amount INTEGER NOT NULL, - expires_at INTEGER NOT NULL, - created_at INTEGER NOT NULL, - commitment_txid TEXT NOT NULL, - spent_by TEXT, - spent BOOLEAN NOT NULL DEFAULT FALSE, - unrolled BOOLEAN NOT NULL DEFAULT FALSE, - swept BOOLEAN NOT NULL DEFAULT FALSE, - preconfirmed BOOLEAN NOT NULL DEFAULT FALSE, - settled_by TEXT, - ark_txid TEXT, - intent_id TEXT, - updated_at BIGINT, - PRIMARY KEY (txid, vout), - FOREIGN KEY (intent_id) REFERENCES intent(id) -); - -INSERT INTO vtxo_new (txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, intent_id, updated_at) -SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, intent_id, updated_at -FROM vtxo; - -DROP TABLE vtxo; -ALTER TABLE vtxo_new RENAME TO vtxo; - --- Recreate foreign key index -CREATE INDEX IF NOT EXISTS fk_vtxo_intent_id ON vtxo(intent_id); - --- Recreate views without depth column -CREATE VIEW vtxo_vw AS -SELECT v.*, COALESCE(group_concat(vc.commitment_txid), '') AS commitments -FROM vtxo v -LEFT JOIN vtxo_commitment_txid vc -ON v.txid = vc.vtxo_txid AND v.vout = vc.vtxo_vout -GROUP BY v.txid, v.vout; - -CREATE VIEW intent_with_inputs_vw AS -SELECT vtxo_vw.*, - intent.id, - intent.round_id, - intent.proof, - intent.message -FROM intent -LEFT OUTER JOIN vtxo_vw -ON intent.id = vtxo_vw.intent_id; diff --git a/internal/infrastructure/db/sqlite/migration/20260210000000_add_vtxo_depth.up.sql b/internal/infrastructure/db/sqlite/migration/20260210000000_add_vtxo_depth.up.sql deleted file mode 100644 index 1bf880bcb..000000000 --- a/internal/infrastructure/db/sqlite/migration/20260210000000_add_vtxo_depth.up.sql +++ /dev/null @@ -1,22 +0,0 @@ -ALTER TABLE vtxo ADD COLUMN depth INTEGER NOT NULL DEFAULT 0; - --- Recreate views to include the new depth column -DROP VIEW IF EXISTS intent_with_inputs_vw; -DROP VIEW IF EXISTS vtxo_vw; - -CREATE VIEW vtxo_vw AS -SELECT v.*, COALESCE(group_concat(vc.commitment_txid), '') AS commitments -FROM vtxo v -LEFT JOIN vtxo_commitment_txid vc -ON v.txid = vc.vtxo_txid AND v.vout = vc.vtxo_vout -GROUP BY v.txid, v.vout; - -CREATE VIEW intent_with_inputs_vw AS -SELECT vtxo_vw.*, - intent.id, - intent.round_id, - intent.proof, - intent.message -FROM intent -LEFT OUTER JOIN vtxo_vw -ON intent.id = vtxo_vw.intent_id; diff --git a/internal/infrastructure/db/sqlite/sqlc/queries/models.go b/internal/infrastructure/db/sqlite/sqlc/queries/models.go index ff55213ba..00bf2add0 100644 --- a/internal/infrastructure/db/sqlite/sqlc/queries/models.go +++ b/internal/infrastructure/db/sqlite/sqlc/queries/models.go @@ -63,7 +63,7 @@ type IntentWithInputsVw struct { IntentID sql.NullString UpdatedAt sql.NullInt64 Depth sql.NullInt64 - MarkerID sql.NullString + Markers sql.NullString Commitments interface{} ID sql.NullString RoundID sql.NullString @@ -213,7 +213,7 @@ type Vtxo struct { IntentID sql.NullString UpdatedAt sql.NullInt64 Depth int64 - MarkerID sql.NullString + Markers sql.NullString } type VtxoCommitmentTxid struct { @@ -240,6 +240,6 @@ type VtxoVw struct { IntentID sql.NullString UpdatedAt sql.NullInt64 Depth int64 - MarkerID sql.NullString + Markers sql.NullString Commitments interface{} } diff --git a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go index a415641c8..79dbbf294 100644 --- a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go @@ -240,7 +240,7 @@ func (q *Queries) SelectAllRoundIds(ctx context.Context) ([]string, error) { } const selectAllVtxos = `-- name: SelectAllVtxos :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw ` type SelectAllVtxosRow struct { @@ -274,7 +274,7 @@ func (q *Queries) SelectAllVtxos(ctx context.Context) ([]SelectAllVtxosRow, erro &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -586,7 +586,7 @@ func (q *Queries) SelectMarkersByIds(ctx context.Context, ids []string) ([]Marke } const selectNotUnrolledVtxos = `-- name: SelectNotUnrolledVtxos :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false ` type SelectNotUnrolledVtxosRow struct { @@ -620,7 +620,7 @@ func (q *Queries) SelectNotUnrolledVtxos(ctx context.Context) ([]SelectNotUnroll &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -637,7 +637,7 @@ func (q *Queries) SelectNotUnrolledVtxos(ctx context.Context) ([]SelectNotUnroll } const selectNotUnrolledVtxosWithPubkey = `-- name: SelectNotUnrolledVtxosWithPubkey :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false AND pubkey = ?1 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false AND pubkey = ?1 ` type SelectNotUnrolledVtxosWithPubkeyRow struct { @@ -671,7 +671,7 @@ func (q *Queries) SelectNotUnrolledVtxosWithPubkey(ctx context.Context, pubkey s &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -732,7 +732,7 @@ func (q *Queries) SelectOffchainTx(ctx context.Context, txid string) ([]SelectOf } const selectPendingSpentVtxo = `-- name: SelectPendingSpentVtxo :one -SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.marker_id, v.commitments +SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.markers, v.commitments FROM vtxo_vw v WHERE v.txid = ?1 AND v.vout = ?2 AND v.spent = TRUE AND v.unrolled = FALSE AND COALESCE(v.settled_by, '') = '' @@ -767,14 +767,14 @@ func (q *Queries) SelectPendingSpentVtxo(ctx context.Context, arg SelectPendingS &i.IntentID, &i.UpdatedAt, &i.Depth, - &i.MarkerID, + &i.Markers, &i.Commitments, ) return i, err } const selectPendingSpentVtxosWithPubkeys = `-- name: SelectPendingSpentVtxosWithPubkeys :many -SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.marker_id, v.commitments +SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.markers, v.commitments FROM vtxo_vw v WHERE v.spent = TRUE AND v.unrolled = FALSE AND COALESCE(v.settled_by, '') = '' AND v.pubkey IN (/*SLICE:pubkeys*/?) @@ -830,7 +830,7 @@ func (q *Queries) SelectPendingSpentVtxosWithPubkeys(ctx context.Context, arg Se &i.IntentID, &i.UpdatedAt, &i.Depth, - &i.MarkerID, + &i.Markers, &i.Commitments, ); err != nil { return nil, err @@ -1048,7 +1048,7 @@ SELECT r.ending_timestamp, ( SELECT COALESCE(SUM(amount), 0) FROM ( - SELECT DISTINCT v2.txid, v2.vout, v2.pubkey, v2.amount, v2.expires_at, v2.created_at, v2.commitment_txid, v2.spent_by, v2.spent, v2.unrolled, v2.swept, v2.preconfirmed, v2.settled_by, v2.ark_txid, v2.intent_id, v2.updated_at, v2.depth, v2.marker_id FROM vtxo v2 JOIN intent i2 ON i2.id = v2.intent_id WHERE i2.round_id = r.id + SELECT DISTINCT v2.txid, v2.vout, v2.pubkey, v2.amount, v2.expires_at, v2.created_at, v2.commitment_txid, v2.spent_by, v2.spent, v2.unrolled, v2.swept, v2.preconfirmed, v2.settled_by, v2.ark_txid, v2.intent_id, v2.updated_at, v2.depth, v2.markers FROM vtxo v2 JOIN intent i2 ON i2.id = v2.intent_id WHERE i2.round_id = r.id ) as intent_with_inputs_amount ) AS total_forfeit_amount, ( @@ -1135,7 +1135,7 @@ func (q *Queries) SelectRoundVtxoTree(ctx context.Context, txid string) ([]Tx, e } const selectRoundVtxoTreeLeaves = `-- name: SelectRoundVtxoTreeLeaves :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE commitment_txid = ?1 AND preconfirmed = false +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE commitment_txid = ?1 AND preconfirmed = false ` type SelectRoundVtxoTreeLeavesRow struct { @@ -1169,7 +1169,7 @@ func (q *Queries) SelectRoundVtxoTreeLeaves(ctx context.Context, commitmentTxid &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -1190,7 +1190,7 @@ SELECT round.id, round.starting_timestamp, round.ending_timestamp, round.ended, round_intents_vw.id, round_intents_vw.round_id, round_intents_vw.proof, round_intents_vw.message, round_txs_vw.txid, round_txs_vw.tx, round_txs_vw.round_id, round_txs_vw.type, round_txs_vw.position, round_txs_vw.children, intent_with_receivers_vw.intent_id, intent_with_receivers_vw.pubkey, intent_with_receivers_vw.onchain_address, intent_with_receivers_vw.amount, intent_with_receivers_vw.id, intent_with_receivers_vw.round_id, intent_with_receivers_vw.proof, intent_with_receivers_vw.message, - intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.marker_id, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message + intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.markers, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message FROM round LEFT OUTER JOIN round_intents_vw ON round.id=round_intents_vw.round_id LEFT OUTER JOIN round_txs_vw ON round.id=round_txs_vw.round_id @@ -1263,7 +1263,7 @@ func (q *Queries) SelectRoundWithId(ctx context.Context, id string) ([]SelectRou &i.IntentWithInputsVw.IntentID, &i.IntentWithInputsVw.UpdatedAt, &i.IntentWithInputsVw.Depth, - &i.IntentWithInputsVw.MarkerID, + &i.IntentWithInputsVw.Markers, &i.IntentWithInputsVw.Commitments, &i.IntentWithInputsVw.ID, &i.IntentWithInputsVw.RoundID, @@ -1288,7 +1288,7 @@ SELECT round.id, round.starting_timestamp, round.ending_timestamp, round.ended, round_intents_vw.id, round_intents_vw.round_id, round_intents_vw.proof, round_intents_vw.message, round_txs_vw.txid, round_txs_vw.tx, round_txs_vw.round_id, round_txs_vw.type, round_txs_vw.position, round_txs_vw.children, intent_with_receivers_vw.intent_id, intent_with_receivers_vw.pubkey, intent_with_receivers_vw.onchain_address, intent_with_receivers_vw.amount, intent_with_receivers_vw.id, intent_with_receivers_vw.round_id, intent_with_receivers_vw.proof, intent_with_receivers_vw.message, - intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.marker_id, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message + intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.markers, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message FROM round LEFT OUTER JOIN round_intents_vw ON round.id=round_intents_vw.round_id LEFT OUTER JOIN round_txs_vw ON round.id=round_txs_vw.round_id @@ -1363,7 +1363,7 @@ func (q *Queries) SelectRoundWithTxid(ctx context.Context, txid string) ([]Selec &i.IntentWithInputsVw.IntentID, &i.IntentWithInputsVw.UpdatedAt, &i.IntentWithInputsVw.Depth, - &i.IntentWithInputsVw.MarkerID, + &i.IntentWithInputsVw.Markers, &i.IntentWithInputsVw.Commitments, &i.IntentWithInputsVw.ID, &i.IntentWithInputsVw.RoundID, @@ -1453,7 +1453,7 @@ func (q *Queries) SelectSweepableRounds(ctx context.Context) ([]string, error) { } const selectSweepableUnrolledVtxos = `-- name: SelectSweepableUnrolledVtxos :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE spent = true AND unrolled = true AND swept = false AND (COALESCE(settled_by, '') = '') +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE spent = true AND unrolled = true AND swept = false AND (COALESCE(settled_by, '') = '') ` type SelectSweepableUnrolledVtxosRow struct { @@ -1487,7 +1487,7 @@ func (q *Queries) SelectSweepableUnrolledVtxos(ctx context.Context) ([]SelectSwe &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -1684,7 +1684,7 @@ func (q *Queries) SelectTxs(ctx context.Context, arg SelectTxsParams) ([]SelectT } const selectVtxo = `-- name: SelectVtxo :one -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE txid = ?1 AND vout = ?2 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE txid = ?1 AND vout = ?2 ` type SelectVtxoParams struct { @@ -1717,35 +1717,26 @@ func (q *Queries) SelectVtxo(ctx context.Context, arg SelectVtxoParams) (SelectV &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ) return i, err } const selectVtxoChainByMarker = `-- name: SelectVtxoChainByMarker :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw -WHERE marker_id IN (/*SLICE:marker_ids*/?) -ORDER BY depth DESC +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw +WHERE markers LIKE '%"' || ?1 || '"%' +ORDER BY vtxo_vw.depth DESC ` type SelectVtxoChainByMarkerRow struct { VtxoVw VtxoVw } -// Get VTXOs that share the same marker or have markers in the parent chain -func (q *Queries) SelectVtxoChainByMarker(ctx context.Context, markerIds []sql.NullString) ([]SelectVtxoChainByMarkerRow, error) { - query := selectVtxoChainByMarker - var queryParams []interface{} - if len(markerIds) > 0 { - for _, v := range markerIds { - queryParams = append(queryParams, v) - } - query = strings.Replace(query, "/*SLICE:marker_ids*/?", strings.Repeat(",?", len(markerIds))[1:], 1) - } else { - query = strings.Replace(query, "/*SLICE:marker_ids*/?", "NULL", 1) - } - rows, err := q.db.QueryContext(ctx, query, queryParams...) +// Get VTXOs whose markers array contains the given marker_id +// For multiple markers, call this multiple times and deduplicate in Go +func (q *Queries) SelectVtxoChainByMarker(ctx context.Context, markerID sql.NullString) ([]SelectVtxoChainByMarkerRow, error) { + rows, err := q.db.QueryContext(ctx, selectVtxoChainByMarker, markerID) if err != nil { return nil, err } @@ -1771,7 +1762,7 @@ func (q *Queries) SelectVtxoChainByMarker(ctx context.Context, markerIds []sql.N &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -1824,7 +1815,7 @@ func (q *Queries) SelectVtxoPubKeysByCommitmentTxid(ctx context.Context, arg Sel } const selectVtxosByArkTxid = `-- name: SelectVtxosByArkTxid :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE txid = ?1 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE txid = ?1 ` type SelectVtxosByArkTxidRow struct { @@ -1859,7 +1850,7 @@ func (q *Queries) SelectVtxosByArkTxid(ctx context.Context, arkTxid string) ([]S &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -1877,7 +1868,7 @@ func (q *Queries) SelectVtxosByArkTxid(ctx context.Context, arkTxid string) ([]S const selectVtxosByDepthRange = `-- name: SelectVtxosByDepthRange :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE depth >= ?1 AND depth <= ?2 ORDER BY depth DESC ` @@ -1920,7 +1911,7 @@ func (q *Queries) SelectVtxosByDepthRange(ctx context.Context, arg SelectVtxosBy &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -1937,13 +1928,14 @@ func (q *Queries) SelectVtxosByDepthRange(ctx context.Context, arg SelectVtxosBy } const selectVtxosByMarkerId = `-- name: SelectVtxosByMarkerId :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE marker_id = ?1 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE markers LIKE '%"' || ?1 || '"%' ` type SelectVtxosByMarkerIdRow struct { VtxoVw VtxoVw } +// Find VTXOs whose markers JSON array contains the given marker_id func (q *Queries) SelectVtxosByMarkerId(ctx context.Context, markerID sql.NullString) ([]SelectVtxosByMarkerIdRow, error) { rows, err := q.db.QueryContext(ctx, selectVtxosByMarkerId, markerID) if err != nil { @@ -1971,7 +1963,7 @@ func (q *Queries) SelectVtxosByMarkerId(ctx context.Context, markerID sql.NullSt &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -2048,7 +2040,7 @@ func (q *Queries) SelectVtxosOutpointsByArkTxidRecursive(ctx context.Context, tx } const selectVtxosWithPubkeys = `-- name: SelectVtxosWithPubkeys :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.marker_id, vtxo_vw.commitments FROM vtxo_vw WHERE pubkey IN (/*SLICE:pubkeys*/?) +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE pubkey IN (/*SLICE:pubkeys*/?) AND updated_at >= ?2 AND (CAST(?3 AS INTEGER) = 0 OR updated_at <= CAST(?3 AS INTEGER)) ` @@ -2102,7 +2094,7 @@ func (q *Queries) SelectVtxosWithPubkeys(ctx context.Context, arg SelectVtxosWit &i.VtxoVw.IntentID, &i.VtxoVw.UpdatedAt, &i.VtxoVw.Depth, - &i.VtxoVw.MarkerID, + &i.VtxoVw.Markers, &i.VtxoVw.Commitments, ); err != nil { return nil, err @@ -2120,9 +2112,10 @@ func (q *Queries) SelectVtxosWithPubkeys(ctx context.Context, arg SelectVtxosWit const sweepVtxosByMarkerId = `-- name: SweepVtxosByMarkerId :execrows UPDATE vtxo SET swept = true, updated_at = (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)) -WHERE marker_id = ?1 AND swept = false +WHERE markers LIKE '%"' || ?1 || '"%' AND swept = false ` +// Sweep VTXOs whose markers JSON array contains the given marker_id func (q *Queries) SweepVtxosByMarkerId(ctx context.Context, markerID sql.NullString) (int64, error) { result, err := q.db.ExecContext(ctx, sweepVtxosByMarkerId, markerID) if err != nil { @@ -2170,18 +2163,18 @@ func (q *Queries) UpdateVtxoIntentId(ctx context.Context, arg UpdateVtxoIntentId return err } -const updateVtxoMarkerId = `-- name: UpdateVtxoMarkerId :exec -UPDATE vtxo SET marker_id = ?1 WHERE txid = ?2 AND vout = ?3 +const updateVtxoMarkers = `-- name: UpdateVtxoMarkers :exec +UPDATE vtxo SET markers = ?1 WHERE txid = ?2 AND vout = ?3 ` -type UpdateVtxoMarkerIdParams struct { - MarkerID sql.NullString - Txid string - Vout int64 +type UpdateVtxoMarkersParams struct { + Markers sql.NullString + Txid string + Vout int64 } -func (q *Queries) UpdateVtxoMarkerId(ctx context.Context, arg UpdateVtxoMarkerIdParams) error { - _, err := q.db.ExecContext(ctx, updateVtxoMarkerId, arg.MarkerID, arg.Txid, arg.Vout) +func (q *Queries) UpdateVtxoMarkers(ctx context.Context, arg UpdateVtxoMarkersParams) error { + _, err := q.db.ExecContext(ctx, updateVtxoMarkers, arg.Markers, arg.Txid, arg.Vout) return err } diff --git a/internal/infrastructure/db/sqlite/sqlc/query.sql b/internal/infrastructure/db/sqlite/sqlc/query.sql index 88bd31d87..a59039409 100644 --- a/internal/infrastructure/db/sqlite/sqlc/query.sql +++ b/internal/infrastructure/db/sqlite/sqlc/query.sql @@ -479,15 +479,17 @@ WITH RECURSIVE descendant_markers(id) AS ( SELECT descendant_markers.id AS marker_id FROM descendant_markers WHERE descendant_markers.id NOT IN (SELECT sm.marker_id FROM swept_marker sm); --- name: UpdateVtxoMarkerId :exec -UPDATE vtxo SET marker_id = @marker_id WHERE txid = @txid AND vout = @vout; +-- name: UpdateVtxoMarkers :exec +UPDATE vtxo SET markers = @markers WHERE txid = @txid AND vout = @vout; -- name: SelectVtxosByMarkerId :many -SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE marker_id = @marker_id; +-- Find VTXOs whose markers JSON array contains the given marker_id +SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE markers LIKE '%"' || @marker_id || '"%'; -- name: SweepVtxosByMarkerId :execrows +-- Sweep VTXOs whose markers JSON array contains the given marker_id UPDATE vtxo SET swept = true, updated_at = (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)) -WHERE marker_id = @marker_id AND swept = false; +WHERE markers LIKE '%"' || @marker_id || '"%' AND swept = false; -- Chain traversal queries for GetVtxoChain optimization @@ -502,7 +504,8 @@ ORDER BY depth DESC; SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE txid = @ark_txid; -- name: SelectVtxoChainByMarker :many --- Get VTXOs that share the same marker or have markers in the parent chain +-- Get VTXOs whose markers array contains the given marker_id +-- For multiple markers, call this multiple times and deduplicate in Go SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw -WHERE marker_id IN (sqlc.slice('marker_ids')) -ORDER BY depth DESC; \ No newline at end of file +WHERE markers LIKE '%"' || @marker_id || '"%' +ORDER BY vtxo_vw.depth DESC; \ No newline at end of file diff --git a/internal/infrastructure/db/sqlite/vtxo_repo.go b/internal/infrastructure/db/sqlite/vtxo_repo.go index 94ef37d7c..49ea2175d 100644 --- a/internal/infrastructure/db/sqlite/vtxo_repo.go +++ b/internal/infrastructure/db/sqlite/vtxo_repo.go @@ -3,6 +3,7 @@ package sqlitedb import ( "context" "database/sql" + "encoding/json" "errors" "fmt" "sort" @@ -538,10 +539,22 @@ func rowToVtxo(row queries.VtxoVw) domain.Vtxo { ExpiresAt: row.ExpiresAt, CreatedAt: row.CreatedAt, Depth: uint32(row.Depth), - MarkerID: row.MarkerID.String, + MarkerIDs: parseMarkersJSONFromVtxo(row.Markers.String), } } +// parseMarkersJSONFromVtxo parses a JSON array string into a slice of strings for vtxo repo +func parseMarkersJSONFromVtxo(markersJSON string) []string { + if markersJSON == "" { + return nil + } + var markerIDs []string + if err := json.Unmarshal([]byte(markersJSON), &markerIDs); err != nil { + return nil + } + return markerIDs +} + func readRows(rows []queries.VtxoVw) ([]domain.Vtxo, error) { vtxos := make([]domain.Vtxo, 0, len(rows)) for _, vtxo := range rows { From 5c4ebc6f4c3c483041c27cf9f295e4f769f3caa7 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Wed, 11 Feb 2026 21:27:29 -0500 Subject: [PATCH 06/54] remove serivce.go markerStore nil checks, rearrange sql statements, lint --- ...0260210100000_add_depth_and_markers.up.sql | 11 +-- internal/infrastructure/db/service.go | 96 +++++++------------ .../infrastructure/db/sqlite/marker_repo.go | 5 +- 3 files changed, 45 insertions(+), 67 deletions(-) diff --git a/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql index 4503bc723..6c77dc17e 100644 --- a/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql +++ b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql @@ -1,5 +1,8 @@ --- Add depth column -ALTER TABLE vtxo ADD COLUMN IF NOT EXISTS depth INTEGER NOT NULL DEFAULT 0; +-- Add depth and markers columns to vtxo +ALTER TABLE vtxo + ADD COLUMN IF NOT EXISTS depth INTEGER NOT NULL DEFAULT 0, + ADD COLUMN IF NOT EXISTS markers JSONB; +CREATE INDEX IF NOT EXISTS idx_vtxo_markers ON vtxo USING GIN (markers); -- Create marker table CREATE TABLE IF NOT EXISTS marker ( @@ -15,10 +18,6 @@ CREATE TABLE IF NOT EXISTS swept_marker ( swept_at BIGINT NOT NULL ); --- Add markers column (JSONB array) -ALTER TABLE vtxo ADD COLUMN IF NOT EXISTS markers JSONB; -CREATE INDEX IF NOT EXISTS idx_vtxo_markers ON vtxo USING GIN (markers); - -- Recreate views to include the new columns DROP VIEW IF EXISTS intent_with_inputs_vw; DROP VIEW IF EXISTS vtxo_vw; diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index 02af6ce8b..a7804898f 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -133,8 +133,10 @@ func NewService(config ServiceConfig, txDecoder ports.TxDecoder) (ports.RepoMana if !ok { return nil, fmt.Errorf("invalid data store type: %s", config.DataStoreType) } - markerStoreFactory := markerStoreTypes[config.DataStoreType] - + markerStoreFactory, ok := markerStoreTypes[config.DataStoreType] + if !ok { + return nil, fmt.Errorf("invalid data store type: %s", config.DataStoreType) + } var eventStore domain.EventRepository var roundStore domain.RoundRepository var vtxoStore domain.VtxoRepository @@ -205,19 +207,11 @@ func NewService(config ServiceConfig, txDecoder ports.TxDecoder) (ports.RepoMana if err != nil { return nil, fmt.Errorf("failed to create intent fees store: %w", err) } - if markerStoreFactory != nil { - // For badger, pass the vtxo store to marker repo to share the same database - badgerVtxoRepo, ok := vtxoStore.(*badgerdb.VtxoRepository) - if ok { - markerConfig := append(config.DataStoreConfig, badgerVtxoRepo.Store()) - markerStore, err = markerStoreFactory(markerConfig...) - } else { - markerStore, err = markerStoreFactory(config.DataStoreConfig...) - } - if err != nil { - return nil, fmt.Errorf("failed to create marker store: %w", err) - } + markerStore, err = markerStoreFactory(config.DataStoreConfig...) + if err != nil { + return nil, fmt.Errorf("failed to create marker store: %w", err) } + case "postgres": if len(config.DataStoreConfig) != 2 { return nil, fmt.Errorf("invalid data store config for postgres") @@ -289,12 +283,11 @@ func NewService(config ServiceConfig, txDecoder ports.TxDecoder) (ports.RepoMana if err != nil { return nil, fmt.Errorf("failed to create intent fees store: %w", err) } - if markerStoreFactory != nil { - markerStore, err = markerStoreFactory(db) - if err != nil { - return nil, fmt.Errorf("failed to create marker store: %w", err) - } + markerStore, err = markerStoreFactory(db) + if err != nil { + return nil, fmt.Errorf("failed to create marker store: %w", err) } + case "sqlite": if len(config.DataStoreConfig) != 1 { return nil, fmt.Errorf("invalid data store config") @@ -359,11 +352,9 @@ func NewService(config ServiceConfig, txDecoder ports.TxDecoder) (ports.RepoMana if err != nil { return nil, fmt.Errorf("failed to create intent fees store: %w", err) } - if markerStoreFactory != nil { - markerStore, err = markerStoreFactory(db) - if err != nil { - return nil, fmt.Errorf("failed to create marker store: %w", err) - } + markerStore, err = markerStoreFactory(db) + if err != nil { + return nil, fmt.Errorf("failed to create marker store: %w", err) } } @@ -426,9 +417,7 @@ func (s *service) Close() { s.eventStore.Close() s.roundStore.Close() s.vtxoStore.Close() - if s.markerStore != nil { - s.markerStore.Close() - } + s.markerStore.Close() s.scheduledSessionStore.Close() s.offchainTxStore.Close() s.convictionStore.Close() @@ -455,20 +444,10 @@ func (s *service) updateProjectionsAfterRoundEvents(events []domain.Event) { event := lastEvent.(domain.BatchSwept) allSweptVtxos := append(event.LeafVtxos, event.PreconfirmedVtxos...) - // Try marker-based sweeping first if marker store is available - if s.markerStore != nil { - sweptCount := s.sweepVtxosWithMarkers(ctx, allSweptVtxos) - if sweptCount > 0 { - log.Debugf("swept %d vtxos using marker-based sweeping", sweptCount) - } - } else { - // Fall back to individual VTXO sweeping - sweptCount, err := repo.SweepVtxos(ctx, allSweptVtxos) - if err != nil { - log.WithError(err).Warn("failed to sweep vtxos") - } else { - log.Debugf("swept %d vtxos", sweptCount) - } + // marker-based sweeping + sweptCount := s.sweepVtxosWithMarkers(ctx, allSweptVtxos) + if sweptCount > 0 { + log.Debugf("swept %d vtxos using marker-based sweeping", sweptCount) } if event.FullySwept { @@ -506,26 +485,23 @@ func (s *service) updateProjectionsAfterRoundEvents(events []domain.Event) { } // Create root markers for batch VTXOs (depth 0 is always at marker boundary) - if s.markerStore != nil { - for _, vtxo := range newVtxos { - // Each batch VTXO at depth 0 gets its own root marker - markerID := vtxo.Outpoint.String() - marker := domain.Marker{ - ID: markerID, - Depth: 0, - ParentMarkerIDs: nil, // Root markers have no parents - } - if err := s.markerStore.AddMarker(ctx, marker); err != nil { - log.WithError(err). - Warnf("failed to create root marker for vtxo %s", vtxo.Outpoint.String()) - continue - } - if err := s.markerStore.UpdateVtxoMarkers(ctx, vtxo.Outpoint, []string{markerID}); err != nil { - log.WithError(err). - Warnf("failed to update markers for vtxo %s", vtxo.Outpoint.String()) - } + for _, vtxo := range newVtxos { + // Each batch VTXO at depth 0 gets its own root marker + markerID := vtxo.Outpoint.String() + marker := domain.Marker{ + ID: markerID, + Depth: 0, + ParentMarkerIDs: nil, // Root markers have no parents + } + if err := s.markerStore.AddMarker(ctx, marker); err != nil { + log.WithError(err). + Warnf("failed to create root marker for vtxo %s", vtxo.Outpoint.String()) + continue + } + if err := s.markerStore.UpdateVtxoMarkers(ctx, vtxo.Outpoint, []string{markerID}); err != nil { + log.WithError(err). + Warnf("failed to update markers for vtxo %s", vtxo.Outpoint.String()) } - log.Debugf("created %d root markers for batch vtxos", len(newVtxos)) } } } diff --git a/internal/infrastructure/db/sqlite/marker_repo.go b/internal/infrastructure/db/sqlite/marker_repo.go index ee7c7cd63..d7fe13e54 100644 --- a/internal/infrastructure/db/sqlite/marker_repo.go +++ b/internal/infrastructure/db/sqlite/marker_repo.go @@ -287,7 +287,10 @@ func (m *markerRepository) GetVtxoChainByMarkers( vtxos := make([]domain.Vtxo, 0) for _, markerID := range markerIDs { - rows, err := m.querier.SelectVtxoChainByMarker(ctx, sql.NullString{String: markerID, Valid: true}) + rows, err := m.querier.SelectVtxoChainByMarker( + ctx, + sql.NullString{String: markerID, Valid: true}, + ) if err != nil { return nil, err } From 7f613f125745f8107da6118b1165cc4ba4c57335 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Wed, 11 Feb 2026 22:46:19 -0500 Subject: [PATCH 07/54] swept column from vtxo table removed --- internal/core/application/sweeper.go | 36 ++++- internal/core/domain/marker_repo.go | 8 +- internal/core/domain/vtxo_repo.go | 1 - .../infrastructure/db/badger/marker_repo.go | 68 +++++++++ .../infrastructure/db/badger/vtxo_repo.go | 28 +--- .../infrastructure/db/postgres/marker_repo.go | 87 ++++++++++- ...0260210100000_add_depth_and_markers.up.sql | 54 +++++++ .../db/postgres/sqlc/queries/models.go | 5 +- .../db/postgres/sqlc/queries/query.sql.go | 138 ++++++++--------- .../infrastructure/db/postgres/sqlc/query.sql | 43 +++--- .../infrastructure/db/postgres/vtxo_repo.go | 30 ---- internal/infrastructure/db/service.go | 69 +++++---- internal/infrastructure/db/service_test.go | 123 +++++++++------ .../infrastructure/db/sqlite/marker_repo.go | 99 ++++++++++++- ...0260210000000_add_depth_and_markers.up.sql | 91 ++++++++++++ .../infrastructure/db/sqlite/round_repo.go | 2 +- .../db/sqlite/sqlc/queries/models.go | 5 +- .../db/sqlite/sqlc/queries/query.sql.go | 140 ++++++++---------- .../infrastructure/db/sqlite/sqlc/query.sql | 43 +++--- .../infrastructure/db/sqlite/vtxo_repo.go | 32 +--- 20 files changed, 730 insertions(+), 372 deletions(-) diff --git a/internal/core/application/sweeper.go b/internal/core/application/sweeper.go index 299cd3625..66a894ac1 100644 --- a/internal/core/application/sweeper.go +++ b/internal/core/application/sweeper.go @@ -755,9 +755,39 @@ func (s *sweeper) createCheckpointSweepTask( return err } - _, err = s.repoManager.Vtxos().SweepVtxos(ctx, childrenVtxos) - log.Debugf("swept %d vtxos", len(childrenVtxos)) - return err + // Get the VTXOs to find their markers + vtxos, err := s.repoManager.Vtxos().GetVtxos(ctx, childrenVtxos) + if err != nil { + return err + } + + // Sweep each VTXO by marking its markers as swept + sweptAt := time.Now().Unix() + markerStore := s.repoManager.Markers() + sweptCount := 0 + for _, v := range vtxos { + if len(v.MarkerIDs) > 0 { + // Sweep via first marker + if err := markerStore.SweepMarker(ctx, v.MarkerIDs[0], sweptAt); err != nil { + log.WithError(err).Warnf("failed to sweep marker %s", v.MarkerIDs[0]) + continue + } + if _, err := markerStore.SweepVtxosByMarker(ctx, v.MarkerIDs[0]); err != nil { + log.WithError(err). + Warnf("failed to process sweep for marker %s", v.MarkerIDs[0]) + continue + } + } else { + // Create a dust marker for vtxos without markers + if err := markerStore.MarkDustVtxoSwept(ctx, v.Outpoint, sweptAt); err != nil { + log.WithError(err).Warnf("failed to mark vtxo %s as swept", v.Outpoint.String()) + continue + } + } + sweptCount++ + } + log.Debugf("swept %d vtxos", sweptCount) + return nil } } diff --git a/internal/core/domain/marker_repo.go b/internal/core/domain/marker_repo.go index c8426c322..adeda0a3f 100644 --- a/internal/core/domain/marker_repo.go +++ b/internal/core/domain/marker_repo.go @@ -28,10 +28,14 @@ type MarkerRepository interface { UpdateVtxoMarkers(ctx context.Context, outpoint Outpoint, markerIDs []string) error // GetVtxosByMarker retrieves all VTXOs associated with a marker GetVtxosByMarker(ctx context.Context, markerID string) ([]Vtxo, error) - // SweepVtxosByMarker marks all VTXOs with the given marker_id as swept - // Returns the number of VTXOs that were swept (not already swept) + // SweepVtxosByMarker inserts the marker into swept_marker table + // Returns the number of VTXOs that will now be considered swept SweepVtxosByMarker(ctx context.Context, markerID string) (int64, error) + // MarkDustVtxoSwept creates a unique dust marker for a vtxo and marks it as swept + // Used for dust vtxos that need to be marked swept immediately on creation + MarkDustVtxoSwept(ctx context.Context, outpoint Outpoint, sweptAt int64) error + // Chain traversal methods for GetVtxoChain optimization // GetVtxosByDepthRange retrieves VTXOs within a depth range GetVtxosByDepthRange(ctx context.Context, minDepth, maxDepth uint32) ([]Vtxo, error) diff --git a/internal/core/domain/vtxo_repo.go b/internal/core/domain/vtxo_repo.go index 793ff2d74..61beb8cca 100644 --- a/internal/core/domain/vtxo_repo.go +++ b/internal/core/domain/vtxo_repo.go @@ -7,7 +7,6 @@ type VtxoRepository interface { SettleVtxos(ctx context.Context, spentVtxos map[Outpoint]string, commitmentTxid string) error SpendVtxos(ctx context.Context, spentVtxos map[Outpoint]string, arkTxid string) error UnrollVtxos(ctx context.Context, outpoints []Outpoint) error - SweepVtxos(ctx context.Context, outpoints []Outpoint) (int, error) GetVtxos(ctx context.Context, outpoints []Outpoint) ([]Vtxo, error) GetAllNonUnrolledVtxos(ctx context.Context, pubkey string) ([]Vtxo, []Vtxo, error) GetAllSweepableUnrolledVtxos(ctx context.Context) ([]Vtxo, error) diff --git a/internal/infrastructure/db/badger/marker_repo.go b/internal/infrastructure/db/badger/marker_repo.go index 828b0f0a5..830969c6e 100644 --- a/internal/infrastructure/db/badger/marker_repo.go +++ b/internal/infrastructure/db/badger/marker_repo.go @@ -423,6 +423,10 @@ func (r *markerRepository) GetVtxosByMarker( } func (r *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID string) (int64, error) { + // For badger, we need to: + // 1. Mark the marker as swept + // 2. Update vtxo.Swept field for all VTXOs with this marker (for query compatibility) + // Find all VTXOs whose MarkerIDs contains markerID and are not swept var allDtos []vtxoDTO err := r.vtxoStore.Find(&allDtos, badgerhold.Where("Swept").Eq(false)) @@ -444,6 +448,7 @@ func (r *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID stri continue } + // Update the vtxo's Swept field dto.Swept = true dto.UpdatedAt = time.Now().UnixMilli() @@ -464,9 +469,72 @@ func (r *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID stri } count++ } + + // Also insert the marker into swept_marker for consistency + if err := r.SweepMarker(ctx, markerID, time.Now().Unix()); err != nil { + // Non-fatal - the vtxos are already marked as swept + _ = err + } + return count, nil } +func (r *markerRepository) MarkDustVtxoSwept( + ctx context.Context, + outpoint domain.Outpoint, + sweptAt int64, +) error { + // Create a unique dust marker for this vtxo + dustMarkerID := outpoint.String() + ":dust" + + // Get the vtxo to find its depth and current markers + var dto vtxoDTO + err := r.vtxoStore.Get(outpoint.String(), &dto) + if err != nil { + if err == badgerhold.ErrNotFound { + return fmt.Errorf("vtxo not found: %s", outpoint.String()) + } + return fmt.Errorf("failed to get vtxo: %w", err) + } + + // Create the dust marker + if err := r.AddMarker(ctx, domain.Marker{ + ID: dustMarkerID, + Depth: dto.Depth, + ParentMarkerIDs: dto.MarkerIDs, + }); err != nil { + return fmt.Errorf("failed to create dust marker: %w", err) + } + + // Insert into swept_marker + if err := r.SweepMarker(ctx, dustMarkerID, sweptAt); err != nil { + return fmt.Errorf("failed to insert swept marker: %w", err) + } + + // Update the vtxo's markers to include the dust marker and mark as swept + dto.MarkerIDs = append(dto.MarkerIDs, dustMarkerID) + dto.Swept = true + dto.UpdatedAt = time.Now().UnixMilli() + + err = r.vtxoStore.Update(outpoint.String(), dto) + if err != nil { + if errors.Is(err, badger.ErrConflict) { + for attempts := 1; attempts <= maxRetries; attempts++ { + time.Sleep(100 * time.Millisecond) + err = r.vtxoStore.Update(outpoint.String(), dto) + if err == nil { + break + } + } + } + if err != nil { + return fmt.Errorf("failed to update vtxo: %w", err) + } + } + + return nil +} + func (r *markerRepository) GetVtxosByDepthRange( ctx context.Context, minDepth, maxDepth uint32, diff --git a/internal/infrastructure/db/badger/vtxo_repo.go b/internal/infrastructure/db/badger/vtxo_repo.go index 69c6d81f1..6ca13ad80 100644 --- a/internal/infrastructure/db/badger/vtxo_repo.go +++ b/internal/infrastructure/db/badger/vtxo_repo.go @@ -20,6 +20,11 @@ type VtxoRepository struct { store *badgerhold.Store } +// GetStore returns the underlying badgerhold store for use by marker repository +func (r *VtxoRepository) GetStore() *badgerhold.Store { + return r.store +} + type vtxoDTO struct { domain.Vtxo UpdatedAt int64 @@ -162,29 +167,6 @@ func (r *VtxoRepository) GetAllVtxos(ctx context.Context) ([]domain.Vtxo, error) return r.findVtxos(ctx, &badgerhold.Query{}) } -func (r *VtxoRepository) SweepVtxos( - ctx context.Context, outpoints []domain.Outpoint, -) (int, error) { - sweptCount := 0 - for _, outpoint := range outpoints { - vtxo, err := r.getVtxo(ctx, outpoint) - if err != nil { - return -1, err - } - if vtxo.Swept { - continue // Skip already swept vtxos - } - - // Mark as swept - vtxo.Swept = true - if err := r.updateVtxo(ctx, vtxo); err != nil { - return -1, err - } - sweptCount++ - } - return sweptCount, nil -} - func (r *VtxoRepository) UpdateVtxosExpiration( ctx context.Context, vtxos []domain.Outpoint, expiresAt int64, ) error { diff --git a/internal/infrastructure/db/postgres/marker_repo.go b/internal/infrastructure/db/postgres/marker_repo.go index e0a2eccb9..900408cda 100644 --- a/internal/infrastructure/db/postgres/marker_repo.go +++ b/internal/infrastructure/db/postgres/marker_repo.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/json" "fmt" + "time" "github.com/arkade-os/arkd/internal/core/domain" "github.com/arkade-os/arkd/internal/infrastructure/db/postgres/sqlc/queries" @@ -232,7 +233,91 @@ func (m *markerRepository) GetVtxosByMarker( } func (m *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID string) (int64, error) { - return m.querier.SweepVtxosByMarkerId(ctx, markerID) + // First check if the marker exists (foreign key constraint on swept_marker) + marker, err := m.GetMarker(ctx, markerID) + if err != nil { + return 0, fmt.Errorf("failed to check marker existence: %w", err) + } + if marker == nil { + return 0, nil // Marker doesn't exist, nothing to sweep + } + + // Count unswept VTXOs with this marker before inserting to swept_marker + count, err := m.querier.CountUnsweptVtxosByMarkerId(ctx, markerID) + if err != nil { + return 0, fmt.Errorf("failed to count unswept vtxos: %w", err) + } + + // Insert the marker into swept_marker (sweep state is computed via view) + if err := m.querier.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ + MarkerID: markerID, + SweptAt: time.Now().Unix(), + }); err != nil { + return 0, fmt.Errorf("failed to insert swept marker: %w", err) + } + + return count, nil +} + +func (m *markerRepository) MarkDustVtxoSwept( + ctx context.Context, + outpoint domain.Outpoint, + sweptAt int64, +) error { + // Create a unique dust marker for this vtxo + dustMarkerID := outpoint.String() + ":dust" + + // First, get the vtxo to find its depth and current markers + vtxoRow, err := m.querier.SelectVtxo(ctx, queries.SelectVtxoParams{ + Txid: outpoint.Txid, + Vout: int32(outpoint.VOut), + }) + if err != nil { + return fmt.Errorf("failed to get vtxo: %w", err) + } + + // Create the dust marker + parentMarkers := parseMarkersJSONB(vtxoRow.VtxoVw.Markers) + parentMarkersJSON, err := json.Marshal(parentMarkers) + if err != nil { + return fmt.Errorf("failed to marshal parent markers: %w", err) + } + + if err := m.querier.UpsertMarker(ctx, queries.UpsertMarkerParams{ + ID: dustMarkerID, + Depth: vtxoRow.VtxoVw.Depth, + ParentMarkers: pqtype.NullRawMessage{ + RawMessage: parentMarkersJSON, + Valid: true, + }, + }); err != nil { + return fmt.Errorf("failed to create dust marker: %w", err) + } + + // Insert into swept_marker + if err := m.querier.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ + MarkerID: dustMarkerID, + SweptAt: sweptAt, + }); err != nil { + return fmt.Errorf("failed to insert swept marker: %w", err) + } + + // Update the vtxo's markers to include the dust marker + newMarkers := append(parentMarkers, dustMarkerID) + newMarkersJSON, err := json.Marshal(newMarkers) + if err != nil { + return fmt.Errorf("failed to marshal new markers: %w", err) + } + + if err := m.querier.UpdateVtxoMarkers(ctx, queries.UpdateVtxoMarkersParams{ + Markers: newMarkersJSON, + Txid: outpoint.Txid, + Vout: int32(outpoint.VOut), + }); err != nil { + return fmt.Errorf("failed to update vtxo markers: %w", err) + } + + return nil } func (m *markerRepository) GetVtxosByDepthRange( diff --git a/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql index 6c77dc17e..9ead5d349 100644 --- a/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql +++ b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql @@ -51,3 +51,57 @@ WHERE v.depth % 100 = 0; -- Assign markers array to VTXOs at boundary depths UPDATE vtxo SET markers = jsonb_build_array(txid || ':' || vout) WHERE depth % 100 = 0; + +-- Migrate existing swept VTXOs to swept_marker table before dropping column +-- For each swept VTXO, create a unique dust marker and insert into swept_marker +INSERT INTO marker (id, depth, parent_markers) +SELECT + v.txid || ':' || v.vout || ':dust', + v.depth, + COALESCE(v.markers, '[]'::jsonb) +FROM vtxo v +WHERE v.swept = true +ON CONFLICT (id) DO NOTHING; + +INSERT INTO swept_marker (marker_id, swept_at) +SELECT + v.txid || ':' || v.vout || ':dust', + EXTRACT(EPOCH FROM NOW())::BIGINT +FROM vtxo v +WHERE v.swept = true +ON CONFLICT (marker_id) DO NOTHING; + +-- Update swept VTXOs to include the dust marker in their markers array +UPDATE vtxo SET markers = COALESCE(markers, '[]'::jsonb) || jsonb_build_array(txid || ':' || vout || ':dust') +WHERE swept = true; + +-- Drop views before dropping the swept column (views depend on it via v.*) +DROP VIEW IF EXISTS intent_with_inputs_vw; +DROP VIEW IF EXISTS vtxo_vw; + +-- Drop swept column from vtxo table (swept state now computed via markers) +ALTER TABLE vtxo DROP COLUMN IF EXISTS swept; + +-- Recreate views to compute swept status dynamically + +CREATE VIEW vtxo_vw AS +SELECT v.*, + string_agg(vc.commitment_txid, ',') AS commitments, + EXISTS ( + SELECT 1 FROM swept_marker sm + WHERE v.markers @> jsonb_build_array(sm.marker_id) + ) AS swept +FROM vtxo v +LEFT JOIN vtxo_commitment_txid vc +ON v.txid = vc.vtxo_txid AND v.vout = vc.vtxo_vout +GROUP BY v.txid, v.vout; + +CREATE VIEW intent_with_inputs_vw AS +SELECT vtxo_vw.*, + intent.id, + intent.round_id, + intent.proof, + intent.message +FROM intent +LEFT OUTER JOIN vtxo_vw +ON intent.id = vtxo_vw.intent_id; diff --git a/internal/infrastructure/db/postgres/sqlc/queries/models.go b/internal/infrastructure/db/postgres/sqlc/queries/models.go index 7eeebaac9..652f1d905 100644 --- a/internal/infrastructure/db/postgres/sqlc/queries/models.go +++ b/internal/infrastructure/db/postgres/sqlc/queries/models.go @@ -58,7 +58,6 @@ type IntentWithInputsVw struct { SpentBy sql.NullString Spent sql.NullBool Unrolled sql.NullBool - Swept sql.NullBool Preconfirmed sql.NullBool SettledBy sql.NullString ArkTxid sql.NullString @@ -67,6 +66,7 @@ type IntentWithInputsVw struct { Depth sql.NullInt32 Markers pqtype.NullRawMessage Commitments []byte + Swept sql.NullBool ID sql.NullString RoundID sql.NullString Proof sql.NullString @@ -219,7 +219,6 @@ type Vtxo struct { SpentBy sql.NullString Spent bool Unrolled bool - Swept bool Preconfirmed bool SettledBy sql.NullString ArkTxid sql.NullString @@ -246,7 +245,6 @@ type VtxoVw struct { SpentBy sql.NullString Spent bool Unrolled bool - Swept bool Preconfirmed bool SettledBy sql.NullString ArkTxid sql.NullString @@ -255,4 +253,5 @@ type VtxoVw struct { Depth int32 Markers pqtype.NullRawMessage Commitments []byte + Swept bool } diff --git a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go index 5aa1fb9a0..dab24ec6c 100644 --- a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go @@ -87,6 +87,18 @@ func (q *Queries) ClearScheduledSession(ctx context.Context) error { return err } +const countUnsweptVtxosByMarkerId = `-- name: CountUnsweptVtxosByMarkerId :one +SELECT COUNT(*) FROM vtxo_vw WHERE markers @> jsonb_build_array($1::TEXT) AND swept = false +` + +// Count VTXOs whose markers JSONB array contains the given marker_id and are not swept +func (q *Queries) CountUnsweptVtxosByMarkerId(ctx context.Context, markerID string) (int64, error) { + row := q.db.QueryRowContext(ctx, countUnsweptVtxosByMarkerId, markerID) + var count int64 + err := row.Scan(&count) + return count, err +} + const getDescendantMarkerIds = `-- name: GetDescendantMarkerIds :many WITH RECURSIVE descendant_markers(id) AS ( -- Base case: the marker being swept @@ -243,7 +255,7 @@ func (q *Queries) SelectAllRoundIds(ctx context.Context) ([]string, error) { } const selectAllVtxos = `-- name: SelectAllVtxos :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw ` type SelectAllVtxosRow struct { @@ -270,7 +282,6 @@ func (q *Queries) SelectAllVtxos(ctx context.Context) ([]SelectAllVtxosRow, erro &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -279,6 +290,7 @@ func (q *Queries) SelectAllVtxos(ctx context.Context) ([]SelectAllVtxosRow, erro &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ); err != nil { return nil, err } @@ -398,13 +410,16 @@ func (q *Queries) SelectConvictionsInTimeRange(ctx context.Context, arg SelectCo } const selectExpiringLiquidityAmount = `-- name: SelectExpiringLiquidityAmount :one -SELECT COALESCE(SUM(amount), 0)::bigint AS amount -FROM vtxo -WHERE swept = false - AND spent = false - AND unrolled = false - AND expires_at > $1 - AND ($2 <= 0 OR expires_at < $2) +SELECT COALESCE(SUM(v.amount), 0)::bigint AS amount +FROM vtxo v +WHERE NOT EXISTS ( + SELECT 1 FROM swept_marker sm + WHERE v.markers @> jsonb_build_array(sm.marker_id) + ) + AND v.spent = false + AND v.unrolled = false + AND v.expires_at > $1 + AND ($2 <= 0 OR v.expires_at < $2) ` type SelectExpiringLiquidityAmountParams struct { @@ -579,7 +594,7 @@ func (q *Queries) SelectMarkersByIds(ctx context.Context, ids []string) ([]Marke } const selectNotUnrolledVtxos = `-- name: SelectNotUnrolledVtxos :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE unrolled = false ` type SelectNotUnrolledVtxosRow struct { @@ -606,7 +621,6 @@ func (q *Queries) SelectNotUnrolledVtxos(ctx context.Context) ([]SelectNotUnroll &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -615,6 +629,7 @@ func (q *Queries) SelectNotUnrolledVtxos(ctx context.Context) ([]SelectNotUnroll &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ); err != nil { return nil, err } @@ -630,7 +645,7 @@ func (q *Queries) SelectNotUnrolledVtxos(ctx context.Context) ([]SelectNotUnroll } const selectNotUnrolledVtxosWithPubkey = `-- name: SelectNotUnrolledVtxosWithPubkey :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false AND pubkey = $1 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE unrolled = false AND pubkey = $1 ` type SelectNotUnrolledVtxosWithPubkeyRow struct { @@ -657,7 +672,6 @@ func (q *Queries) SelectNotUnrolledVtxosWithPubkey(ctx context.Context, pubkey s &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -666,6 +680,7 @@ func (q *Queries) SelectNotUnrolledVtxosWithPubkey(ctx context.Context, pubkey s &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ); err != nil { return nil, err } @@ -725,7 +740,7 @@ func (q *Queries) SelectOffchainTx(ctx context.Context, txid string) ([]SelectOf } const selectPendingSpentVtxo = `-- name: SelectPendingSpentVtxo :one -SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.markers, v.commitments +SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.markers, v.commitments, v.swept FROM vtxo_vw v WHERE v.txid = $1 AND v.vout = $2 AND v.spent = TRUE AND v.unrolled = FALSE and COALESCE(v.settled_by, '') = '' @@ -753,7 +768,6 @@ func (q *Queries) SelectPendingSpentVtxo(ctx context.Context, arg SelectPendingS &i.SpentBy, &i.Spent, &i.Unrolled, - &i.Swept, &i.Preconfirmed, &i.SettledBy, &i.ArkTxid, @@ -762,12 +776,13 @@ func (q *Queries) SelectPendingSpentVtxo(ctx context.Context, arg SelectPendingS &i.Depth, &i.Markers, &i.Commitments, + &i.Swept, ) return i, err } const selectPendingSpentVtxosWithPubkeys = `-- name: SelectPendingSpentVtxosWithPubkeys :many -SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.markers, v.commitments +SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.markers, v.commitments, v.swept FROM vtxo_vw v WHERE v.spent = TRUE AND v.unrolled = FALSE and COALESCE(v.settled_by, '') = '' AND v.pubkey = ANY($1::varchar[]) @@ -804,7 +819,6 @@ func (q *Queries) SelectPendingSpentVtxosWithPubkeys(ctx context.Context, arg Se &i.SpentBy, &i.Spent, &i.Unrolled, - &i.Swept, &i.Preconfirmed, &i.SettledBy, &i.ArkTxid, @@ -813,6 +827,7 @@ func (q *Queries) SelectPendingSpentVtxosWithPubkeys(ctx context.Context, arg Se &i.Depth, &i.Markers, &i.Commitments, + &i.Swept, ); err != nil { return nil, err } @@ -828,10 +843,13 @@ func (q *Queries) SelectPendingSpentVtxosWithPubkeys(ctx context.Context, arg Se } const selectRecoverableLiquidityAmount = `-- name: SelectRecoverableLiquidityAmount :one -SELECT COALESCE(SUM(amount), 0)::bigint AS amount -FROM vtxo -WHERE swept = true - AND spent = false +SELECT COALESCE(SUM(v.amount), 0)::bigint AS amount +FROM vtxo v +WHERE EXISTS ( + SELECT 1 FROM swept_marker sm + WHERE v.markers @> jsonb_build_array(sm.marker_id) + ) + AND v.spent = false ` func (q *Queries) SelectRecoverableLiquidityAmount(ctx context.Context) (int64, error) { @@ -1111,7 +1129,7 @@ func (q *Queries) SelectRoundVtxoTree(ctx context.Context, txid string) ([]Tx, e } const selectRoundVtxoTreeLeaves = `-- name: SelectRoundVtxoTreeLeaves :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE commitment_txid = $1 AND preconfirmed = false +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE commitment_txid = $1 AND preconfirmed = false ` type SelectRoundVtxoTreeLeavesRow struct { @@ -1138,7 +1156,6 @@ func (q *Queries) SelectRoundVtxoTreeLeaves(ctx context.Context, commitmentTxid &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -1147,6 +1164,7 @@ func (q *Queries) SelectRoundVtxoTreeLeaves(ctx context.Context, commitmentTxid &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ); err != nil { return nil, err } @@ -1166,7 +1184,7 @@ SELECT round.id, round.starting_timestamp, round.ending_timestamp, round.ended, round_intents_vw.id, round_intents_vw.round_id, round_intents_vw.proof, round_intents_vw.message, round_txs_vw.txid, round_txs_vw.tx, round_txs_vw.round_id, round_txs_vw.type, round_txs_vw.position, round_txs_vw.children, intent_with_receivers_vw.intent_id, intent_with_receivers_vw.pubkey, intent_with_receivers_vw.onchain_address, intent_with_receivers_vw.amount, intent_with_receivers_vw.id, intent_with_receivers_vw.round_id, intent_with_receivers_vw.proof, intent_with_receivers_vw.message, - intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.markers, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message + intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.markers, intent_with_inputs_vw.commitments, intent_with_inputs_vw.swept, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message FROM round LEFT OUTER JOIN round_intents_vw ON round.id=round_intents_vw.round_id LEFT OUTER JOIN round_txs_vw ON round.id=round_txs_vw.round_id @@ -1232,7 +1250,6 @@ func (q *Queries) SelectRoundWithId(ctx context.Context, id string) ([]SelectRou &i.IntentWithInputsVw.SpentBy, &i.IntentWithInputsVw.Spent, &i.IntentWithInputsVw.Unrolled, - &i.IntentWithInputsVw.Swept, &i.IntentWithInputsVw.Preconfirmed, &i.IntentWithInputsVw.SettledBy, &i.IntentWithInputsVw.ArkTxid, @@ -1241,6 +1258,7 @@ func (q *Queries) SelectRoundWithId(ctx context.Context, id string) ([]SelectRou &i.IntentWithInputsVw.Depth, &i.IntentWithInputsVw.Markers, &i.IntentWithInputsVw.Commitments, + &i.IntentWithInputsVw.Swept, &i.IntentWithInputsVw.ID, &i.IntentWithInputsVw.RoundID, &i.IntentWithInputsVw.Proof, @@ -1264,7 +1282,7 @@ SELECT round.id, round.starting_timestamp, round.ending_timestamp, round.ended, round_intents_vw.id, round_intents_vw.round_id, round_intents_vw.proof, round_intents_vw.message, round_txs_vw.txid, round_txs_vw.tx, round_txs_vw.round_id, round_txs_vw.type, round_txs_vw.position, round_txs_vw.children, intent_with_receivers_vw.intent_id, intent_with_receivers_vw.pubkey, intent_with_receivers_vw.onchain_address, intent_with_receivers_vw.amount, intent_with_receivers_vw.id, intent_with_receivers_vw.round_id, intent_with_receivers_vw.proof, intent_with_receivers_vw.message, - intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.markers, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message + intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.markers, intent_with_inputs_vw.commitments, intent_with_inputs_vw.swept, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message FROM round LEFT OUTER JOIN round_intents_vw ON round.id=round_intents_vw.round_id LEFT OUTER JOIN round_txs_vw ON round.id=round_txs_vw.round_id @@ -1332,7 +1350,6 @@ func (q *Queries) SelectRoundWithTxid(ctx context.Context, txid string) ([]Selec &i.IntentWithInputsVw.SpentBy, &i.IntentWithInputsVw.Spent, &i.IntentWithInputsVw.Unrolled, - &i.IntentWithInputsVw.Swept, &i.IntentWithInputsVw.Preconfirmed, &i.IntentWithInputsVw.SettledBy, &i.IntentWithInputsVw.ArkTxid, @@ -1341,6 +1358,7 @@ func (q *Queries) SelectRoundWithTxid(ctx context.Context, txid string) ([]Selec &i.IntentWithInputsVw.Depth, &i.IntentWithInputsVw.Markers, &i.IntentWithInputsVw.Commitments, + &i.IntentWithInputsVw.Swept, &i.IntentWithInputsVw.ID, &i.IntentWithInputsVw.RoundID, &i.IntentWithInputsVw.Proof, @@ -1419,7 +1437,7 @@ func (q *Queries) SelectSweepableRounds(ctx context.Context) ([]string, error) { } const selectSweepableUnrolledVtxos = `-- name: SelectSweepableUnrolledVtxos :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE spent = true AND unrolled = true AND swept = false AND COALESCE(settled_by, '') = '' +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE spent = true AND unrolled = true AND swept = false AND COALESCE(settled_by, '') = '' ` type SelectSweepableUnrolledVtxosRow struct { @@ -1446,7 +1464,6 @@ func (q *Queries) SelectSweepableUnrolledVtxos(ctx context.Context) ([]SelectSwe &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -1455,6 +1472,7 @@ func (q *Queries) SelectSweepableUnrolledVtxos(ctx context.Context) ([]SelectSwe &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ); err != nil { return nil, err } @@ -1608,7 +1626,7 @@ func (q *Queries) SelectTxs(ctx context.Context, dollar_1 []string) ([]SelectTxs } const selectVtxo = `-- name: SelectVtxo :one -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE txid = $1 AND vout = $2 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE txid = $1 AND vout = $2 ` type SelectVtxoParams struct { @@ -1634,7 +1652,6 @@ func (q *Queries) SelectVtxo(ctx context.Context, arg SelectVtxoParams) (SelectV &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -1643,12 +1660,13 @@ func (q *Queries) SelectVtxo(ctx context.Context, arg SelectVtxoParams) (SelectV &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ) return i, err } const selectVtxoChainByMarker = `-- name: SelectVtxoChainByMarker :many -SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, markers, commitments FROM vtxo_vw +SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, markers, commitments, swept FROM vtxo_vw WHERE markers ?| $1::TEXT[] ORDER BY depth DESC ` @@ -1674,7 +1692,6 @@ func (q *Queries) SelectVtxoChainByMarker(ctx context.Context, markerIds []strin &i.SpentBy, &i.Spent, &i.Unrolled, - &i.Swept, &i.Preconfirmed, &i.SettledBy, &i.ArkTxid, @@ -1683,6 +1700,7 @@ func (q *Queries) SelectVtxoChainByMarker(ctx context.Context, markerIds []strin &i.Depth, &i.Markers, &i.Commitments, + &i.Swept, ); err != nil { return nil, err } @@ -1734,7 +1752,7 @@ func (q *Queries) SelectVtxoPubKeysByCommitmentTxid(ctx context.Context, arg Sel } const selectVtxosByArkTxid = `-- name: SelectVtxosByArkTxid :many -SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, markers, commitments FROM vtxo_vw WHERE txid = $1 +SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, markers, commitments, swept FROM vtxo_vw WHERE txid = $1 ` // Get all VTXOs created by a specific ark tx (offchain tx) @@ -1758,7 +1776,6 @@ func (q *Queries) SelectVtxosByArkTxid(ctx context.Context, arkTxid string) ([]V &i.SpentBy, &i.Spent, &i.Unrolled, - &i.Swept, &i.Preconfirmed, &i.SettledBy, &i.ArkTxid, @@ -1767,6 +1784,7 @@ func (q *Queries) SelectVtxosByArkTxid(ctx context.Context, arkTxid string) ([]V &i.Depth, &i.Markers, &i.Commitments, + &i.Swept, ); err != nil { return nil, err } @@ -1783,7 +1801,7 @@ func (q *Queries) SelectVtxosByArkTxid(ctx context.Context, arkTxid string) ([]V const selectVtxosByDepthRange = `-- name: SelectVtxosByDepthRange :many -SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, markers, commitments FROM vtxo_vw +SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, markers, commitments, swept FROM vtxo_vw WHERE depth >= $1 AND depth <= $2 ORDER BY depth DESC ` @@ -1815,7 +1833,6 @@ func (q *Queries) SelectVtxosByDepthRange(ctx context.Context, arg SelectVtxosBy &i.SpentBy, &i.Spent, &i.Unrolled, - &i.Swept, &i.Preconfirmed, &i.SettledBy, &i.ArkTxid, @@ -1824,6 +1841,7 @@ func (q *Queries) SelectVtxosByDepthRange(ctx context.Context, arg SelectVtxosBy &i.Depth, &i.Markers, &i.Commitments, + &i.Swept, ); err != nil { return nil, err } @@ -1839,7 +1857,7 @@ func (q *Queries) SelectVtxosByDepthRange(ctx context.Context, arg SelectVtxosBy } const selectVtxosByMarkerId = `-- name: SelectVtxosByMarkerId :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE markers @> jsonb_build_array($1::TEXT) +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE markers @> jsonb_build_array($1::TEXT) ` type SelectVtxosByMarkerIdRow struct { @@ -1867,7 +1885,6 @@ func (q *Queries) SelectVtxosByMarkerId(ctx context.Context, markerID string) ([ &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -1876,6 +1893,7 @@ func (q *Queries) SelectVtxosByMarkerId(ctx context.Context, markerID string) ([ &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ); err != nil { return nil, err } @@ -1952,7 +1970,7 @@ func (q *Queries) SelectVtxosOutpointsByArkTxidRecursive(ctx context.Context, tx } const selectVtxosWithPubkeys = `-- name: SelectVtxosWithPubkeys :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE vtxo_vw.pubkey = ANY($1::varchar[]) AND vtxo_vw.updated_at >= $2::bigint AND ($3::bigint = 0 OR vtxo_vw.updated_at <= $3::bigint) @@ -1988,7 +2006,6 @@ func (q *Queries) SelectVtxosWithPubkeys(ctx context.Context, arg SelectVtxosWit &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -1997,6 +2014,7 @@ func (q *Queries) SelectVtxosWithPubkeys(ctx context.Context, arg SelectVtxosWit &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ); err != nil { return nil, err } @@ -2011,20 +2029,6 @@ func (q *Queries) SelectVtxosWithPubkeys(ctx context.Context, arg SelectVtxosWit return items, nil } -const sweepVtxosByMarkerId = `-- name: SweepVtxosByMarkerId :execrows -UPDATE vtxo SET swept = true, updated_at = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT -WHERE markers @> jsonb_build_array($1::TEXT) AND swept = false -` - -// Sweep VTXOs whose markers JSONB array contains the given marker_id -func (q *Queries) SweepVtxosByMarkerId(ctx context.Context, markerID string) (int64, error) { - result, err := q.db.ExecContext(ctx, sweepVtxosByMarkerId, markerID) - if err != nil { - return 0, err - } - return result.RowsAffected() -} - const updateConvictionPardoned = `-- name: UpdateConvictionPardoned :exec UPDATE conviction SET pardoned = true WHERE id = $1 ` @@ -2123,23 +2127,6 @@ func (q *Queries) UpdateVtxoSpent(ctx context.Context, arg UpdateVtxoSpentParams return err } -const updateVtxoSweptIfNotSwept = `-- name: UpdateVtxoSweptIfNotSwept :execrows -UPDATE vtxo SET swept = true, updated_at = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT WHERE txid = $1 AND vout = $2 AND swept = false -` - -type UpdateVtxoSweptIfNotSweptParams struct { - Txid string - Vout int32 -} - -func (q *Queries) UpdateVtxoSweptIfNotSwept(ctx context.Context, arg UpdateVtxoSweptIfNotSweptParams) (int64, error) { - result, err := q.db.ExecContext(ctx, updateVtxoSweptIfNotSwept, arg.Txid, arg.Vout) - if err != nil { - return 0, err - } - return result.RowsAffected() -} - const updateVtxoUnrolled = `-- name: UpdateVtxoUnrolled :exec UPDATE vtxo SET unrolled = true, updated_at = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT WHERE txid = $1 AND vout = $2 ` @@ -2456,11 +2443,11 @@ func (q *Queries) UpsertTx(ctx context.Context, arg UpsertTxParams) error { const upsertVtxo = `-- name: UpsertVtxo :exec INSERT INTO vtxo ( txid, vout, pubkey, amount, commitment_txid, settled_by, ark_txid, - spent_by, spent, unrolled, swept, preconfirmed, expires_at, created_at, updated_at, depth + spent_by, spent, unrolled, preconfirmed, expires_at, created_at, updated_at, depth ) VALUES ( $1, $2, $3, $4, $5, $6, $7, - $8, $9, $10, $11, $12, $13, $14, (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, $15 + $8, $9, $10, $11, $12, $13, (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, $14 ) ON CONFLICT(txid, vout) DO UPDATE SET pubkey = EXCLUDED.pubkey, amount = EXCLUDED.amount, @@ -2470,7 +2457,6 @@ VALUES ( spent_by = EXCLUDED.spent_by, spent = EXCLUDED.spent, unrolled = EXCLUDED.unrolled, - swept = EXCLUDED.swept, preconfirmed = EXCLUDED.preconfirmed, expires_at = EXCLUDED.expires_at, created_at = EXCLUDED.created_at, @@ -2489,7 +2475,6 @@ type UpsertVtxoParams struct { SpentBy sql.NullString Spent bool Unrolled bool - Swept bool Preconfirmed bool ExpiresAt int64 CreatedAt int64 @@ -2508,7 +2493,6 @@ func (q *Queries) UpsertVtxo(ctx context.Context, arg UpsertVtxoParams) error { arg.SpentBy, arg.Spent, arg.Unrolled, - arg.Swept, arg.Preconfirmed, arg.ExpiresAt, arg.CreatedAt, diff --git a/internal/infrastructure/db/postgres/sqlc/query.sql b/internal/infrastructure/db/postgres/sqlc/query.sql index a959852c2..60b408365 100644 --- a/internal/infrastructure/db/postgres/sqlc/query.sql +++ b/internal/infrastructure/db/postgres/sqlc/query.sql @@ -48,11 +48,11 @@ ON CONFLICT(intent_id, pubkey, onchain_address) DO UPDATE SET -- name: UpsertVtxo :exec INSERT INTO vtxo ( txid, vout, pubkey, amount, commitment_txid, settled_by, ark_txid, - spent_by, spent, unrolled, swept, preconfirmed, expires_at, created_at, updated_at, depth + spent_by, spent, unrolled, preconfirmed, expires_at, created_at, updated_at, depth ) VALUES ( @txid, @vout, @pubkey, @amount, @commitment_txid, @settled_by, @ark_txid, - @spent_by, @spent, @unrolled, @swept, @preconfirmed, @expires_at, @created_at, (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, @depth + @spent_by, @spent, @unrolled, @preconfirmed, @expires_at, @created_at, (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, @depth ) ON CONFLICT(txid, vout) DO UPDATE SET pubkey = EXCLUDED.pubkey, amount = EXCLUDED.amount, @@ -62,7 +62,6 @@ VALUES ( spent_by = EXCLUDED.spent_by, spent = EXCLUDED.spent, unrolled = EXCLUDED.unrolled, - swept = EXCLUDED.swept, preconfirmed = EXCLUDED.preconfirmed, expires_at = EXCLUDED.expires_at, created_at = EXCLUDED.created_at, @@ -117,9 +116,6 @@ UPDATE vtxo SET expires_at = @expires_at WHERE txid = @txid AND vout = @vout; -- name: UpdateVtxoUnrolled :exec UPDATE vtxo SET unrolled = true, updated_at = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT WHERE txid = @txid AND vout = @vout; --- name: UpdateVtxoSweptIfNotSwept :execrows -UPDATE vtxo SET swept = true, updated_at = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT WHERE txid = @txid AND vout = @vout AND swept = false; - -- name: UpdateVtxoSettled :exec UPDATE vtxo SET spent = true, spent_by = @spent_by, settled_by = @settled_by, updated_at = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT WHERE txid = @txid AND vout = @vout; @@ -257,19 +253,25 @@ WHERE vtxo_vw.pubkey = ANY($1::varchar[]) AND (@before::bigint = 0 OR vtxo_vw.updated_at <= @before::bigint); -- name: SelectExpiringLiquidityAmount :one -SELECT COALESCE(SUM(amount), 0)::bigint AS amount -FROM vtxo -WHERE swept = false - AND spent = false - AND unrolled = false - AND expires_at > @after - AND (@before <= 0 OR expires_at < @before); +SELECT COALESCE(SUM(v.amount), 0)::bigint AS amount +FROM vtxo v +WHERE NOT EXISTS ( + SELECT 1 FROM swept_marker sm + WHERE v.markers @> jsonb_build_array(sm.marker_id) + ) + AND v.spent = false + AND v.unrolled = false + AND v.expires_at > @after + AND (@before <= 0 OR v.expires_at < @before); -- name: SelectRecoverableLiquidityAmount :one -SELECT COALESCE(SUM(amount), 0)::bigint AS amount -FROM vtxo -WHERE swept = true - AND spent = false; +SELECT COALESCE(SUM(v.amount), 0)::bigint AS amount +FROM vtxo v +WHERE EXISTS ( + SELECT 1 FROM swept_marker sm + WHERE v.markers @> jsonb_build_array(sm.marker_id) + ) + AND v.spent = false; -- name: SelectOffchainTx :many SELECT sqlc.embed(offchain_tx_vw) FROM offchain_tx_vw WHERE txid = @txid AND COALESCE(fail_reason, '') = ''; @@ -483,10 +485,9 @@ UPDATE vtxo SET markers = @markers::jsonb WHERE txid = @txid AND vout = @vout; -- Find VTXOs whose markers JSONB array contains the given marker_id SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE markers @> jsonb_build_array(@marker_id::TEXT); --- name: SweepVtxosByMarkerId :execrows --- Sweep VTXOs whose markers JSONB array contains the given marker_id -UPDATE vtxo SET swept = true, updated_at = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT -WHERE markers @> jsonb_build_array(@marker_id::TEXT) AND swept = false; +-- name: CountUnsweptVtxosByMarkerId :one +-- Count VTXOs whose markers JSONB array contains the given marker_id and are not swept +SELECT COUNT(*) FROM vtxo_vw WHERE markers @> jsonb_build_array(@marker_id::TEXT) AND swept = false; -- Chain traversal queries for GetVtxoChain optimization diff --git a/internal/infrastructure/db/postgres/vtxo_repo.go b/internal/infrastructure/db/postgres/vtxo_repo.go index 3f539a667..d5e7fab14 100644 --- a/internal/infrastructure/db/postgres/vtxo_repo.go +++ b/internal/infrastructure/db/postgres/vtxo_repo.go @@ -51,7 +51,6 @@ func (v *vtxoRepository) AddVtxos(ctx context.Context, vtxos []domain.Vtxo) erro CommitmentTxid: vtxo.RootCommitmentTxid, Spent: vtxo.Spent, Unrolled: vtxo.Unrolled, - Swept: vtxo.Swept, Preconfirmed: vtxo.Preconfirmed, ExpiresAt: vtxo.ExpiresAt, CreatedAt: vtxo.CreatedAt, @@ -300,35 +299,6 @@ func (v *vtxoRepository) SpendVtxos( return execTx(ctx, v.db, txBody) } -func (v *vtxoRepository) SweepVtxos(ctx context.Context, vtxos []domain.Outpoint) (int, error) { - sweptCount := 0 - txBody := func(querierWithTx *queries.Queries) error { - for _, outpoint := range vtxos { - affectedRows, err := querierWithTx.UpdateVtxoSweptIfNotSwept( - ctx, - queries.UpdateVtxoSweptIfNotSweptParams{ - Txid: outpoint.Txid, - Vout: int32(outpoint.VOut), - }, - ) - if err != nil { - return err - } - if affectedRows > 0 { - sweptCount++ - } - } - - return nil - } - - if err := execTx(ctx, v.db, txBody); err != nil { - return -1, err - } - - return sweptCount, nil -} - func (v *vtxoRepository) UpdateVtxosExpiration( ctx context.Context, vtxos []domain.Outpoint, expiresAt int64, ) error { diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index a7804898f..08f98ead3 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -207,7 +207,13 @@ func NewService(config ServiceConfig, txDecoder ports.TxDecoder) (ports.RepoMana if err != nil { return nil, fmt.Errorf("failed to create intent fees store: %w", err) } - markerStore, err = markerStoreFactory(config.DataStoreConfig...) + // Pass the vtxo store to the marker repository so they share the same data + badgerVtxoRepo, ok := vtxoStore.(*badgerdb.VtxoRepository) + if !ok { + return nil, fmt.Errorf("failed to get badger vtxo repository") + } + markerConfig := append(config.DataStoreConfig, badgerVtxoRepo.GetStore()) + markerStore, err = markerStoreFactory(markerConfig...) if err != nil { return nil, fmt.Errorf("failed to create marker store: %w", err) } @@ -611,19 +617,25 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) // once the offchain tx is finalized, the user signed the checkpoint txs // thus, we can create the new vtxos in the db. newVtxos := make([]domain.Vtxo, 0, len(outs)) + dustVtxoOutpoints := make([]domain.Outpoint, 0) for outIndex, out := range outs { // ignore anchors if bytes.Equal(out.PkScript, txutils.ANCHOR_PKSCRIPT) { continue } + outpoint := domain.Outpoint{ + Txid: txid, + VOut: uint32(outIndex), + } + isDust := script.IsSubDustScript(out.PkScript) + if isDust { + dustVtxoOutpoints = append(dustVtxoOutpoints, outpoint) + } newVtxos = append(newVtxos, domain.Vtxo{ - Outpoint: domain.Outpoint{ - Txid: txid, - VOut: uint32(outIndex), - }, + Outpoint: outpoint, PubKey: hex.EncodeToString(out.PkScript[2:]), Amount: uint64(out.Amount), ExpiresAt: offchainTx.ExpiryTimestamp, @@ -632,10 +644,7 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) Preconfirmed: true, CreatedAt: offchainTx.StartingTimestamp, Depth: newDepth, - // mark the vtxo as "swept" if it is below dust limit to prevent it from being spent again in a future offchain tx - // the only way to spend a swept vtxo is by collecting enough dust to cover the minSettlementVtxoAmount and then settle. - // because sub-dust vtxos are using OP_RETURN output script, they can't be unilaterally exited. - Swept: isDust, + // Swept is now computed via markers, not stored directly }) } @@ -654,6 +663,20 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) } } } + + // Mark dust VTXOs as swept via marker + // Dust vtxos are below dust limit and can't be spent again in future offchain tx + // The only way to spend a swept vtxo is by collecting enough dust to cover the minSettlementVtxoAmount and then settle + // Because sub-dust vtxos are using OP_RETURN output script, they can't be unilaterally exited + if s.markerStore != nil { + sweptAt := time.Now().Unix() + for _, outpoint := range dustVtxoOutpoints { + if err := s.markerStore.MarkDustVtxoSwept(ctx, outpoint, sweptAt); err != nil { + log.WithError(err). + Warnf("failed to mark dust vtxo %s as swept", outpoint.String()) + } + } + } } } @@ -728,7 +751,7 @@ func getNewVtxosFromRound(round *domain.Round) []domain.Vtxo { } // sweepVtxosWithMarkers performs marker-based sweeping for VTXOs. -// It groups VTXOs by their marker, sweeps each marker, then bulk-updates all VTXOs. +// It groups VTXOs by their marker, sweeps each marker via swept_marker table. // Returns the total count of VTXOs swept. func (s *service) sweepVtxosWithMarkers( ctx context.Context, @@ -742,9 +765,7 @@ func (s *service) sweepVtxosWithMarkers( vtxos, err := s.vtxoStore.GetVtxos(ctx, vtxoOutpoints) if err != nil { log.WithError(err).Warn("failed to get vtxos for marker-based sweep") - // Fall back to individual sweep - count, _ := s.vtxoStore.SweepVtxos(ctx, vtxoOutpoints) - return int64(count) + return 0 } // Group VTXOs by their first marker ID (for sweep optimization) @@ -769,32 +790,26 @@ func (s *service) sweepVtxosWithMarkers( // Mark the marker as swept if err := s.markerStore.SweepMarker(ctx, markerID, sweptAt); err != nil { log.WithError(err).Warnf("failed to sweep marker %s", markerID) - // Fall back to individual sweep for this marker's VTXOs - count, _ := s.vtxoStore.SweepVtxos(ctx, markerVtxos[markerID]) - totalSwept += int64(count) continue } - // Bulk sweep all VTXOs with this marker + // Count VTXOs that will be swept by this marker count, err := s.markerStore.SweepVtxosByMarker(ctx, markerID) if err != nil { - log.WithError(err).Warnf("failed to bulk sweep vtxos for marker %s", markerID) - // Fall back to individual sweep - count, _ := s.vtxoStore.SweepVtxos(ctx, markerVtxos[markerID]) - totalSwept += int64(count) + log.WithError(err).Warnf("failed to process sweep for marker %s", markerID) continue } totalSwept += count log.Debugf("swept marker %s with %d vtxos", markerID, count) } - // Sweep VTXOs without markers individually - if len(noMarkerVtxos) > 0 { - count, err := s.vtxoStore.SweepVtxos(ctx, noMarkerVtxos) - if err != nil { - log.WithError(err).Warn("failed to sweep vtxos without markers") + // Sweep VTXOs without markers by creating unique dust markers for each + for _, outpoint := range noMarkerVtxos { + if err := s.markerStore.MarkDustVtxoSwept(ctx, outpoint, sweptAt); err != nil { + log.WithError(err).Warnf("failed to sweep vtxo without marker: %s", outpoint.String()) + continue } - totalSwept += int64(count) + totalSwept++ } return totalSwept diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index 2d94b687d..31f19e6a3 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -1152,6 +1152,17 @@ func testVtxoRepository(t *testing.T, svc ports.RepoManager) { before := liquidityNow + 45 liquidityCommitmentTxid := randomString(32) + expiringVtxoToSweep := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 1}, + PubKey: pubkey, + Amount: 200, + RootCommitmentTxid: liquidityCommitmentTxid, + CommitmentTxids: []string{liquidityCommitmentTxid}, + ExpiresAt: liquidityNow + 20, + Swept: false, // Will be marked as swept via markers + Spent: false, + Unrolled: false, + } expiringVtxos := []domain.Vtxo{ { Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 9}, @@ -1175,17 +1186,7 @@ func testVtxoRepository(t *testing.T, svc ports.RepoManager) { Spent: false, Unrolled: false, }, - { - Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 1}, - PubKey: pubkey, - Amount: 200, - RootCommitmentTxid: liquidityCommitmentTxid, - CommitmentTxids: []string{liquidityCommitmentTxid}, - ExpiresAt: liquidityNow + 20, - Swept: true, - Spent: false, - Unrolled: false, - }, + expiringVtxoToSweep, { Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 2}, PubKey: pubkey, @@ -1223,54 +1224,79 @@ func testVtxoRepository(t *testing.T, svc ports.RepoManager) { err = svc.Vtxos().AddVtxos(ctx, expiringVtxos) require.NoError(t, err) + // Mark the swept vtxo via markers (if marker store is available) + if svc.Markers() != nil { + sweptAt := time.Now().Unix() + err = svc.Markers().MarkDustVtxoSwept(ctx, expiringVtxoToSweep.Outpoint, sweptAt) + require.NoError(t, err) + } + amount, err := svc.Vtxos().GetExpiringLiquidity(ctx, after, before) require.NoError(t, err) + // Only vtxo at VOut=0 with Amount=100 is in range (after < expiresAt < before) require.Equal(t, uint64(100), amount) // before=0 means no upper bound. + // Without marker support: 100 + 200 + 500 = 800 (swept vtxo not excluded) + // With marker support: 100 + 500 = 600 (swept vtxo excluded) amount, err = svc.Vtxos().GetExpiringLiquidity(ctx, liquidityNow, 0) require.NoError(t, err) - require.Equal(t, uint64(600), amount) + if svc.Markers() != nil { + require.Equal(t, uint64(600), amount) + } else { + require.Equal(t, uint64(800), amount) + } recoverableBefore, err := svc.Vtxos().GetRecoverableLiquidity(ctx) require.NoError(t, err) recoverableCommitmentTxid := randomString(32) - recoverableVtxos := []domain.Vtxo{ - { - Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 10}, - PubKey: pubkey, - Amount: 111, - RootCommitmentTxid: recoverableCommitmentTxid, - CommitmentTxids: []string{recoverableCommitmentTxid}, - Swept: true, - Spent: false, - }, - { - Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 11}, - PubKey: pubkey, - Amount: 222, - RootCommitmentTxid: recoverableCommitmentTxid, - CommitmentTxids: []string{recoverableCommitmentTxid}, - Swept: true, - Spent: true, - }, - { - Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 12}, - PubKey: pubkey, - Amount: 333, - RootCommitmentTxid: recoverableCommitmentTxid, - CommitmentTxids: []string{recoverableCommitmentTxid}, - Swept: false, - Spent: false, - }, + recoverableVtxo1 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 10}, + PubKey: pubkey, + Amount: 111, + RootCommitmentTxid: recoverableCommitmentTxid, + CommitmentTxids: []string{recoverableCommitmentTxid}, + Swept: false, // Will be marked as swept via markers + Spent: false, + } + recoverableVtxo2 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 11}, + PubKey: pubkey, + Amount: 222, + RootCommitmentTxid: recoverableCommitmentTxid, + CommitmentTxids: []string{recoverableCommitmentTxid}, + Swept: false, // Will be marked as swept via markers + Spent: true, + } + recoverableVtxo3 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 12}, + PubKey: pubkey, + Amount: 333, + RootCommitmentTxid: recoverableCommitmentTxid, + CommitmentTxids: []string{recoverableCommitmentTxid}, + Swept: false, + Spent: false, } + recoverableVtxos := []domain.Vtxo{recoverableVtxo1, recoverableVtxo2, recoverableVtxo3} err = svc.Vtxos().AddVtxos(ctx, recoverableVtxos) require.NoError(t, err) + // Mark first two vtxos as swept via markers (if marker store is available) + if svc.Markers() != nil { + sweptAt := time.Now().Unix() + err = svc.Markers().MarkDustVtxoSwept(ctx, recoverableVtxo1.Outpoint, sweptAt) + require.NoError(t, err) + err = svc.Markers().MarkDustVtxoSwept(ctx, recoverableVtxo2.Outpoint, sweptAt) + require.NoError(t, err) + } + recoverableAfter, err := svc.Vtxos().GetRecoverableLiquidity(ctx) require.NoError(t, err) - require.Equal(t, recoverableBefore+uint64(111), recoverableAfter) + // Only recoverableVtxo1 is swept and not spent, so it contributes 111 + if svc.Markers() != nil { + require.Equal(t, recoverableBefore+uint64(111), recoverableAfter) + } }) t.Run("test_vtxo_depth", func(t *testing.T) { @@ -1738,7 +1764,7 @@ func testSweepVtxosByMarker(t *testing.T, svc ports.RepoManager) { err := svc.Markers().AddMarker(ctx, marker) require.NoError(t, err) - // Add 5 VTXOs - 3 unswept, 2 already swept + // Add 5 VTXOs - all start as unswept vtxos := make([]domain.Vtxo, 5) for i := 0; i < 5; i++ { vtxos[i] = domain.Vtxo{ @@ -1748,7 +1774,7 @@ func testSweepVtxosByMarker(t *testing.T, svc ports.RepoManager) { RootCommitmentTxid: commitmentTxid, CommitmentTxids: []string{commitmentTxid}, Depth: uint32(i * 10), - Swept: i >= 3, // vtxos[3] and vtxos[4] are already swept + Swept: false, } } @@ -1761,7 +1787,14 @@ func testSweepVtxosByMarker(t *testing.T, svc ports.RepoManager) { require.NoError(t, err) } - // Verify initial state + // Mark vtxos[3] and vtxos[4] as swept via MarkDustVtxoSwept + sweptAt := time.Now().Unix() + err = svc.Markers().MarkDustVtxoSwept(ctx, vtxos[3].Outpoint, sweptAt) + require.NoError(t, err) + err = svc.Markers().MarkDustVtxoSwept(ctx, vtxos[4].Outpoint, sweptAt) + require.NoError(t, err) + + // Verify initial state - vtxos[3] and vtxos[4] should be swept vtxosByMarker, err := svc.Markers().GetVtxosByMarker(ctx, markerID) require.NoError(t, err) require.Len(t, vtxosByMarker, 5) @@ -1774,7 +1807,7 @@ func testSweepVtxosByMarker(t *testing.T, svc ports.RepoManager) { } require.Equal(t, 2, sweptCount) - // Call SweepVtxosByMarker + // Call SweepVtxosByMarker - this sweeps by marking the marker itself as swept count, err := svc.Markers().SweepVtxosByMarker(ctx, markerID) require.NoError(t, err) require.Equal(t, int64(3), count) // Only 3 were newly swept diff --git a/internal/infrastructure/db/sqlite/marker_repo.go b/internal/infrastructure/db/sqlite/marker_repo.go index d7fe13e54..e65c83428 100644 --- a/internal/infrastructure/db/sqlite/marker_repo.go +++ b/internal/infrastructure/db/sqlite/marker_repo.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "strings" + "time" "github.com/arkade-os/arkd/internal/core/domain" "github.com/arkade-os/arkd/internal/infrastructure/db/sqlite/sqlc/queries" @@ -232,10 +233,98 @@ func (m *markerRepository) GetVtxosByMarker( } func (m *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID string) (int64, error) { - return m.querier.SweepVtxosByMarkerId( + // First check if the marker exists (foreign key constraint on swept_marker) + marker, err := m.GetMarker(ctx, markerID) + if err != nil { + return 0, fmt.Errorf("failed to check marker existence: %w", err) + } + if marker == nil { + return 0, nil // Marker doesn't exist, nothing to sweep + } + + // Count unswept VTXOs with this marker before inserting to swept_marker + count, err := m.querier.CountUnsweptVtxosByMarkerId( ctx, sql.NullString{String: markerID, Valid: len(markerID) > 0}, ) + if err != nil { + return 0, fmt.Errorf("failed to count unswept vtxos: %w", err) + } + + // Insert the marker into swept_marker (sweep state is computed via view) + if err := m.querier.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ + MarkerID: markerID, + SweptAt: time.Now().Unix(), + }); err != nil { + return 0, fmt.Errorf("failed to insert swept marker: %w", err) + } + + return count, nil +} + +func (m *markerRepository) MarkDustVtxoSwept( + ctx context.Context, + outpoint domain.Outpoint, + sweptAt int64, +) error { + // Create a unique dust marker for this vtxo + dustMarkerID := outpoint.String() + ":dust" + + // First, get the vtxo to find its depth and current markers + vtxoRow, err := m.querier.SelectVtxo(ctx, queries.SelectVtxoParams{ + Txid: outpoint.Txid, + Vout: int64(outpoint.VOut), + }) + if err != nil { + return fmt.Errorf("failed to get vtxo: %w", err) + } + + // Get current markers from the vtxo + var parentMarkers []string + if vtxoRow.VtxoVw.Markers.Valid && vtxoRow.VtxoVw.Markers.String != "" { + if err := json.Unmarshal([]byte(vtxoRow.VtxoVw.Markers.String), &parentMarkers); err != nil { + parentMarkers = nil + } + } + + parentMarkersJSON, err := json.Marshal(parentMarkers) + if err != nil { + return fmt.Errorf("failed to marshal parent markers: %w", err) + } + + // Create the dust marker + if err := m.querier.UpsertMarker(ctx, queries.UpsertMarkerParams{ + ID: dustMarkerID, + Depth: vtxoRow.VtxoVw.Depth, + ParentMarkers: sql.NullString{String: string(parentMarkersJSON), Valid: true}, + }); err != nil { + return fmt.Errorf("failed to create dust marker: %w", err) + } + + // Insert into swept_marker + if err := m.querier.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ + MarkerID: dustMarkerID, + SweptAt: sweptAt, + }); err != nil { + return fmt.Errorf("failed to insert swept marker: %w", err) + } + + // Update the vtxo's markers to include the dust marker + newMarkers := append(parentMarkers, dustMarkerID) + newMarkersJSON, err := json.Marshal(newMarkers) + if err != nil { + return fmt.Errorf("failed to marshal new markers: %w", err) + } + + if err := m.querier.UpdateVtxoMarkers(ctx, queries.UpdateVtxoMarkersParams{ + Markers: sql.NullString{String: string(newMarkersJSON), Valid: true}, + Txid: outpoint.Txid, + Vout: int64(outpoint.VOut), + }); err != nil { + return fmt.Errorf("failed to update vtxo markers: %w", err) + } + + return nil } func (m *markerRepository) GetVtxosByDepthRange( @@ -340,7 +429,7 @@ func rowToVtxoFromMarkerQuery(row queries.SelectVtxosByMarkerIdRow) domain.Vtxo SpentBy: row.VtxoVw.SpentBy.String, Spent: row.VtxoVw.Spent, Unrolled: row.VtxoVw.Unrolled, - Swept: row.VtxoVw.Swept, + Swept: row.VtxoVw.Swept != 0, Preconfirmed: row.VtxoVw.Preconfirmed, ExpiresAt: row.VtxoVw.ExpiresAt, CreatedAt: row.VtxoVw.CreatedAt, @@ -368,7 +457,7 @@ func rowToVtxoFromDepthRangeQuery(row queries.SelectVtxosByDepthRangeRow) domain SpentBy: row.VtxoVw.SpentBy.String, Spent: row.VtxoVw.Spent, Unrolled: row.VtxoVw.Unrolled, - Swept: row.VtxoVw.Swept, + Swept: row.VtxoVw.Swept != 0, Preconfirmed: row.VtxoVw.Preconfirmed, ExpiresAt: row.VtxoVw.ExpiresAt, CreatedAt: row.VtxoVw.CreatedAt, @@ -396,7 +485,7 @@ func rowToVtxoFromArkTxidQuery(row queries.SelectVtxosByArkTxidRow) domain.Vtxo SpentBy: row.VtxoVw.SpentBy.String, Spent: row.VtxoVw.Spent, Unrolled: row.VtxoVw.Unrolled, - Swept: row.VtxoVw.Swept, + Swept: row.VtxoVw.Swept != 0, Preconfirmed: row.VtxoVw.Preconfirmed, ExpiresAt: row.VtxoVw.ExpiresAt, CreatedAt: row.VtxoVw.CreatedAt, @@ -424,7 +513,7 @@ func rowToVtxoFromChainQuery(row queries.SelectVtxoChainByMarkerRow) domain.Vtxo SpentBy: row.VtxoVw.SpentBy.String, Spent: row.VtxoVw.Spent, Unrolled: row.VtxoVw.Unrolled, - Swept: row.VtxoVw.Swept, + Swept: row.VtxoVw.Swept != 0, Preconfirmed: row.VtxoVw.Preconfirmed, ExpiresAt: row.VtxoVw.ExpiresAt, CreatedAt: row.VtxoVw.CreatedAt, diff --git a/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql index c1daab1f9..a3c4d07cb 100644 --- a/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql +++ b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql @@ -55,3 +55,94 @@ WHERE v.depth % 100 = 0; -- Assign markers array to VTXOs at boundary depths UPDATE vtxo SET markers = '["' || txid || ':' || vout || '"]' WHERE depth % 100 = 0; + +-- Migrate existing swept VTXOs to swept_marker table before dropping column +-- For each swept VTXO, create a unique dust marker and insert into swept_marker +INSERT OR IGNORE INTO marker (id, depth, parent_markers) +SELECT + v.txid || ':' || v.vout || ':dust', + v.depth, + COALESCE(v.markers, '[]') +FROM vtxo v +WHERE v.swept = 1; + +INSERT OR IGNORE INTO swept_marker (marker_id, swept_at) +SELECT + v.txid || ':' || v.vout || ':dust', + strftime('%s', 'now') +FROM vtxo v +WHERE v.swept = 1; + +-- Update swept VTXOs to include the dust marker in their markers array +UPDATE vtxo SET markers = + CASE + WHEN markers IS NULL OR markers = '' THEN '["' || txid || ':' || vout || ':dust"]' + ELSE substr(markers, 1, length(markers)-1) || ',"' || txid || ':' || vout || ':dust"]' + END +WHERE swept = 1; + +-- SQLite doesn't support DROP COLUMN easily, so we recreate the table +-- Create new vtxo table without swept column +CREATE TABLE vtxo_new ( + txid TEXT NOT NULL, + vout INTEGER NOT NULL, + pubkey TEXT NOT NULL, + amount INTEGER NOT NULL, + expires_at INTEGER NOT NULL, + created_at INTEGER NOT NULL, + commitment_txid TEXT NOT NULL, + spent_by TEXT, + spent BOOLEAN NOT NULL DEFAULT FALSE, + unrolled BOOLEAN NOT NULL DEFAULT FALSE, + preconfirmed BOOLEAN NOT NULL DEFAULT FALSE, + settled_by TEXT, + ark_txid TEXT, + intent_id TEXT, + updated_at INTEGER, + depth INTEGER NOT NULL DEFAULT 0, + markers TEXT, + PRIMARY KEY (txid, vout), + FOREIGN KEY (intent_id) REFERENCES intent(id) +); + +-- Copy data from old table (excluding swept column) +INSERT INTO vtxo_new (txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, + spent_by, spent, unrolled, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, markers) +SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, + spent_by, spent, unrolled, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, markers +FROM vtxo; + +-- Drop old views that depend on vtxo +DROP VIEW IF EXISTS intent_with_inputs_vw; +DROP VIEW IF EXISTS vtxo_vw; + +-- Drop old table and rename new one +DROP TABLE vtxo; +ALTER TABLE vtxo_new RENAME TO vtxo; + +-- Recreate indexes +CREATE INDEX IF NOT EXISTS fk_vtxo_intent_id ON vtxo(intent_id); +CREATE INDEX IF NOT EXISTS idx_vtxo_markers ON vtxo(markers); + +-- Recreate views to compute swept status dynamically +CREATE VIEW vtxo_vw AS +SELECT v.*, + COALESCE(group_concat(vc.commitment_txid), '') AS commitments, + EXISTS ( + SELECT 1 FROM swept_marker sm + WHERE v.markers LIKE '%"' || sm.marker_id || '"%' + ) AS swept +FROM vtxo v +LEFT JOIN vtxo_commitment_txid vc +ON v.txid = vc.vtxo_txid AND v.vout = vc.vtxo_vout +GROUP BY v.txid, v.vout; + +CREATE VIEW intent_with_inputs_vw AS +SELECT vtxo_vw.*, + intent.id, + intent.round_id, + intent.proof, + intent.message +FROM intent +LEFT OUTER JOIN vtxo_vw +ON intent.id = vtxo_vw.intent_id; diff --git a/internal/infrastructure/db/sqlite/round_repo.go b/internal/infrastructure/db/sqlite/round_repo.go index b1f202a75..b544201fa 100644 --- a/internal/infrastructure/db/sqlite/round_repo.go +++ b/internal/infrastructure/db/sqlite/round_repo.go @@ -670,7 +670,7 @@ func combinedRowToVtxo(row queries.IntentWithInputsVw) domain.Vtxo { SpentBy: row.SpentBy.String, Spent: row.Spent.Bool, Unrolled: row.Unrolled.Bool, - Swept: row.Swept.Bool, + Swept: row.Swept.Int64 != 0, Preconfirmed: row.Preconfirmed.Bool, ExpiresAt: row.ExpiresAt.Int64, CreatedAt: row.CreatedAt.Int64, diff --git a/internal/infrastructure/db/sqlite/sqlc/queries/models.go b/internal/infrastructure/db/sqlite/sqlc/queries/models.go index 00bf2add0..11112c1ce 100644 --- a/internal/infrastructure/db/sqlite/sqlc/queries/models.go +++ b/internal/infrastructure/db/sqlite/sqlc/queries/models.go @@ -56,7 +56,6 @@ type IntentWithInputsVw struct { SpentBy sql.NullString Spent sql.NullBool Unrolled sql.NullBool - Swept sql.NullBool Preconfirmed sql.NullBool SettledBy sql.NullString ArkTxid sql.NullString @@ -65,6 +64,7 @@ type IntentWithInputsVw struct { Depth sql.NullInt64 Markers sql.NullString Commitments interface{} + Swept sql.NullInt64 ID sql.NullString RoundID sql.NullString Proof sql.NullString @@ -206,7 +206,6 @@ type Vtxo struct { SpentBy sql.NullString Spent bool Unrolled bool - Swept bool Preconfirmed bool SettledBy sql.NullString ArkTxid sql.NullString @@ -233,7 +232,6 @@ type VtxoVw struct { SpentBy sql.NullString Spent bool Unrolled bool - Swept bool Preconfirmed bool SettledBy sql.NullString ArkTxid sql.NullString @@ -242,4 +240,5 @@ type VtxoVw struct { Depth int64 Markers sql.NullString Commitments interface{} + Swept int64 } diff --git a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go index 79dbbf294..c7b461601 100644 --- a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go @@ -84,6 +84,18 @@ func (q *Queries) ClearScheduledSession(ctx context.Context) error { return err } +const countUnsweptVtxosByMarkerId = `-- name: CountUnsweptVtxosByMarkerId :one +SELECT COUNT(*) FROM vtxo_vw WHERE markers LIKE '%"' || ?1 || '"%' AND swept = false +` + +// Count VTXOs whose markers JSON array contains the given marker_id and are not swept +func (q *Queries) CountUnsweptVtxosByMarkerId(ctx context.Context, markerID sql.NullString) (int64, error) { + row := q.db.QueryRowContext(ctx, countUnsweptVtxosByMarkerId, markerID) + var count int64 + err := row.Scan(&count) + return count, err +} + const getDescendantMarkerIds = `-- name: GetDescendantMarkerIds :many WITH RECURSIVE descendant_markers(id) AS ( -- Base case: the marker being swept @@ -240,7 +252,7 @@ func (q *Queries) SelectAllRoundIds(ctx context.Context) ([]string, error) { } const selectAllVtxos = `-- name: SelectAllVtxos :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw ` type SelectAllVtxosRow struct { @@ -267,7 +279,6 @@ func (q *Queries) SelectAllVtxos(ctx context.Context) ([]SelectAllVtxosRow, erro &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -276,6 +287,7 @@ func (q *Queries) SelectAllVtxos(ctx context.Context) ([]SelectAllVtxosRow, erro &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ); err != nil { return nil, err } @@ -395,13 +407,16 @@ func (q *Queries) SelectConvictionsInTimeRange(ctx context.Context, arg SelectCo } const selectExpiringLiquidityAmount = `-- name: SelectExpiringLiquidityAmount :one -SELECT COALESCE(SUM(amount), 0) AS amount -FROM vtxo -WHERE swept = false - AND spent = false - AND unrolled = false - AND expires_at > ?1 - AND (?2 <= 0 OR expires_at < ?2) +SELECT COALESCE(SUM(v.amount), 0) AS amount +FROM vtxo v +WHERE NOT EXISTS ( + SELECT 1 FROM swept_marker sm + WHERE v.markers LIKE '%"' || sm.marker_id || '"%' + ) + AND v.spent = false + AND v.unrolled = false + AND v.expires_at > ?1 + AND (?2 <= 0 OR v.expires_at < ?2) ` type SelectExpiringLiquidityAmountParams struct { @@ -586,7 +601,7 @@ func (q *Queries) SelectMarkersByIds(ctx context.Context, ids []string) ([]Marke } const selectNotUnrolledVtxos = `-- name: SelectNotUnrolledVtxos :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE unrolled = false ` type SelectNotUnrolledVtxosRow struct { @@ -613,7 +628,6 @@ func (q *Queries) SelectNotUnrolledVtxos(ctx context.Context) ([]SelectNotUnroll &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -622,6 +636,7 @@ func (q *Queries) SelectNotUnrolledVtxos(ctx context.Context) ([]SelectNotUnroll &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ); err != nil { return nil, err } @@ -637,7 +652,7 @@ func (q *Queries) SelectNotUnrolledVtxos(ctx context.Context) ([]SelectNotUnroll } const selectNotUnrolledVtxosWithPubkey = `-- name: SelectNotUnrolledVtxosWithPubkey :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE unrolled = false AND pubkey = ?1 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE unrolled = false AND pubkey = ?1 ` type SelectNotUnrolledVtxosWithPubkeyRow struct { @@ -664,7 +679,6 @@ func (q *Queries) SelectNotUnrolledVtxosWithPubkey(ctx context.Context, pubkey s &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -673,6 +687,7 @@ func (q *Queries) SelectNotUnrolledVtxosWithPubkey(ctx context.Context, pubkey s &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ); err != nil { return nil, err } @@ -732,7 +747,7 @@ func (q *Queries) SelectOffchainTx(ctx context.Context, txid string) ([]SelectOf } const selectPendingSpentVtxo = `-- name: SelectPendingSpentVtxo :one -SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.markers, v.commitments +SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.markers, v.commitments, v.swept FROM vtxo_vw v WHERE v.txid = ?1 AND v.vout = ?2 AND v.spent = TRUE AND v.unrolled = FALSE AND COALESCE(v.settled_by, '') = '' @@ -760,7 +775,6 @@ func (q *Queries) SelectPendingSpentVtxo(ctx context.Context, arg SelectPendingS &i.SpentBy, &i.Spent, &i.Unrolled, - &i.Swept, &i.Preconfirmed, &i.SettledBy, &i.ArkTxid, @@ -769,12 +783,13 @@ func (q *Queries) SelectPendingSpentVtxo(ctx context.Context, arg SelectPendingS &i.Depth, &i.Markers, &i.Commitments, + &i.Swept, ) return i, err } const selectPendingSpentVtxosWithPubkeys = `-- name: SelectPendingSpentVtxosWithPubkeys :many -SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.markers, v.commitments +SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.markers, v.commitments, v.swept FROM vtxo_vw v WHERE v.spent = TRUE AND v.unrolled = FALSE AND COALESCE(v.settled_by, '') = '' AND v.pubkey IN (/*SLICE:pubkeys*/?) @@ -823,7 +838,6 @@ func (q *Queries) SelectPendingSpentVtxosWithPubkeys(ctx context.Context, arg Se &i.SpentBy, &i.Spent, &i.Unrolled, - &i.Swept, &i.Preconfirmed, &i.SettledBy, &i.ArkTxid, @@ -832,6 +846,7 @@ func (q *Queries) SelectPendingSpentVtxosWithPubkeys(ctx context.Context, arg Se &i.Depth, &i.Markers, &i.Commitments, + &i.Swept, ); err != nil { return nil, err } @@ -847,10 +862,13 @@ func (q *Queries) SelectPendingSpentVtxosWithPubkeys(ctx context.Context, arg Se } const selectRecoverableLiquidityAmount = `-- name: SelectRecoverableLiquidityAmount :one -SELECT COALESCE(SUM(amount), 0) AS amount -FROM vtxo -WHERE swept = true - AND spent = false +SELECT COALESCE(SUM(v.amount), 0) AS amount +FROM vtxo v +WHERE EXISTS ( + SELECT 1 FROM swept_marker sm + WHERE v.markers LIKE '%"' || sm.marker_id || '"%' + ) + AND v.spent = false ` func (q *Queries) SelectRecoverableLiquidityAmount(ctx context.Context) (interface{}, error) { @@ -1048,7 +1066,7 @@ SELECT r.ending_timestamp, ( SELECT COALESCE(SUM(amount), 0) FROM ( - SELECT DISTINCT v2.txid, v2.vout, v2.pubkey, v2.amount, v2.expires_at, v2.created_at, v2.commitment_txid, v2.spent_by, v2.spent, v2.unrolled, v2.swept, v2.preconfirmed, v2.settled_by, v2.ark_txid, v2.intent_id, v2.updated_at, v2.depth, v2.markers FROM vtxo v2 JOIN intent i2 ON i2.id = v2.intent_id WHERE i2.round_id = r.id + SELECT DISTINCT v2.txid, v2.vout, v2.pubkey, v2.amount, v2.expires_at, v2.created_at, v2.commitment_txid, v2.spent_by, v2.spent, v2.unrolled, v2.preconfirmed, v2.settled_by, v2.ark_txid, v2.intent_id, v2.updated_at, v2.depth, v2.markers FROM vtxo v2 JOIN intent i2 ON i2.id = v2.intent_id WHERE i2.round_id = r.id ) as intent_with_inputs_amount ) AS total_forfeit_amount, ( @@ -1135,7 +1153,7 @@ func (q *Queries) SelectRoundVtxoTree(ctx context.Context, txid string) ([]Tx, e } const selectRoundVtxoTreeLeaves = `-- name: SelectRoundVtxoTreeLeaves :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE commitment_txid = ?1 AND preconfirmed = false +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE commitment_txid = ?1 AND preconfirmed = false ` type SelectRoundVtxoTreeLeavesRow struct { @@ -1162,7 +1180,6 @@ func (q *Queries) SelectRoundVtxoTreeLeaves(ctx context.Context, commitmentTxid &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -1171,6 +1188,7 @@ func (q *Queries) SelectRoundVtxoTreeLeaves(ctx context.Context, commitmentTxid &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ); err != nil { return nil, err } @@ -1190,7 +1208,7 @@ SELECT round.id, round.starting_timestamp, round.ending_timestamp, round.ended, round_intents_vw.id, round_intents_vw.round_id, round_intents_vw.proof, round_intents_vw.message, round_txs_vw.txid, round_txs_vw.tx, round_txs_vw.round_id, round_txs_vw.type, round_txs_vw.position, round_txs_vw.children, intent_with_receivers_vw.intent_id, intent_with_receivers_vw.pubkey, intent_with_receivers_vw.onchain_address, intent_with_receivers_vw.amount, intent_with_receivers_vw.id, intent_with_receivers_vw.round_id, intent_with_receivers_vw.proof, intent_with_receivers_vw.message, - intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.markers, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message + intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.markers, intent_with_inputs_vw.commitments, intent_with_inputs_vw.swept, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message FROM round LEFT OUTER JOIN round_intents_vw ON round.id=round_intents_vw.round_id LEFT OUTER JOIN round_txs_vw ON round.id=round_txs_vw.round_id @@ -1256,7 +1274,6 @@ func (q *Queries) SelectRoundWithId(ctx context.Context, id string) ([]SelectRou &i.IntentWithInputsVw.SpentBy, &i.IntentWithInputsVw.Spent, &i.IntentWithInputsVw.Unrolled, - &i.IntentWithInputsVw.Swept, &i.IntentWithInputsVw.Preconfirmed, &i.IntentWithInputsVw.SettledBy, &i.IntentWithInputsVw.ArkTxid, @@ -1265,6 +1282,7 @@ func (q *Queries) SelectRoundWithId(ctx context.Context, id string) ([]SelectRou &i.IntentWithInputsVw.Depth, &i.IntentWithInputsVw.Markers, &i.IntentWithInputsVw.Commitments, + &i.IntentWithInputsVw.Swept, &i.IntentWithInputsVw.ID, &i.IntentWithInputsVw.RoundID, &i.IntentWithInputsVw.Proof, @@ -1288,7 +1306,7 @@ SELECT round.id, round.starting_timestamp, round.ending_timestamp, round.ended, round_intents_vw.id, round_intents_vw.round_id, round_intents_vw.proof, round_intents_vw.message, round_txs_vw.txid, round_txs_vw.tx, round_txs_vw.round_id, round_txs_vw.type, round_txs_vw.position, round_txs_vw.children, intent_with_receivers_vw.intent_id, intent_with_receivers_vw.pubkey, intent_with_receivers_vw.onchain_address, intent_with_receivers_vw.amount, intent_with_receivers_vw.id, intent_with_receivers_vw.round_id, intent_with_receivers_vw.proof, intent_with_receivers_vw.message, - intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.swept, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.markers, intent_with_inputs_vw.commitments, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message + intent_with_inputs_vw.txid, intent_with_inputs_vw.vout, intent_with_inputs_vw.pubkey, intent_with_inputs_vw.amount, intent_with_inputs_vw.expires_at, intent_with_inputs_vw.created_at, intent_with_inputs_vw.commitment_txid, intent_with_inputs_vw.spent_by, intent_with_inputs_vw.spent, intent_with_inputs_vw.unrolled, intent_with_inputs_vw.preconfirmed, intent_with_inputs_vw.settled_by, intent_with_inputs_vw.ark_txid, intent_with_inputs_vw.intent_id, intent_with_inputs_vw.updated_at, intent_with_inputs_vw.depth, intent_with_inputs_vw.markers, intent_with_inputs_vw.commitments, intent_with_inputs_vw.swept, intent_with_inputs_vw.id, intent_with_inputs_vw.round_id, intent_with_inputs_vw.proof, intent_with_inputs_vw.message FROM round LEFT OUTER JOIN round_intents_vw ON round.id=round_intents_vw.round_id LEFT OUTER JOIN round_txs_vw ON round.id=round_txs_vw.round_id @@ -1356,7 +1374,6 @@ func (q *Queries) SelectRoundWithTxid(ctx context.Context, txid string) ([]Selec &i.IntentWithInputsVw.SpentBy, &i.IntentWithInputsVw.Spent, &i.IntentWithInputsVw.Unrolled, - &i.IntentWithInputsVw.Swept, &i.IntentWithInputsVw.Preconfirmed, &i.IntentWithInputsVw.SettledBy, &i.IntentWithInputsVw.ArkTxid, @@ -1365,6 +1382,7 @@ func (q *Queries) SelectRoundWithTxid(ctx context.Context, txid string) ([]Selec &i.IntentWithInputsVw.Depth, &i.IntentWithInputsVw.Markers, &i.IntentWithInputsVw.Commitments, + &i.IntentWithInputsVw.Swept, &i.IntentWithInputsVw.ID, &i.IntentWithInputsVw.RoundID, &i.IntentWithInputsVw.Proof, @@ -1453,7 +1471,7 @@ func (q *Queries) SelectSweepableRounds(ctx context.Context) ([]string, error) { } const selectSweepableUnrolledVtxos = `-- name: SelectSweepableUnrolledVtxos :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE spent = true AND unrolled = true AND swept = false AND (COALESCE(settled_by, '') = '') +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE spent = true AND unrolled = true AND swept = false AND (COALESCE(settled_by, '') = '') ` type SelectSweepableUnrolledVtxosRow struct { @@ -1480,7 +1498,6 @@ func (q *Queries) SelectSweepableUnrolledVtxos(ctx context.Context) ([]SelectSwe &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -1489,6 +1506,7 @@ func (q *Queries) SelectSweepableUnrolledVtxos(ctx context.Context) ([]SelectSwe &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ); err != nil { return nil, err } @@ -1684,7 +1702,7 @@ func (q *Queries) SelectTxs(ctx context.Context, arg SelectTxsParams) ([]SelectT } const selectVtxo = `-- name: SelectVtxo :one -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE txid = ?1 AND vout = ?2 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE txid = ?1 AND vout = ?2 ` type SelectVtxoParams struct { @@ -1710,7 +1728,6 @@ func (q *Queries) SelectVtxo(ctx context.Context, arg SelectVtxoParams) (SelectV &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -1719,12 +1736,13 @@ func (q *Queries) SelectVtxo(ctx context.Context, arg SelectVtxoParams) (SelectV &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ) return i, err } const selectVtxoChainByMarker = `-- name: SelectVtxoChainByMarker :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE markers LIKE '%"' || ?1 || '"%' ORDER BY vtxo_vw.depth DESC ` @@ -1755,7 +1773,6 @@ func (q *Queries) SelectVtxoChainByMarker(ctx context.Context, markerID sql.Null &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -1764,6 +1781,7 @@ func (q *Queries) SelectVtxoChainByMarker(ctx context.Context, markerID sql.Null &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ); err != nil { return nil, err } @@ -1815,7 +1833,7 @@ func (q *Queries) SelectVtxoPubKeysByCommitmentTxid(ctx context.Context, arg Sel } const selectVtxosByArkTxid = `-- name: SelectVtxosByArkTxid :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE txid = ?1 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE txid = ?1 ` type SelectVtxosByArkTxidRow struct { @@ -1843,7 +1861,6 @@ func (q *Queries) SelectVtxosByArkTxid(ctx context.Context, arkTxid string) ([]S &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -1852,6 +1869,7 @@ func (q *Queries) SelectVtxosByArkTxid(ctx context.Context, arkTxid string) ([]S &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ); err != nil { return nil, err } @@ -1868,7 +1886,7 @@ func (q *Queries) SelectVtxosByArkTxid(ctx context.Context, arkTxid string) ([]S const selectVtxosByDepthRange = `-- name: SelectVtxosByDepthRange :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE depth >= ?1 AND depth <= ?2 ORDER BY depth DESC ` @@ -1904,7 +1922,6 @@ func (q *Queries) SelectVtxosByDepthRange(ctx context.Context, arg SelectVtxosBy &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -1913,6 +1930,7 @@ func (q *Queries) SelectVtxosByDepthRange(ctx context.Context, arg SelectVtxosBy &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ); err != nil { return nil, err } @@ -1928,7 +1946,7 @@ func (q *Queries) SelectVtxosByDepthRange(ctx context.Context, arg SelectVtxosBy } const selectVtxosByMarkerId = `-- name: SelectVtxosByMarkerId :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE markers LIKE '%"' || ?1 || '"%' +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE markers LIKE '%"' || ?1 || '"%' ` type SelectVtxosByMarkerIdRow struct { @@ -1956,7 +1974,6 @@ func (q *Queries) SelectVtxosByMarkerId(ctx context.Context, markerID sql.NullSt &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -1965,6 +1982,7 @@ func (q *Queries) SelectVtxosByMarkerId(ctx context.Context, markerID sql.NullSt &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ); err != nil { return nil, err } @@ -2040,7 +2058,7 @@ func (q *Queries) SelectVtxosOutpointsByArkTxidRecursive(ctx context.Context, tx } const selectVtxosWithPubkeys = `-- name: SelectVtxosWithPubkeys :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.swept, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments FROM vtxo_vw WHERE pubkey IN (/*SLICE:pubkeys*/?) +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE pubkey IN (/*SLICE:pubkeys*/?) AND updated_at >= ?2 AND (CAST(?3 AS INTEGER) = 0 OR updated_at <= CAST(?3 AS INTEGER)) ` @@ -2087,7 +2105,6 @@ func (q *Queries) SelectVtxosWithPubkeys(ctx context.Context, arg SelectVtxosWit &i.VtxoVw.SpentBy, &i.VtxoVw.Spent, &i.VtxoVw.Unrolled, - &i.VtxoVw.Swept, &i.VtxoVw.Preconfirmed, &i.VtxoVw.SettledBy, &i.VtxoVw.ArkTxid, @@ -2096,6 +2113,7 @@ func (q *Queries) SelectVtxosWithPubkeys(ctx context.Context, arg SelectVtxosWit &i.VtxoVw.Depth, &i.VtxoVw.Markers, &i.VtxoVw.Commitments, + &i.VtxoVw.Swept, ); err != nil { return nil, err } @@ -2110,20 +2128,6 @@ func (q *Queries) SelectVtxosWithPubkeys(ctx context.Context, arg SelectVtxosWit return items, nil } -const sweepVtxosByMarkerId = `-- name: SweepVtxosByMarkerId :execrows -UPDATE vtxo SET swept = true, updated_at = (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)) -WHERE markers LIKE '%"' || ?1 || '"%' AND swept = false -` - -// Sweep VTXOs whose markers JSON array contains the given marker_id -func (q *Queries) SweepVtxosByMarkerId(ctx context.Context, markerID sql.NullString) (int64, error) { - result, err := q.db.ExecContext(ctx, sweepVtxosByMarkerId, markerID) - if err != nil { - return 0, err - } - return result.RowsAffected() -} - const updateConvictionPardoned = `-- name: UpdateConvictionPardoned :exec UPDATE conviction SET pardoned = true WHERE id = ?1 ` @@ -2222,23 +2226,6 @@ func (q *Queries) UpdateVtxoSpent(ctx context.Context, arg UpdateVtxoSpentParams return err } -const updateVtxoSweptIfNotSwept = `-- name: UpdateVtxoSweptIfNotSwept :execrows -UPDATE vtxo SET swept = true, updated_at = (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)) WHERE txid = ?1 AND vout = ?2 AND swept = false -` - -type UpdateVtxoSweptIfNotSweptParams struct { - Txid string - Vout int64 -} - -func (q *Queries) UpdateVtxoSweptIfNotSwept(ctx context.Context, arg UpdateVtxoSweptIfNotSweptParams) (int64, error) { - result, err := q.db.ExecContext(ctx, updateVtxoSweptIfNotSwept, arg.Txid, arg.Vout) - if err != nil { - return 0, err - } - return result.RowsAffected() -} - const updateVtxoUnrolled = `-- name: UpdateVtxoUnrolled :exec UPDATE vtxo SET unrolled = true, updated_at = (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)) WHERE txid = ?1 AND vout = ?2 ` @@ -2555,11 +2542,11 @@ func (q *Queries) UpsertTx(ctx context.Context, arg UpsertTxParams) error { const upsertVtxo = `-- name: UpsertVtxo :exec INSERT INTO vtxo ( txid, vout, pubkey, amount, commitment_txid, settled_by, ark_txid, - spent_by, spent, unrolled, swept, preconfirmed, expires_at, created_at, updated_at, depth + spent_by, spent, unrolled, preconfirmed, expires_at, created_at, updated_at, depth ) VALUES ( ?1, ?2, ?3, ?4, ?5, ?6, ?7, - ?8, ?9, ?10, ?11, ?12, ?13, ?14, (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)), ?15 + ?8, ?9, ?10, ?11, ?12, ?13, (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)), ?14 ) ON CONFLICT(txid, vout) DO UPDATE SET pubkey = EXCLUDED.pubkey, amount = EXCLUDED.amount, @@ -2569,7 +2556,6 @@ VALUES ( spent_by = EXCLUDED.spent_by, spent = EXCLUDED.spent, unrolled = EXCLUDED.unrolled, - swept = EXCLUDED.swept, preconfirmed = EXCLUDED.preconfirmed, expires_at = EXCLUDED.expires_at, created_at = EXCLUDED.created_at, @@ -2588,7 +2574,6 @@ type UpsertVtxoParams struct { SpentBy sql.NullString Spent bool Unrolled bool - Swept bool Preconfirmed bool ExpiresAt int64 CreatedAt int64 @@ -2607,7 +2592,6 @@ func (q *Queries) UpsertVtxo(ctx context.Context, arg UpsertVtxoParams) error { arg.SpentBy, arg.Spent, arg.Unrolled, - arg.Swept, arg.Preconfirmed, arg.ExpiresAt, arg.CreatedAt, diff --git a/internal/infrastructure/db/sqlite/sqlc/query.sql b/internal/infrastructure/db/sqlite/sqlc/query.sql index a59039409..b14a1f334 100644 --- a/internal/infrastructure/db/sqlite/sqlc/query.sql +++ b/internal/infrastructure/db/sqlite/sqlc/query.sql @@ -48,11 +48,11 @@ ON CONFLICT(intent_id, pubkey, onchain_address) DO UPDATE SET -- name: UpsertVtxo :exec INSERT INTO vtxo ( txid, vout, pubkey, amount, commitment_txid, settled_by, ark_txid, - spent_by, spent, unrolled, swept, preconfirmed, expires_at, created_at, updated_at, depth + spent_by, spent, unrolled, preconfirmed, expires_at, created_at, updated_at, depth ) VALUES ( @txid, @vout, @pubkey, @amount, @commitment_txid, @settled_by, @ark_txid, - @spent_by, @spent, @unrolled, @swept, @preconfirmed, @expires_at, @created_at, (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)), @depth + @spent_by, @spent, @unrolled, @preconfirmed, @expires_at, @created_at, (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)), @depth ) ON CONFLICT(txid, vout) DO UPDATE SET pubkey = EXCLUDED.pubkey, amount = EXCLUDED.amount, @@ -62,7 +62,6 @@ VALUES ( spent_by = EXCLUDED.spent_by, spent = EXCLUDED.spent, unrolled = EXCLUDED.unrolled, - swept = EXCLUDED.swept, preconfirmed = EXCLUDED.preconfirmed, expires_at = EXCLUDED.expires_at, created_at = EXCLUDED.created_at, @@ -117,9 +116,6 @@ UPDATE vtxo SET expires_at = @expires_at WHERE txid = @txid AND vout = @vout; -- name: UpdateVtxoUnrolled :exec UPDATE vtxo SET unrolled = true, updated_at = (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)) WHERE txid = @txid AND vout = @vout; --- name: UpdateVtxoSweptIfNotSwept :execrows -UPDATE vtxo SET swept = true, updated_at = (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)) WHERE txid = @txid AND vout = @vout AND swept = false; - -- name: UpdateVtxoSettled :exec UPDATE vtxo SET spent = true, spent_by = @spent_by, settled_by = @settled_by, updated_at = (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)) WHERE txid = @txid AND vout = @vout; @@ -261,19 +257,25 @@ SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE pubkey IN (sqlc.slice('pubkeys')) AND (CAST(:before AS INTEGER) = 0 OR updated_at <= CAST(:before AS INTEGER)); -- name: SelectExpiringLiquidityAmount :one -SELECT COALESCE(SUM(amount), 0) AS amount -FROM vtxo -WHERE swept = false - AND spent = false - AND unrolled = false - AND expires_at > sqlc.arg('after') - AND (sqlc.arg('before') <= 0 OR expires_at < sqlc.arg('before')); +SELECT COALESCE(SUM(v.amount), 0) AS amount +FROM vtxo v +WHERE NOT EXISTS ( + SELECT 1 FROM swept_marker sm + WHERE v.markers LIKE '%"' || sm.marker_id || '"%' + ) + AND v.spent = false + AND v.unrolled = false + AND v.expires_at > sqlc.arg('after') + AND (sqlc.arg('before') <= 0 OR v.expires_at < sqlc.arg('before')); -- name: SelectRecoverableLiquidityAmount :one -SELECT COALESCE(SUM(amount), 0) AS amount -FROM vtxo -WHERE swept = true - AND spent = false; +SELECT COALESCE(SUM(v.amount), 0) AS amount +FROM vtxo v +WHERE EXISTS ( + SELECT 1 FROM swept_marker sm + WHERE v.markers LIKE '%"' || sm.marker_id || '"%' + ) + AND v.spent = false; -- name: SelectOffchainTx :many SELECT sqlc.embed(offchain_tx_vw) FROM offchain_tx_vw WHERE txid = @txid AND COALESCE(fail_reason, '') = ''; @@ -486,10 +488,9 @@ UPDATE vtxo SET markers = @markers WHERE txid = @txid AND vout = @vout; -- Find VTXOs whose markers JSON array contains the given marker_id SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE markers LIKE '%"' || @marker_id || '"%'; --- name: SweepVtxosByMarkerId :execrows --- Sweep VTXOs whose markers JSON array contains the given marker_id -UPDATE vtxo SET swept = true, updated_at = (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)) -WHERE markers LIKE '%"' || @marker_id || '"%' AND swept = false; +-- name: CountUnsweptVtxosByMarkerId :one +-- Count VTXOs whose markers JSON array contains the given marker_id and are not swept +SELECT COUNT(*) FROM vtxo_vw WHERE markers LIKE '%"' || @marker_id || '"%' AND swept = false; -- Chain traversal queries for GetVtxoChain optimization diff --git a/internal/infrastructure/db/sqlite/vtxo_repo.go b/internal/infrastructure/db/sqlite/vtxo_repo.go index 49ea2175d..385bd93b3 100644 --- a/internal/infrastructure/db/sqlite/vtxo_repo.go +++ b/internal/infrastructure/db/sqlite/vtxo_repo.go @@ -52,7 +52,6 @@ func (v *vtxoRepository) AddVtxos(ctx context.Context, vtxos []domain.Vtxo) erro SpentBy: sql.NullString{String: vtxo.SpentBy, Valid: len(vtxo.SpentBy) > 0}, Spent: vtxo.Spent, Unrolled: vtxo.Unrolled, - Swept: vtxo.Swept, Preconfirmed: vtxo.Preconfirmed, ExpiresAt: vtxo.ExpiresAt, CreatedAt: vtxo.CreatedAt, @@ -305,35 +304,6 @@ func (v *vtxoRepository) SpendVtxos( return execTx(ctx, v.db, txBody) } -func (v *vtxoRepository) SweepVtxos(ctx context.Context, vtxos []domain.Outpoint) (int, error) { - sweptCount := 0 - txBody := func(querierWithTx *queries.Queries) error { - for _, outpoint := range vtxos { - affectedRows, err := querierWithTx.UpdateVtxoSweptIfNotSwept( - ctx, - queries.UpdateVtxoSweptIfNotSweptParams{ - Txid: outpoint.Txid, - Vout: int64(outpoint.VOut), - }, - ) - if err != nil { - return err - } - if affectedRows > 0 { - sweptCount++ - } - } - - return nil - } - - if err := execTx(ctx, v.db, txBody); err != nil { - return -1, err - } - - return sweptCount, nil -} - func (v *vtxoRepository) UpdateVtxosExpiration( ctx context.Context, vtxos []domain.Outpoint, expiresAt int64, ) error { @@ -534,7 +504,7 @@ func rowToVtxo(row queries.VtxoVw) domain.Vtxo { SpentBy: row.SpentBy.String, Spent: row.Spent, Unrolled: row.Unrolled, - Swept: row.Swept, + Swept: row.Swept != 0, Preconfirmed: row.Preconfirmed, ExpiresAt: row.ExpiresAt, CreatedAt: row.CreatedAt, From 395e128072c76eb88356524e37ae950c98793195 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Wed, 11 Feb 2026 23:27:43 -0500 Subject: [PATCH 08/54] bulksweepmarkers, sweep all of a vtxo's markers --- internal/core/domain/marker_repo.go | 2 + .../infrastructure/db/badger/marker_repo.go | 13 ++++++ .../infrastructure/db/postgres/marker_repo.go | 14 +++++++ .../db/postgres/sqlc/queries/query.sql.go | 16 ++++++++ .../infrastructure/db/postgres/sqlc/query.sql | 5 +++ internal/infrastructure/db/service.go | 40 ++++++++++--------- .../infrastructure/db/sqlite/marker_repo.go | 22 ++++++++++ .../infrastructure/db/sqlite/sqlc/query.sql | 1 + 8 files changed, 95 insertions(+), 18 deletions(-) diff --git a/internal/core/domain/marker_repo.go b/internal/core/domain/marker_repo.go index adeda0a3f..911c36100 100644 --- a/internal/core/domain/marker_repo.go +++ b/internal/core/domain/marker_repo.go @@ -16,6 +16,8 @@ type MarkerRepository interface { // SweepMarker marks a marker as swept at the given timestamp SweepMarker(ctx context.Context, markerID string, sweptAt int64) error + // BulkSweepMarkers marks multiple markers as swept in a single operation + BulkSweepMarkers(ctx context.Context, markerIDs []string, sweptAt int64) error // SweepMarkerWithDescendants marks a marker and all its descendants as swept // Returns the number of markers swept (including descendants) SweepMarkerWithDescendants(ctx context.Context, markerID string, sweptAt int64) (int64, error) diff --git a/internal/infrastructure/db/badger/marker_repo.go b/internal/infrastructure/db/badger/marker_repo.go index 830969c6e..6cb306a20 100644 --- a/internal/infrastructure/db/badger/marker_repo.go +++ b/internal/infrastructure/db/badger/marker_repo.go @@ -253,6 +253,19 @@ func (r *markerRepository) SweepMarker(ctx context.Context, markerID string, swe return nil } +func (r *markerRepository) BulkSweepMarkers( + ctx context.Context, + markerIDs []string, + sweptAt int64, +) error { + for _, markerID := range markerIDs { + if err := r.SweepMarker(ctx, markerID, sweptAt); err != nil { + return err + } + } + return nil +} + func (r *markerRepository) SweepMarkerWithDescendants( ctx context.Context, markerID string, diff --git a/internal/infrastructure/db/postgres/marker_repo.go b/internal/infrastructure/db/postgres/marker_repo.go index 900408cda..d2e2530da 100644 --- a/internal/infrastructure/db/postgres/marker_repo.go +++ b/internal/infrastructure/db/postgres/marker_repo.go @@ -142,6 +142,20 @@ func (m *markerRepository) SweepMarker(ctx context.Context, markerID string, swe }) } +func (m *markerRepository) BulkSweepMarkers( + ctx context.Context, + markerIDs []string, + sweptAt int64, +) error { + if len(markerIDs) == 0 { + return nil + } + return m.querier.BulkInsertSweptMarkers(ctx, queries.BulkInsertSweptMarkersParams{ + MarkerIds: markerIDs, + SweptAt: sweptAt, + }) +} + func (m *markerRepository) SweepMarkerWithDescendants( ctx context.Context, markerID string, diff --git a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go index dab24ec6c..7afafd095 100644 --- a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go @@ -63,6 +63,22 @@ func (q *Queries) AddIntentFees(ctx context.Context, arg AddIntentFeesParams) er return err } +const bulkInsertSweptMarkers = `-- name: BulkInsertSweptMarkers :exec +INSERT INTO swept_marker (marker_id, swept_at) +SELECT unnest($1::text[]), $2 +ON CONFLICT(marker_id) DO NOTHING +` + +type BulkInsertSweptMarkersParams struct { + MarkerIds []string + SweptAt int64 +} + +func (q *Queries) BulkInsertSweptMarkers(ctx context.Context, arg BulkInsertSweptMarkersParams) error { + _, err := q.db.ExecContext(ctx, bulkInsertSweptMarkers, pq.Array(arg.MarkerIds), arg.SweptAt) + return err +} + const clearIntentFees = `-- name: ClearIntentFees :exec INSERT INTO intent_fees ( offchain_input_fee_program, diff --git a/internal/infrastructure/db/postgres/sqlc/query.sql b/internal/infrastructure/db/postgres/sqlc/query.sql index 60b408365..562e8ec9b 100644 --- a/internal/infrastructure/db/postgres/sqlc/query.sql +++ b/internal/infrastructure/db/postgres/sqlc/query.sql @@ -454,6 +454,11 @@ INSERT INTO swept_marker (marker_id, swept_at) VALUES (@marker_id, @swept_at) ON CONFLICT(marker_id) DO NOTHING; +-- name: BulkInsertSweptMarkers :exec +INSERT INTO swept_marker (marker_id, swept_at) +SELECT unnest(@marker_ids::text[]), @swept_at +ON CONFLICT(marker_id) DO NOTHING; + -- name: SelectSweptMarker :one SELECT * FROM swept_marker WHERE marker_id = @marker_id; diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index 08f98ead3..659e6237c 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -768,15 +768,16 @@ func (s *service) sweepVtxosWithMarkers( return 0 } - // Group VTXOs by their first marker ID (for sweep optimization) - // We use first marker to avoid duplicate sweeps when vtxo has multiple markers - markerVtxos := make(map[string][]domain.Outpoint) + // Collect all unique markers from all VTXOs + uniqueMarkers := make(map[string]struct{}) noMarkerVtxos := make([]domain.Outpoint, 0) for _, vtxo := range vtxos { if len(vtxo.MarkerIDs) > 0 { - // Use first marker ID for grouping - markerVtxos[vtxo.MarkerIDs[0]] = append(markerVtxos[vtxo.MarkerIDs[0]], vtxo.Outpoint) + // Collect all markers for this vtxo + for _, markerID := range vtxo.MarkerIDs { + uniqueMarkers[markerID] = struct{}{} + } } else { noMarkerVtxos = append(noMarkerVtxos, vtxo.Outpoint) } @@ -785,24 +786,27 @@ func (s *service) sweepVtxosWithMarkers( var totalSwept int64 sweptAt := time.Now().Unix() - // Sweep each marker - for markerID := range markerVtxos { - // Mark the marker as swept - if err := s.markerStore.SweepMarker(ctx, markerID, sweptAt); err != nil { - log.WithError(err).Warnf("failed to sweep marker %s", markerID) - continue + // Bulk sweep all markers at once + if len(uniqueMarkers) > 0 { + // Convert marker set to slice for bulk sweeping + markerIDs := make([]string, 0, len(uniqueMarkers)) + for markerID := range uniqueMarkers { + markerIDs = append(markerIDs, markerID) } - // Count VTXOs that will be swept by this marker - count, err := s.markerStore.SweepVtxosByMarker(ctx, markerID) - if err != nil { - log.WithError(err).Warnf("failed to process sweep for marker %s", markerID) - continue + if err := s.markerStore.BulkSweepMarkers(ctx, markerIDs, sweptAt); err != nil { + log.WithError(err).Warn("failed to bulk sweep markers") + } else { + // Count VTXOs that have at least one marker (they're all swept now) + totalSwept = int64(len(vtxos) - len(noMarkerVtxos)) + log.Debugf("bulk swept %d markers affecting %d vtxos", len(markerIDs), totalSwept) } - totalSwept += count - log.Debugf("swept marker %s with %d vtxos", markerID, count) } + // Bob: I dont quite understand this part. If there are VTXOs without markers, does that mean they were not swept by the marker-based sweeping? Why do we need to sweep them with unique dust markers? Are these VTXOs that were missed by the marker-based sweeping, or are they a different category of VTXOs that require special handling? + // Bob: I think we cant get rid of this is we assume that every vtxo has >=1 marker. + // Bob: In the current implementation, we create a root marker for every batch VTXO at depth 0, but if there are any VTXOs that for some reason dont have markers (maybe they were created before we implemented marker-based sweeping), we need to sweep them as well. Since they dont have markers, we can create unique dust markers for each of them to mark them as swept. This way, we ensure that all VTXOs are accounted for in the sweeping process, even if they dont have markers. + // Sweep VTXOs without markers by creating unique dust markers for each for _, outpoint := range noMarkerVtxos { if err := s.markerStore.MarkDustVtxoSwept(ctx, outpoint, sweptAt); err != nil { diff --git a/internal/infrastructure/db/sqlite/marker_repo.go b/internal/infrastructure/db/sqlite/marker_repo.go index e65c83428..94d7519f4 100644 --- a/internal/infrastructure/db/sqlite/marker_repo.go +++ b/internal/infrastructure/db/sqlite/marker_repo.go @@ -139,6 +139,28 @@ func (m *markerRepository) SweepMarker(ctx context.Context, markerID string, swe }) } +func (m *markerRepository) BulkSweepMarkers( + ctx context.Context, + markerIDs []string, + sweptAt int64, +) error { + if len(markerIDs) == 0 { + return nil + } + txBody := func(querierWithTx *queries.Queries) error { + for _, markerID := range markerIDs { + if err := querierWithTx.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ + MarkerID: markerID, + SweptAt: sweptAt, + }); err != nil { + return err + } + } + return nil + } + return execTx(ctx, m.db, txBody) +} + func (m *markerRepository) SweepMarkerWithDescendants( ctx context.Context, markerID string, diff --git a/internal/infrastructure/db/sqlite/sqlc/query.sql b/internal/infrastructure/db/sqlite/sqlc/query.sql index b14a1f334..6137a3c7e 100644 --- a/internal/infrastructure/db/sqlite/sqlc/query.sql +++ b/internal/infrastructure/db/sqlite/sqlc/query.sql @@ -457,6 +457,7 @@ INSERT INTO swept_marker (marker_id, swept_at) VALUES (@marker_id, @swept_at) ON CONFLICT(marker_id) DO NOTHING; + -- name: SelectSweptMarker :one SELECT * FROM swept_marker WHERE marker_id = @marker_id; From cc93afbbe68331ce5a1bbf89b408a82a62f12280 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 12 Feb 2026 00:09:25 -0500 Subject: [PATCH 09/54] bulk add new markers and vtxos --- internal/core/domain/marker_repo.go | 4 ++ .../infrastructure/db/badger/marker_repo.go | 29 +++++++++++++ .../infrastructure/db/postgres/marker_repo.go | 43 +++++++++++++++++++ internal/infrastructure/db/service.go | 19 +------- .../infrastructure/db/sqlite/marker_repo.go | 40 +++++++++++++++++ 5 files changed, 118 insertions(+), 17 deletions(-) diff --git a/internal/core/domain/marker_repo.go b/internal/core/domain/marker_repo.go index 911c36100..c999b2113 100644 --- a/internal/core/domain/marker_repo.go +++ b/internal/core/domain/marker_repo.go @@ -38,6 +38,10 @@ type MarkerRepository interface { // Used for dust vtxos that need to be marked swept immediately on creation MarkDustVtxoSwept(ctx context.Context, outpoint Outpoint, sweptAt int64) error + // CreateRootMarkersForVtxos creates root markers for batch VTXOs and updates their marker references + // in a single transaction. Each VTXO gets a marker with ID equal to its outpoint string. + CreateRootMarkersForVtxos(ctx context.Context, vtxos []Vtxo) error + // Chain traversal methods for GetVtxoChain optimization // GetVtxosByDepthRange retrieves VTXOs within a depth range GetVtxosByDepthRange(ctx context.Context, minDepth, maxDepth uint32) ([]Vtxo, error) diff --git a/internal/infrastructure/db/badger/marker_repo.go b/internal/infrastructure/db/badger/marker_repo.go index 6cb306a20..bcb03891f 100644 --- a/internal/infrastructure/db/badger/marker_repo.go +++ b/internal/infrastructure/db/badger/marker_repo.go @@ -492,6 +492,35 @@ func (r *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID stri return count, nil } +func (r *markerRepository) CreateRootMarkersForVtxos( + ctx context.Context, + vtxos []domain.Vtxo, +) error { + if len(vtxos) == 0 { + return nil + } + + for _, vtxo := range vtxos { + markerID := vtxo.Outpoint.String() + + // Create the root marker (depth 0, no parents) + if err := r.AddMarker(ctx, domain.Marker{ + ID: markerID, + Depth: 0, + ParentMarkerIDs: nil, + }); err != nil { + return fmt.Errorf("failed to create marker for vtxo %s: %w", markerID, err) + } + + // Update the vtxo's markers + if err := r.UpdateVtxoMarkers(ctx, vtxo.Outpoint, []string{markerID}); err != nil { + return fmt.Errorf("failed to update markers for vtxo %s: %w", markerID, err) + } + } + + return nil +} + func (r *markerRepository) MarkDustVtxoSwept( ctx context.Context, outpoint domain.Outpoint, diff --git a/internal/infrastructure/db/postgres/marker_repo.go b/internal/infrastructure/db/postgres/marker_repo.go index d2e2530da..f31fa5e93 100644 --- a/internal/infrastructure/db/postgres/marker_repo.go +++ b/internal/infrastructure/db/postgres/marker_repo.go @@ -273,6 +273,49 @@ func (m *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID stri return count, nil } +func (m *markerRepository) CreateRootMarkersForVtxos( + ctx context.Context, + vtxos []domain.Vtxo, +) error { + if len(vtxos) == 0 { + return nil + } + + txBody := func(querierWithTx *queries.Queries) error { + for _, vtxo := range vtxos { + markerID := vtxo.Outpoint.String() + + // Create the root marker (depth 0, no parents) + if err := querierWithTx.UpsertMarker(ctx, queries.UpsertMarkerParams{ + ID: markerID, + Depth: 0, + ParentMarkers: pqtype.NullRawMessage{ + RawMessage: []byte("[]"), + Valid: true, + }, + }); err != nil { + return fmt.Errorf("failed to create marker for vtxo %s: %w", markerID, err) + } + + // Update the vtxo's markers + markersJSON, err := json.Marshal([]string{markerID}) + if err != nil { + return fmt.Errorf("failed to marshal markers: %w", err) + } + if err := querierWithTx.UpdateVtxoMarkers(ctx, queries.UpdateVtxoMarkersParams{ + Markers: markersJSON, + Txid: vtxo.Txid, + Vout: int32(vtxo.VOut), + }); err != nil { + return fmt.Errorf("failed to update markers for vtxo %s: %w", markerID, err) + } + } + return nil + } + + return execTx(ctx, m.db, txBody) +} + func (m *markerRepository) MarkDustVtxoSwept( ctx context.Context, outpoint domain.Outpoint, diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index 659e6237c..13faa609a 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -491,23 +491,8 @@ func (s *service) updateProjectionsAfterRoundEvents(events []domain.Event) { } // Create root markers for batch VTXOs (depth 0 is always at marker boundary) - for _, vtxo := range newVtxos { - // Each batch VTXO at depth 0 gets its own root marker - markerID := vtxo.Outpoint.String() - marker := domain.Marker{ - ID: markerID, - Depth: 0, - ParentMarkerIDs: nil, // Root markers have no parents - } - if err := s.markerStore.AddMarker(ctx, marker); err != nil { - log.WithError(err). - Warnf("failed to create root marker for vtxo %s", vtxo.Outpoint.String()) - continue - } - if err := s.markerStore.UpdateVtxoMarkers(ctx, vtxo.Outpoint, []string{markerID}); err != nil { - log.WithError(err). - Warnf("failed to update markers for vtxo %s", vtxo.Outpoint.String()) - } + if err := s.markerStore.CreateRootMarkersForVtxos(ctx, newVtxos); err != nil { + log.WithError(err).Warn("failed to create root markers for vtxos") } } } diff --git a/internal/infrastructure/db/sqlite/marker_repo.go b/internal/infrastructure/db/sqlite/marker_repo.go index 94d7519f4..2f767b46d 100644 --- a/internal/infrastructure/db/sqlite/marker_repo.go +++ b/internal/infrastructure/db/sqlite/marker_repo.go @@ -284,6 +284,46 @@ func (m *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID stri return count, nil } +func (m *markerRepository) CreateRootMarkersForVtxos( + ctx context.Context, + vtxos []domain.Vtxo, +) error { + if len(vtxos) == 0 { + return nil + } + + txBody := func(querierWithTx *queries.Queries) error { + for _, vtxo := range vtxos { + markerID := vtxo.Outpoint.String() + + // Create the root marker (depth 0, no parents) + if err := querierWithTx.UpsertMarker(ctx, queries.UpsertMarkerParams{ + ID: markerID, + Depth: 0, + ParentMarkers: sql.NullString{String: "[]", Valid: true}, + }); err != nil { + return fmt.Errorf("failed to create marker for vtxo %s: %w", markerID, err) + } + + // Update the vtxo's markers + markersJSON, err := json.Marshal([]string{markerID}) + if err != nil { + return fmt.Errorf("failed to marshal markers: %w", err) + } + if err := querierWithTx.UpdateVtxoMarkers(ctx, queries.UpdateVtxoMarkersParams{ + Markers: sql.NullString{String: string(markersJSON), Valid: true}, + Txid: vtxo.Txid, + Vout: int64(vtxo.VOut), + }); err != nil { + return fmt.Errorf("failed to update markers for vtxo %s: %w", markerID, err) + } + } + return nil + } + + return execTx(ctx, m.db, txBody) +} + func (m *markerRepository) MarkDustVtxoSwept( ctx context.Context, outpoint domain.Outpoint, From 9588705a87ac7af016fc7166f48a09d9fae5c9f3 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 12 Feb 2026 00:36:02 -0500 Subject: [PATCH 10/54] new vtxos get markers set in AddVtxos --- .../db/postgres/sqlc/queries/query.sql.go | 9 ++++++--- .../infrastructure/db/postgres/sqlc/query.sql | 7 ++++--- internal/infrastructure/db/postgres/vtxo_repo.go | 12 +++++++++++- internal/infrastructure/db/service.go | 15 ++------------- .../db/sqlite/sqlc/queries/query.sql.go | 9 ++++++--- internal/infrastructure/db/sqlite/sqlc/query.sql | 7 ++++--- internal/infrastructure/db/sqlite/vtxo_repo.go | 10 ++++++++++ 7 files changed, 43 insertions(+), 26 deletions(-) diff --git a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go index 7afafd095..00667c30d 100644 --- a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go @@ -2459,11 +2459,11 @@ func (q *Queries) UpsertTx(ctx context.Context, arg UpsertTxParams) error { const upsertVtxo = `-- name: UpsertVtxo :exec INSERT INTO vtxo ( txid, vout, pubkey, amount, commitment_txid, settled_by, ark_txid, - spent_by, spent, unrolled, preconfirmed, expires_at, created_at, updated_at, depth + spent_by, spent, unrolled, preconfirmed, expires_at, created_at, updated_at, depth, markers ) VALUES ( $1, $2, $3, $4, $5, $6, $7, - $8, $9, $10, $11, $12, $13, (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, $14 + $8, $9, $10, $11, $12, $13, (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, $14, $15 ) ON CONFLICT(txid, vout) DO UPDATE SET pubkey = EXCLUDED.pubkey, amount = EXCLUDED.amount, @@ -2477,7 +2477,8 @@ VALUES ( expires_at = EXCLUDED.expires_at, created_at = EXCLUDED.created_at, updated_at = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, - depth = EXCLUDED.depth + depth = EXCLUDED.depth, + markers = EXCLUDED.markers ` type UpsertVtxoParams struct { @@ -2495,6 +2496,7 @@ type UpsertVtxoParams struct { ExpiresAt int64 CreatedAt int64 Depth int32 + Markers pqtype.NullRawMessage } func (q *Queries) UpsertVtxo(ctx context.Context, arg UpsertVtxoParams) error { @@ -2513,6 +2515,7 @@ func (q *Queries) UpsertVtxo(ctx context.Context, arg UpsertVtxoParams) error { arg.ExpiresAt, arg.CreatedAt, arg.Depth, + arg.Markers, ) return err } diff --git a/internal/infrastructure/db/postgres/sqlc/query.sql b/internal/infrastructure/db/postgres/sqlc/query.sql index 562e8ec9b..cd0ffddbe 100644 --- a/internal/infrastructure/db/postgres/sqlc/query.sql +++ b/internal/infrastructure/db/postgres/sqlc/query.sql @@ -48,11 +48,11 @@ ON CONFLICT(intent_id, pubkey, onchain_address) DO UPDATE SET -- name: UpsertVtxo :exec INSERT INTO vtxo ( txid, vout, pubkey, amount, commitment_txid, settled_by, ark_txid, - spent_by, spent, unrolled, preconfirmed, expires_at, created_at, updated_at, depth + spent_by, spent, unrolled, preconfirmed, expires_at, created_at, updated_at, depth, markers ) VALUES ( @txid, @vout, @pubkey, @amount, @commitment_txid, @settled_by, @ark_txid, - @spent_by, @spent, @unrolled, @preconfirmed, @expires_at, @created_at, (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, @depth + @spent_by, @spent, @unrolled, @preconfirmed, @expires_at, @created_at, (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, @depth, @markers ) ON CONFLICT(txid, vout) DO UPDATE SET pubkey = EXCLUDED.pubkey, amount = EXCLUDED.amount, @@ -66,7 +66,8 @@ VALUES ( expires_at = EXCLUDED.expires_at, created_at = EXCLUDED.created_at, updated_at = (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, - depth = EXCLUDED.depth; + depth = EXCLUDED.depth, + markers = EXCLUDED.markers; -- name: InsertVtxoCommitmentTxid :exec INSERT INTO vtxo_commitment_txid (vtxo_txid, vtxo_vout, commitment_txid) diff --git a/internal/infrastructure/db/postgres/vtxo_repo.go b/internal/infrastructure/db/postgres/vtxo_repo.go index d5e7fab14..693752251 100644 --- a/internal/infrastructure/db/postgres/vtxo_repo.go +++ b/internal/infrastructure/db/postgres/vtxo_repo.go @@ -42,6 +42,15 @@ func (v *vtxoRepository) AddVtxos(ctx context.Context, vtxos []domain.Vtxo) erro for i := range vtxos { vtxo := vtxos[i] + var markersJSON pqtype.NullRawMessage + if len(vtxo.MarkerIDs) > 0 { + data, err := json.Marshal(vtxo.MarkerIDs) + if err != nil { + return fmt.Errorf("failed to marshal markers: %w", err) + } + markersJSON = pqtype.NullRawMessage{RawMessage: data, Valid: true} + } + if err := querierWithTx.UpsertVtxo( ctx, queries.UpsertVtxoParams{ Txid: vtxo.Txid, @@ -63,7 +72,8 @@ func (v *vtxoRepository) AddVtxos(ctx context.Context, vtxos []domain.Vtxo) erro ArkTxid: sql.NullString{ String: vtxo.ArkTxid, Valid: len(vtxo.ArkTxid) > 0, }, - Depth: int32(vtxo.Depth), + Depth: int32(vtxo.Depth), + Markers: markersJSON, }, ); err != nil { return err diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index 13faa609a..c1e9c3eb4 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -569,6 +569,7 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) } } newDepth = maxDepth + 1 + // Convert parent marker set to slice for id := range parentMarkerSet { parentMarkerIDs = append(parentMarkerIDs, id) } @@ -599,8 +600,6 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) } } - // once the offchain tx is finalized, the user signed the checkpoint txs - // thus, we can create the new vtxos in the db. newVtxos := make([]domain.Vtxo, 0, len(outs)) dustVtxoOutpoints := make([]domain.Outpoint, 0) for outIndex, out := range outs { @@ -629,7 +628,7 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) Preconfirmed: true, CreatedAt: offchainTx.StartingTimestamp, Depth: newDepth, - // Swept is now computed via markers, not stored directly + MarkerIDs: markerIDs, }) } @@ -639,16 +638,6 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) } log.Debugf("added %d vtxos at depth %d", len(newVtxos), newDepth) - // Update markers for VTXOs (new marker at boundary, inherited at non-boundary) - if len(markerIDs) > 0 && s.markerStore != nil { - for _, vtxo := range newVtxos { - if err := s.markerStore.UpdateVtxoMarkers(ctx, vtxo.Outpoint, markerIDs); err != nil { - log.WithError(err). - Warnf("failed to update markers for vtxo %s", vtxo.Outpoint.String()) - } - } - } - // Mark dust VTXOs as swept via marker // Dust vtxos are below dust limit and can't be spent again in future offchain tx // The only way to spend a swept vtxo is by collecting enough dust to cover the minSettlementVtxoAmount and then settle diff --git a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go index c7b461601..16df64b80 100644 --- a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go @@ -2542,11 +2542,11 @@ func (q *Queries) UpsertTx(ctx context.Context, arg UpsertTxParams) error { const upsertVtxo = `-- name: UpsertVtxo :exec INSERT INTO vtxo ( txid, vout, pubkey, amount, commitment_txid, settled_by, ark_txid, - spent_by, spent, unrolled, preconfirmed, expires_at, created_at, updated_at, depth + spent_by, spent, unrolled, preconfirmed, expires_at, created_at, updated_at, depth, markers ) VALUES ( ?1, ?2, ?3, ?4, ?5, ?6, ?7, - ?8, ?9, ?10, ?11, ?12, ?13, (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)), ?14 + ?8, ?9, ?10, ?11, ?12, ?13, (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)), ?14, ?15 ) ON CONFLICT(txid, vout) DO UPDATE SET pubkey = EXCLUDED.pubkey, amount = EXCLUDED.amount, @@ -2560,7 +2560,8 @@ VALUES ( expires_at = EXCLUDED.expires_at, created_at = EXCLUDED.created_at, updated_at = (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)), - depth = EXCLUDED.depth + depth = EXCLUDED.depth, + markers = EXCLUDED.markers ` type UpsertVtxoParams struct { @@ -2578,6 +2579,7 @@ type UpsertVtxoParams struct { ExpiresAt int64 CreatedAt int64 Depth int64 + Markers sql.NullString } func (q *Queries) UpsertVtxo(ctx context.Context, arg UpsertVtxoParams) error { @@ -2596,6 +2598,7 @@ func (q *Queries) UpsertVtxo(ctx context.Context, arg UpsertVtxoParams) error { arg.ExpiresAt, arg.CreatedAt, arg.Depth, + arg.Markers, ) return err } diff --git a/internal/infrastructure/db/sqlite/sqlc/query.sql b/internal/infrastructure/db/sqlite/sqlc/query.sql index 6137a3c7e..98996023d 100644 --- a/internal/infrastructure/db/sqlite/sqlc/query.sql +++ b/internal/infrastructure/db/sqlite/sqlc/query.sql @@ -48,11 +48,11 @@ ON CONFLICT(intent_id, pubkey, onchain_address) DO UPDATE SET -- name: UpsertVtxo :exec INSERT INTO vtxo ( txid, vout, pubkey, amount, commitment_txid, settled_by, ark_txid, - spent_by, spent, unrolled, preconfirmed, expires_at, created_at, updated_at, depth + spent_by, spent, unrolled, preconfirmed, expires_at, created_at, updated_at, depth, markers ) VALUES ( @txid, @vout, @pubkey, @amount, @commitment_txid, @settled_by, @ark_txid, - @spent_by, @spent, @unrolled, @preconfirmed, @expires_at, @created_at, (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)), @depth + @spent_by, @spent, @unrolled, @preconfirmed, @expires_at, @created_at, (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)), @depth, @markers ) ON CONFLICT(txid, vout) DO UPDATE SET pubkey = EXCLUDED.pubkey, amount = EXCLUDED.amount, @@ -66,7 +66,8 @@ VALUES ( expires_at = EXCLUDED.expires_at, created_at = EXCLUDED.created_at, updated_at = (CAST((strftime('%s','now') || substr(strftime('%f','now'),4,3)) AS INTEGER)), - depth = EXCLUDED.depth; + depth = EXCLUDED.depth, + markers = EXCLUDED.markers; -- name: InsertVtxoCommitmentTxid :exec INSERT INTO vtxo_commitment_txid (vtxo_txid, vtxo_vout, commitment_txid) diff --git a/internal/infrastructure/db/sqlite/vtxo_repo.go b/internal/infrastructure/db/sqlite/vtxo_repo.go index 385bd93b3..5980673d1 100644 --- a/internal/infrastructure/db/sqlite/vtxo_repo.go +++ b/internal/infrastructure/db/sqlite/vtxo_repo.go @@ -42,6 +42,15 @@ func (v *vtxoRepository) AddVtxos(ctx context.Context, vtxos []domain.Vtxo) erro for i := range vtxos { vtxo := vtxos[i] + var markersJSON sql.NullString + if len(vtxo.MarkerIDs) > 0 { + data, err := json.Marshal(vtxo.MarkerIDs) + if err != nil { + return fmt.Errorf("failed to marshal markers: %w", err) + } + markersJSON = sql.NullString{String: string(data), Valid: true} + } + if err := querierWithTx.UpsertVtxo( ctx, queries.UpsertVtxoParams{ Txid: vtxo.Txid, @@ -58,6 +67,7 @@ func (v *vtxoRepository) AddVtxos(ctx context.Context, vtxos []domain.Vtxo) erro ArkTxid: sql.NullString{String: vtxo.ArkTxid, Valid: len(vtxo.ArkTxid) > 0}, SettledBy: sql.NullString{String: vtxo.SettledBy, Valid: len(vtxo.SettledBy) > 0}, Depth: int64(vtxo.Depth), + Markers: markersJSON, }, ); err != nil { return err From 18932e9fcc73cab85b0e1084cc8351e18a8a5541 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 12 Feb 2026 02:27:20 -0500 Subject: [PATCH 11/54] populate MarkerIds in getNewVtxosFromRound, createCheckpointSweepTask to use BulkSweepMarkers --- internal/core/application/indexer.go | 2 + internal/core/application/sweeper.go | 45 ++++++++++++------- internal/core/application/utils.go | 6 ++- .../infrastructure/db/badger/marker_repo.go | 6 +-- .../infrastructure/db/postgres/marker_repo.go | 14 +----- internal/infrastructure/db/service.go | 7 ++- .../infrastructure/db/sqlite/marker_repo.go | 14 +----- 7 files changed, 44 insertions(+), 50 deletions(-) diff --git a/internal/core/application/indexer.go b/internal/core/application/indexer.go index 19ee90c73..05145cbbb 100644 --- a/internal/core/application/indexer.go +++ b/internal/core/application/indexer.go @@ -378,6 +378,7 @@ func (i *indexerService) GetVtxoChain( func (i *indexerService) prefetchVtxosByMarkers( ctx context.Context, startKey Outpoint, ) map[string]domain.Vtxo { + // outpoint string -> VTXO cache cache := make(map[string]domain.Vtxo) if i.repoManager.Markers() == nil { @@ -402,6 +403,7 @@ func (i *indexerService) prefetchVtxosByMarkers( markerIDs := make([]string, 0, len(startVtxo.MarkerIDs)) markerIDs = append(markerIDs, startVtxo.MarkerIDs...) + // Bob: we have to follow all parent markers because a vtxo can be associated with multiple markers if it was created at a depth that is a multiple of the marker interval. For example, if the marker interval is 100, a vtxo created at depth 200 would be associated with the markers at depth 100 and 200. To ensure we prefetch all relevant VTXOs, we need to follow all parent markers up the chain until we reach the root marker (depth 0). // BFS to follow all parent markers visited := make(map[string]bool) for _, id := range startVtxo.MarkerIDs { diff --git a/internal/core/application/sweeper.go b/internal/core/application/sweeper.go index 66a894ac1..bff18ef8f 100644 --- a/internal/core/application/sweeper.go +++ b/internal/core/application/sweeper.go @@ -761,28 +761,41 @@ func (s *sweeper) createCheckpointSweepTask( return err } - // Sweep each VTXO by marking its markers as swept + // Collect all unique markers and vtxos without markers sweptAt := time.Now().Unix() markerStore := s.repoManager.Markers() - sweptCount := 0 + uniqueMarkers := make(map[string]struct{}) + var noMarkerVtxos []domain.Outpoint for _, v := range vtxos { if len(v.MarkerIDs) > 0 { - // Sweep via first marker - if err := markerStore.SweepMarker(ctx, v.MarkerIDs[0], sweptAt); err != nil { - log.WithError(err).Warnf("failed to sweep marker %s", v.MarkerIDs[0]) - continue - } - if _, err := markerStore.SweepVtxosByMarker(ctx, v.MarkerIDs[0]); err != nil { - log.WithError(err). - Warnf("failed to process sweep for marker %s", v.MarkerIDs[0]) - continue + for _, markerID := range v.MarkerIDs { + uniqueMarkers[markerID] = struct{}{} } } else { - // Create a dust marker for vtxos without markers - if err := markerStore.MarkDustVtxoSwept(ctx, v.Outpoint, sweptAt); err != nil { - log.WithError(err).Warnf("failed to mark vtxo %s as swept", v.Outpoint.String()) - continue - } + noMarkerVtxos = append(noMarkerVtxos, v.Outpoint) + } + } + + // Bulk sweep all markers at once + sweptCount := 0 + if len(uniqueMarkers) > 0 { + markerIDs := make([]string, 0, len(uniqueMarkers)) + for markerID := range uniqueMarkers { + markerIDs = append(markerIDs, markerID) + } + if err := markerStore.BulkSweepMarkers(ctx, markerIDs, sweptAt); err != nil { + log.WithError(err).Warn("failed to bulk sweep markers") + } else { + sweptCount = len(vtxos) - len(noMarkerVtxos) + log.Debugf("bulk swept %d markers affecting %d vtxos", len(markerIDs), sweptCount) + } + } + + // Sweep VTXOs without markers by creating unique dust markers + for _, outpoint := range noMarkerVtxos { + if err := markerStore.MarkDustVtxoSwept(ctx, outpoint, sweptAt); err != nil { + log.WithError(err).Warnf("failed to mark vtxo %s as swept", outpoint.String()) + continue } sweptCount++ } diff --git a/internal/core/application/utils.go b/internal/core/application/utils.go index bd03d2663..650cf7ff1 100644 --- a/internal/core/application/utils.go +++ b/internal/core/application/utils.go @@ -234,15 +234,17 @@ func getNewVtxosFromRound(round *domain.Round) []domain.Vtxo { } vtxoPubkey := hex.EncodeToString(schnorr.SerializePubKey(vtxoTapKey)) + outpoint := domain.Outpoint{Txid: tx.UnsignedTx.TxID(), VOut: uint32(i)} vtxos = append(vtxos, domain.Vtxo{ - Outpoint: domain.Outpoint{Txid: tx.UnsignedTx.TxID(), VOut: uint32(i)}, + Outpoint: outpoint, PubKey: vtxoPubkey, Amount: uint64(out.Value), CommitmentTxids: []string{round.CommitmentTxid}, RootCommitmentTxid: round.CommitmentTxid, CreatedAt: createdAt, ExpiresAt: expireAt, - Depth: 0, // new vtxo from batch starts at depth 0 + Depth: 0, + MarkerIDs: []string{outpoint.String()}, }) } } diff --git a/internal/infrastructure/db/badger/marker_repo.go b/internal/infrastructure/db/badger/marker_repo.go index bcb03891f..68d073775 100644 --- a/internal/infrastructure/db/badger/marker_repo.go +++ b/internal/infrastructure/db/badger/marker_repo.go @@ -504,6 +504,7 @@ func (r *markerRepository) CreateRootMarkersForVtxos( markerID := vtxo.Outpoint.String() // Create the root marker (depth 0, no parents) + // Note: vtxo.MarkerIDs should already be set before AddVtxos is called if err := r.AddMarker(ctx, domain.Marker{ ID: markerID, Depth: 0, @@ -511,11 +512,6 @@ func (r *markerRepository) CreateRootMarkersForVtxos( }); err != nil { return fmt.Errorf("failed to create marker for vtxo %s: %w", markerID, err) } - - // Update the vtxo's markers - if err := r.UpdateVtxoMarkers(ctx, vtxo.Outpoint, []string{markerID}); err != nil { - return fmt.Errorf("failed to update markers for vtxo %s: %w", markerID, err) - } } return nil diff --git a/internal/infrastructure/db/postgres/marker_repo.go b/internal/infrastructure/db/postgres/marker_repo.go index f31fa5e93..cccca8a4f 100644 --- a/internal/infrastructure/db/postgres/marker_repo.go +++ b/internal/infrastructure/db/postgres/marker_repo.go @@ -286,6 +286,7 @@ func (m *markerRepository) CreateRootMarkersForVtxos( markerID := vtxo.Outpoint.String() // Create the root marker (depth 0, no parents) + // Note: vtxo.MarkerIDs should already be set before AddVtxos is called if err := querierWithTx.UpsertMarker(ctx, queries.UpsertMarkerParams{ ID: markerID, Depth: 0, @@ -296,19 +297,6 @@ func (m *markerRepository) CreateRootMarkersForVtxos( }); err != nil { return fmt.Errorf("failed to create marker for vtxo %s: %w", markerID, err) } - - // Update the vtxo's markers - markersJSON, err := json.Marshal([]string{markerID}) - if err != nil { - return fmt.Errorf("failed to marshal markers: %w", err) - } - if err := querierWithTx.UpdateVtxoMarkers(ctx, queries.UpdateVtxoMarkersParams{ - Markers: markersJSON, - Txid: vtxo.Txid, - Vout: int32(vtxo.VOut), - }); err != nil { - return fmt.Errorf("failed to update markers for vtxo %s: %w", markerID, err) - } } return nil } diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index c1e9c3eb4..22200d25f 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -638,6 +638,8 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) } log.Debugf("added %d vtxos at depth %d", len(newVtxos), newDepth) + // Bob: do we need to handle dust differently? same question in sweepVtxosWithMarkers + // where we do it as well. // Mark dust VTXOs as swept via marker // Dust vtxos are below dust limit and can't be spent again in future offchain tx // The only way to spend a swept vtxo is by collecting enough dust to cover the minSettlementVtxoAmount and then settle @@ -710,14 +712,17 @@ func getNewVtxosFromRound(round *domain.Round) []domain.Vtxo { } vtxoPubkey := hex.EncodeToString(schnorr.SerializePubKey(vtxoTapKey)) + outpoint := domain.Outpoint{Txid: tx.UnsignedTx.TxID(), VOut: uint32(i)} vtxos = append(vtxos, domain.Vtxo{ - Outpoint: domain.Outpoint{Txid: tx.UnsignedTx.TxID(), VOut: uint32(i)}, + Outpoint: outpoint, PubKey: vtxoPubkey, Amount: uint64(out.Value), CommitmentTxids: []string{round.CommitmentTxid}, RootCommitmentTxid: round.CommitmentTxid, CreatedAt: round.EndingTimestamp, ExpiresAt: round.ExpiryTimestamp(), + Depth: 0, + MarkerIDs: []string{outpoint.String()}, }) } } diff --git a/internal/infrastructure/db/sqlite/marker_repo.go b/internal/infrastructure/db/sqlite/marker_repo.go index 2f767b46d..20e76daf1 100644 --- a/internal/infrastructure/db/sqlite/marker_repo.go +++ b/internal/infrastructure/db/sqlite/marker_repo.go @@ -297,6 +297,7 @@ func (m *markerRepository) CreateRootMarkersForVtxos( markerID := vtxo.Outpoint.String() // Create the root marker (depth 0, no parents) + // Note: vtxo.MarkerIDs should already be set before AddVtxos is called if err := querierWithTx.UpsertMarker(ctx, queries.UpsertMarkerParams{ ID: markerID, Depth: 0, @@ -304,19 +305,6 @@ func (m *markerRepository) CreateRootMarkersForVtxos( }); err != nil { return fmt.Errorf("failed to create marker for vtxo %s: %w", markerID, err) } - - // Update the vtxo's markers - markersJSON, err := json.Marshal([]string{markerID}) - if err != nil { - return fmt.Errorf("failed to marshal markers: %w", err) - } - if err := querierWithTx.UpdateVtxoMarkers(ctx, queries.UpdateVtxoMarkersParams{ - Markers: sql.NullString{String: string(markersJSON), Valid: true}, - Txid: vtxo.Txid, - Vout: int64(vtxo.VOut), - }); err != nil { - return fmt.Errorf("failed to update markers for vtxo %s: %w", markerID, err) - } } return nil } From 56f6cd1ad5311b7e9ffbf8101e2d675e6e876e57 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 12 Feb 2026 02:33:37 -0500 Subject: [PATCH 12/54] linted with go1.25.7 --- cmd/arkd/commands.go | 9 ++- internal/core/application/fraud.go | 6 +- internal/core/application/sweeper.go | 12 +++- internal/core/application/utils.go | 6 +- .../infrastructure/db/sqlite/marker_repo.go | 5 +- .../infrastructure/db/sqlite/vtxo_repo.go | 42 +++++++----- .../tx-builder/covenantless/builder.go | 6 +- internal/test/e2e/e2e_test.go | 65 ++++++++++++++++--- 8 files changed, 117 insertions(+), 34 deletions(-) diff --git a/cmd/arkd/commands.go b/cmd/arkd/commands.go index 6b68700cd..9113b5115 100644 --- a/cmd/arkd/commands.go +++ b/cmd/arkd/commands.go @@ -156,9 +156,12 @@ var ( Name: "update", Usage: "Update the scheduled session configuration", Flags: []cli.Flag{ - scheduledSessionStartDateFlag, scheduledSessionEndDateFlag, - scheduledSessionDurationFlag, scheduledSessionPeriodFlag, - scheduledSessionRoundMinParticipantsCountFlag, scheduledSessionRoundMaxParticipantsCountFlag, + scheduledSessionStartDateFlag, + scheduledSessionEndDateFlag, + scheduledSessionDurationFlag, + scheduledSessionPeriodFlag, + scheduledSessionRoundMinParticipantsCountFlag, + scheduledSessionRoundMaxParticipantsCountFlag, }, Action: updateScheduledSessionAction, } diff --git a/internal/core/application/fraud.go b/internal/core/application/fraud.go index d1f3440ed..1fd530426 100644 --- a/internal/core/application/fraud.go +++ b/internal/core/application/fraud.go @@ -61,7 +61,11 @@ func (s *service) reactToFraud(ctx context.Context, vtxo domain.Vtxo, mutx *sync return } } - if err := s.sweeper.scheduleCheckpointSweep(vtxo.Outpoint, ptx, blockTimestamp); err != nil { + if err := s.sweeper.scheduleCheckpointSweep( + vtxo.Outpoint, + ptx, + blockTimestamp, + ); err != nil { log.Errorf("failed to schedule checkpoint sweep: %s", err) } }() diff --git a/internal/core/application/sweeper.go b/internal/core/application/sweeper.go index bff18ef8f..a32f212ec 100644 --- a/internal/core/application/sweeper.go +++ b/internal/core/application/sweeper.go @@ -187,7 +187,11 @@ func (s *sweeper) start(ctx context.Context) error { return } - if err := s.scheduleCheckpointSweep(vtxo.Outpoint, checkpointTx, blockTimestamp); err != nil { + if err := s.scheduleCheckpointSweep( + vtxo.Outpoint, + checkpointTx, + blockTimestamp, + ); err != nil { log.WithError(err).Errorf( "failed to schedule sweep task for checkpoint %s", checkpointTxid, ) @@ -480,7 +484,11 @@ func (s *sweeper) createBatchSweepTask(commitmentTxid, vtxoTreeRootTxid string) expirationTimestamp = blockTimestamp.Time + vtxoTreeExpiry.Seconds() } - if err := s.scheduleBatchSweep(expirationTimestamp, txid, tree.Root.UnsignedTx.TxID()); err != nil { + if err := s.scheduleBatchSweep( + expirationTimestamp, + txid, + tree.Root.UnsignedTx.TxID(), + ); err != nil { log.WithError(err).Errorf( "failed to schedule sweep for vtxo tree %s of batch %s", tree.Root.UnsignedTx.TxID(), commitmentTxid, diff --git a/internal/core/application/utils.go b/internal/core/application/utils.go index 650cf7ff1..fcbce8c38 100644 --- a/internal/core/application/utils.go +++ b/internal/core/application/utils.go @@ -285,7 +285,11 @@ func treeTxNoncesEvents( txNonce, ok := noncesForCosigner[txid] if !ok { - return false, fmt.Errorf("missing nonce for cosigner key %s and txid %s", keyStr, txid) + return false, fmt.Errorf( + "missing nonce for cosigner key %s and txid %s", + keyStr, + txid, + ) } noncesByPubkey[keyStr] = txNonce diff --git a/internal/infrastructure/db/sqlite/marker_repo.go b/internal/infrastructure/db/sqlite/marker_repo.go index 20e76daf1..44dbd9e7b 100644 --- a/internal/infrastructure/db/sqlite/marker_repo.go +++ b/internal/infrastructure/db/sqlite/marker_repo.go @@ -332,7 +332,10 @@ func (m *markerRepository) MarkDustVtxoSwept( // Get current markers from the vtxo var parentMarkers []string if vtxoRow.VtxoVw.Markers.Valid && vtxoRow.VtxoVw.Markers.String != "" { - if err := json.Unmarshal([]byte(vtxoRow.VtxoVw.Markers.String), &parentMarkers); err != nil { + if err := json.Unmarshal( + []byte(vtxoRow.VtxoVw.Markers.String), + &parentMarkers, + ); err != nil { parentMarkers = nil } } diff --git a/internal/infrastructure/db/sqlite/vtxo_repo.go b/internal/infrastructure/db/sqlite/vtxo_repo.go index 5980673d1..64ef27d2a 100644 --- a/internal/infrastructure/db/sqlite/vtxo_repo.go +++ b/internal/infrastructure/db/sqlite/vtxo_repo.go @@ -58,26 +58,38 @@ func (v *vtxoRepository) AddVtxos(ctx context.Context, vtxos []domain.Vtxo) erro Pubkey: vtxo.PubKey, Amount: int64(vtxo.Amount), CommitmentTxid: vtxo.RootCommitmentTxid, - SpentBy: sql.NullString{String: vtxo.SpentBy, Valid: len(vtxo.SpentBy) > 0}, - Spent: vtxo.Spent, - Unrolled: vtxo.Unrolled, - Preconfirmed: vtxo.Preconfirmed, - ExpiresAt: vtxo.ExpiresAt, - CreatedAt: vtxo.CreatedAt, - ArkTxid: sql.NullString{String: vtxo.ArkTxid, Valid: len(vtxo.ArkTxid) > 0}, - SettledBy: sql.NullString{String: vtxo.SettledBy, Valid: len(vtxo.SettledBy) > 0}, - Depth: int64(vtxo.Depth), - Markers: markersJSON, + SpentBy: sql.NullString{ + String: vtxo.SpentBy, + Valid: len(vtxo.SpentBy) > 0, + }, + Spent: vtxo.Spent, + Unrolled: vtxo.Unrolled, + Preconfirmed: vtxo.Preconfirmed, + ExpiresAt: vtxo.ExpiresAt, + CreatedAt: vtxo.CreatedAt, + ArkTxid: sql.NullString{ + String: vtxo.ArkTxid, + Valid: len(vtxo.ArkTxid) > 0, + }, + SettledBy: sql.NullString{ + String: vtxo.SettledBy, + Valid: len(vtxo.SettledBy) > 0, + }, + Depth: int64(vtxo.Depth), + Markers: markersJSON, }, ); err != nil { return err } for _, txid := range vtxo.CommitmentTxids { - if err := querierWithTx.InsertVtxoCommitmentTxid(ctx, queries.InsertVtxoCommitmentTxidParams{ - VtxoTxid: vtxo.Txid, - VtxoVout: int64(vtxo.VOut), - CommitmentTxid: txid, - }); err != nil { + if err := querierWithTx.InsertVtxoCommitmentTxid( + ctx, + queries.InsertVtxoCommitmentTxidParams{ + VtxoTxid: vtxo.Txid, + VtxoVout: int64(vtxo.VOut), + CommitmentTxid: txid, + }, + ); err != nil { return err } } diff --git a/internal/infrastructure/tx-builder/covenantless/builder.go b/internal/infrastructure/tx-builder/covenantless/builder.go index d1caf6a09..560224c03 100644 --- a/internal/infrastructure/tx-builder/covenantless/builder.go +++ b/internal/infrastructure/tx-builder/covenantless/builder.go @@ -109,7 +109,11 @@ func (b *txBuilder) verifyTapscriptPartialSigs( keys[hex.EncodeToString(schnorr.SerializePubKey(key))] = false } case *script.ConditionMultisigClosure: - witnessFields, err := txutils.GetArkPsbtFields(ptx, index, txutils.ConditionWitnessField) + witnessFields, err := txutils.GetArkPsbtFields( + ptx, + index, + txutils.ConditionWitnessField, + ) if err != nil { return false, nil, err } diff --git a/internal/test/e2e/e2e_test.go b/internal/test/e2e/e2e_test.go index dfe4b1b5b..97896f180 100644 --- a/internal/test/e2e/e2e_test.go +++ b/internal/test/e2e/e2e_test.go @@ -3224,7 +3224,11 @@ func TestBan(t *testing.T) { sweepTapTree := txscript.AssembleTaprootScriptTree(sweepTapLeaf) root := sweepTapTree.RootNode.TapHash() - if err := signerSession.Init(root.CloneBytes(), batchOutputAmount, vtxoTree); err != nil { + if err := signerSession.Init( + root.CloneBytes(), + batchOutputAmount, + vtxoTree, + ); err != nil { return false, err } @@ -3233,7 +3237,12 @@ func TestBan(t *testing.T) { return false, err } - if err = grpcAlice.SubmitTreeNonces(ctx, event.Id, signerSession.GetPublicKey(), nonces); err != nil { + if err = grpcAlice.SubmitTreeNonces( + ctx, + event.Id, + signerSession.GetPublicKey(), + nonces, + ); err != nil { return false, err } @@ -3336,7 +3345,11 @@ func TestBan(t *testing.T) { // use a fake sweep to create invalid signatures fakeSweepTapHash := sha256.Sum256([]byte("random_sweep_tap_hash")) - if err := signerSession.Init(fakeSweepTapHash[:], batchOutputAmount, vtxoTree); err != nil { + if err := signerSession.Init( + fakeSweepTapHash[:], + batchOutputAmount, + vtxoTree, + ); err != nil { return false, err } @@ -3345,7 +3358,12 @@ func TestBan(t *testing.T) { return false, err } - if err = grpcAlice.SubmitTreeNonces(ctx, event.Id, signerSession.GetPublicKey(), nonces); err != nil { + if err = grpcAlice.SubmitTreeNonces( + ctx, + event.Id, + signerSession.GetPublicKey(), + nonces, + ); err != nil { return false, err } @@ -3478,7 +3496,11 @@ func TestBan(t *testing.T) { sweepTapTree := txscript.AssembleTaprootScriptTree(sweepTapLeaf) root := sweepTapTree.RootNode.TapHash() - if err := signerSession.Init(root.CloneBytes(), batchOutputAmount, vtxoTree); err != nil { + if err := signerSession.Init( + root.CloneBytes(), + batchOutputAmount, + vtxoTree, + ); err != nil { return false, err } @@ -3487,7 +3509,12 @@ func TestBan(t *testing.T) { return false, err } - if err = grpcAlice.SubmitTreeNonces(ctx, event.Id, signerSession.GetPublicKey(), nonces); err != nil { + if err = grpcAlice.SubmitTreeNonces( + ctx, + event.Id, + signerSession.GetPublicKey(), + nonces, + ); err != nil { return false, err } @@ -3626,7 +3653,11 @@ func TestBan(t *testing.T) { sweepTapTree := txscript.AssembleTaprootScriptTree(sweepTapLeaf) root := sweepTapTree.RootNode.TapHash() - if err := signerSession.Init(root.CloneBytes(), batchOutputAmount, vtxoTree); err != nil { + if err := signerSession.Init( + root.CloneBytes(), + batchOutputAmount, + vtxoTree, + ); err != nil { return false, err } @@ -3635,7 +3666,12 @@ func TestBan(t *testing.T) { return false, err } - if err = grpcAlice.SubmitTreeNonces(ctx, event.Id, signerSession.GetPublicKey(), nonces); err != nil { + if err = grpcAlice.SubmitTreeNonces( + ctx, + event.Id, + signerSession.GetPublicKey(), + nonces, + ); err != nil { return false, err } @@ -3836,7 +3872,11 @@ func TestBan(t *testing.T) { sweepTapTree := txscript.AssembleTaprootScriptTree(sweepTapLeaf) root := sweepTapTree.RootNode.TapHash() - if err := signerSession.Init(root.CloneBytes(), batchOutputAmount, vtxoTree); err != nil { + if err := signerSession.Init( + root.CloneBytes(), + batchOutputAmount, + vtxoTree, + ); err != nil { return false, err } @@ -3845,7 +3885,12 @@ func TestBan(t *testing.T) { return false, err } - if err = grpcAlice.SubmitTreeNonces(ctx, event.Id, signerSession.GetPublicKey(), nonces); err != nil { + if err = grpcAlice.SubmitTreeNonces( + ctx, + event.Id, + signerSession.GetPublicKey(), + nonces, + ); err != nil { return false, err } From c706f567887a9595c4e99b90437c6dded7858e83 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 12 Feb 2026 02:42:31 -0500 Subject: [PATCH 13/54] fix github action for linter version --- .github/workflows/unit.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit.yaml b/.github/workflows/unit.yaml index 36de73a00..937e706a9 100755 --- a/.github/workflows/unit.yaml +++ b/.github/workflows/unit.yaml @@ -41,7 +41,7 @@ jobs: - name: check linting uses: golangci/golangci-lint-action@v8 with: - version: v2.5.0 + version: v2.6.2 args: --timeout 5m - name: check code integrity From a88e89552da9a510a6f41a0d32c5a7cbc837c1b4 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 12 Feb 2026 02:52:00 -0500 Subject: [PATCH 14/54] github action linter to use latest golangci/golangci-lint-action@v8 --- .github/workflows/unit.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit.yaml b/.github/workflows/unit.yaml index 937e706a9..486577c6f 100755 --- a/.github/workflows/unit.yaml +++ b/.github/workflows/unit.yaml @@ -41,7 +41,7 @@ jobs: - name: check linting uses: golangci/golangci-lint-action@v8 with: - version: v2.6.2 + version: latest args: --timeout 5m - name: check code integrity From 2a737f65143da2c2b7c61a80d644f3e81b27aacb Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 12 Feb 2026 11:19:17 -0500 Subject: [PATCH 15/54] revert lint changes --- .github/workflows/unit.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit.yaml b/.github/workflows/unit.yaml index 486577c6f..36de73a00 100755 --- a/.github/workflows/unit.yaml +++ b/.github/workflows/unit.yaml @@ -41,7 +41,7 @@ jobs: - name: check linting uses: golangci/golangci-lint-action@v8 with: - version: latest + version: v2.5.0 args: --timeout 5m - name: check code integrity From bfa79c71833c3e84454c838d8f35c581f4a82ce7 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 12 Feb 2026 12:49:47 -0500 Subject: [PATCH 16/54] migration to ensure each vtxo has at least 1 marker, dust vtxos bulk swept --- internal/core/application/sweeper.go | 50 +++----- internal/core/domain/marker_repo.go | 4 - .../infrastructure/db/badger/marker_repo.go | 100 +++++++--------- .../infrastructure/db/postgres/marker_repo.go | 61 ---------- ...0260210100000_add_depth_and_markers.up.sql | 28 ++--- internal/infrastructure/db/service.go | 108 +++++++----------- internal/infrastructure/db/service_test.go | 46 +++++++- .../infrastructure/db/sqlite/marker_repo.go | 68 ----------- ...0260210000000_add_depth_and_markers.up.sql | 38 ++---- 9 files changed, 158 insertions(+), 345 deletions(-) diff --git a/internal/core/application/sweeper.go b/internal/core/application/sweeper.go index a32f212ec..b17d04baa 100644 --- a/internal/core/application/sweeper.go +++ b/internal/core/application/sweeper.go @@ -769,45 +769,33 @@ func (s *sweeper) createCheckpointSweepTask( return err } - // Collect all unique markers and vtxos without markers - sweptAt := time.Now().Unix() - markerStore := s.repoManager.Markers() + // Collect all unique markers from all VTXOs + // Every VTXO is guaranteed to have at least 1 marker after migration uniqueMarkers := make(map[string]struct{}) - var noMarkerVtxos []domain.Outpoint for _, v := range vtxos { - if len(v.MarkerIDs) > 0 { - for _, markerID := range v.MarkerIDs { - uniqueMarkers[markerID] = struct{}{} - } - } else { - noMarkerVtxos = append(noMarkerVtxos, v.Outpoint) + for _, markerID := range v.MarkerIDs { + uniqueMarkers[markerID] = struct{}{} } } - // Bulk sweep all markers at once - sweptCount := 0 - if len(uniqueMarkers) > 0 { - markerIDs := make([]string, 0, len(uniqueMarkers)) - for markerID := range uniqueMarkers { - markerIDs = append(markerIDs, markerID) - } - if err := markerStore.BulkSweepMarkers(ctx, markerIDs, sweptAt); err != nil { - log.WithError(err).Warn("failed to bulk sweep markers") - } else { - sweptCount = len(vtxos) - len(noMarkerVtxos) - log.Debugf("bulk swept %d markers affecting %d vtxos", len(markerIDs), sweptCount) - } + if len(uniqueMarkers) == 0 { + return nil } - // Sweep VTXOs without markers by creating unique dust markers - for _, outpoint := range noMarkerVtxos { - if err := markerStore.MarkDustVtxoSwept(ctx, outpoint, sweptAt); err != nil { - log.WithError(err).Warnf("failed to mark vtxo %s as swept", outpoint.String()) - continue - } - sweptCount++ + // Convert marker set to slice for bulk sweeping + markerIDs := make([]string, 0, len(uniqueMarkers)) + for markerID := range uniqueMarkers { + markerIDs = append(markerIDs, markerID) + } + + sweptAt := time.Now().Unix() + markerStore := s.repoManager.Markers() + if err := markerStore.BulkSweepMarkers(ctx, markerIDs, sweptAt); err != nil { + log.WithError(err).Warn("failed to bulk sweep markers") + return err } - log.Debugf("swept %d vtxos", sweptCount) + + log.Debugf("bulk swept %d markers affecting %d vtxos", len(markerIDs), len(vtxos)) return nil } } diff --git a/internal/core/domain/marker_repo.go b/internal/core/domain/marker_repo.go index c999b2113..085d8dabb 100644 --- a/internal/core/domain/marker_repo.go +++ b/internal/core/domain/marker_repo.go @@ -34,10 +34,6 @@ type MarkerRepository interface { // Returns the number of VTXOs that will now be considered swept SweepVtxosByMarker(ctx context.Context, markerID string) (int64, error) - // MarkDustVtxoSwept creates a unique dust marker for a vtxo and marks it as swept - // Used for dust vtxos that need to be marked swept immediately on creation - MarkDustVtxoSwept(ctx context.Context, outpoint Outpoint, sweptAt int64) error - // CreateRootMarkersForVtxos creates root markers for batch VTXOs and updates their marker references // in a single transaction. Each VTXO gets a marker with ID equal to its outpoint string. CreateRootMarkersForVtxos(ctx context.Context, vtxos []Vtxo) error diff --git a/internal/infrastructure/db/badger/marker_repo.go b/internal/infrastructure/db/badger/marker_repo.go index 68d073775..209fbf69d 100644 --- a/internal/infrastructure/db/badger/marker_repo.go +++ b/internal/infrastructure/db/badger/marker_repo.go @@ -244,12 +244,35 @@ func (r *markerRepository) SweepMarker(ctx context.Context, markerID string, swe time.Sleep(100 * time.Millisecond) err = r.sweptMarkerStore.Insert(markerID, dto) if err == nil || errors.Is(err, badgerhold.ErrKeyExists) { - return nil + break } } + if err != nil && !errors.Is(err, badgerhold.ErrKeyExists) { + return err + } + } else { + return err } - return err } + + // Update Swept field on all VTXOs that have this marker + // This keeps the stored Swept field in sync for query compatibility + var allDtos []vtxoDTO + if err := r.vtxoStore.Find(&allDtos, &badgerhold.Query{}); err != nil { + return nil // Non-fatal, swept_marker is already updated + } + + for _, vtxoDto := range allDtos { + for _, id := range vtxoDto.MarkerIDs { + if id == markerID && !vtxoDto.Swept { + vtxoDto.Swept = true + vtxoDto.UpdatedAt = time.Now().UnixMilli() + _ = r.vtxoStore.Update(vtxoDto.Outpoint.String(), vtxoDto) + break + } + } + } + return nil } @@ -427,7 +450,10 @@ func (r *markerRepository) GetVtxosByMarker( for _, dto := range dtos { for _, id := range dto.MarkerIDs { if id == markerID { - vtxos = append(vtxos, dto.Vtxo) + vtxo := dto.Vtxo + // Compute Swept status dynamically by checking if any marker is swept + vtxo.Swept = r.isAnyMarkerSwept(dto.MarkerIDs) + vtxos = append(vtxos, vtxo) break } } @@ -435,6 +461,18 @@ func (r *markerRepository) GetVtxosByMarker( return vtxos, nil } +// isAnyMarkerSwept checks if any of the given markers are in the swept_marker store +func (r *markerRepository) isAnyMarkerSwept(markerIDs []string) bool { + for _, markerID := range markerIDs { + var dto sweptMarkerDTO + err := r.sweptMarkerStore.Get(markerID, &dto) + if err == nil { + return true + } + } + return false +} + func (r *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID string) (int64, error) { // For badger, we need to: // 1. Mark the marker as swept @@ -517,62 +555,6 @@ func (r *markerRepository) CreateRootMarkersForVtxos( return nil } -func (r *markerRepository) MarkDustVtxoSwept( - ctx context.Context, - outpoint domain.Outpoint, - sweptAt int64, -) error { - // Create a unique dust marker for this vtxo - dustMarkerID := outpoint.String() + ":dust" - - // Get the vtxo to find its depth and current markers - var dto vtxoDTO - err := r.vtxoStore.Get(outpoint.String(), &dto) - if err != nil { - if err == badgerhold.ErrNotFound { - return fmt.Errorf("vtxo not found: %s", outpoint.String()) - } - return fmt.Errorf("failed to get vtxo: %w", err) - } - - // Create the dust marker - if err := r.AddMarker(ctx, domain.Marker{ - ID: dustMarkerID, - Depth: dto.Depth, - ParentMarkerIDs: dto.MarkerIDs, - }); err != nil { - return fmt.Errorf("failed to create dust marker: %w", err) - } - - // Insert into swept_marker - if err := r.SweepMarker(ctx, dustMarkerID, sweptAt); err != nil { - return fmt.Errorf("failed to insert swept marker: %w", err) - } - - // Update the vtxo's markers to include the dust marker and mark as swept - dto.MarkerIDs = append(dto.MarkerIDs, dustMarkerID) - dto.Swept = true - dto.UpdatedAt = time.Now().UnixMilli() - - err = r.vtxoStore.Update(outpoint.String(), dto) - if err != nil { - if errors.Is(err, badger.ErrConflict) { - for attempts := 1; attempts <= maxRetries; attempts++ { - time.Sleep(100 * time.Millisecond) - err = r.vtxoStore.Update(outpoint.String(), dto) - if err == nil { - break - } - } - } - if err != nil { - return fmt.Errorf("failed to update vtxo: %w", err) - } - } - - return nil -} - func (r *markerRepository) GetVtxosByDepthRange( ctx context.Context, minDepth, maxDepth uint32, diff --git a/internal/infrastructure/db/postgres/marker_repo.go b/internal/infrastructure/db/postgres/marker_repo.go index cccca8a4f..a88bd7e19 100644 --- a/internal/infrastructure/db/postgres/marker_repo.go +++ b/internal/infrastructure/db/postgres/marker_repo.go @@ -304,67 +304,6 @@ func (m *markerRepository) CreateRootMarkersForVtxos( return execTx(ctx, m.db, txBody) } -func (m *markerRepository) MarkDustVtxoSwept( - ctx context.Context, - outpoint domain.Outpoint, - sweptAt int64, -) error { - // Create a unique dust marker for this vtxo - dustMarkerID := outpoint.String() + ":dust" - - // First, get the vtxo to find its depth and current markers - vtxoRow, err := m.querier.SelectVtxo(ctx, queries.SelectVtxoParams{ - Txid: outpoint.Txid, - Vout: int32(outpoint.VOut), - }) - if err != nil { - return fmt.Errorf("failed to get vtxo: %w", err) - } - - // Create the dust marker - parentMarkers := parseMarkersJSONB(vtxoRow.VtxoVw.Markers) - parentMarkersJSON, err := json.Marshal(parentMarkers) - if err != nil { - return fmt.Errorf("failed to marshal parent markers: %w", err) - } - - if err := m.querier.UpsertMarker(ctx, queries.UpsertMarkerParams{ - ID: dustMarkerID, - Depth: vtxoRow.VtxoVw.Depth, - ParentMarkers: pqtype.NullRawMessage{ - RawMessage: parentMarkersJSON, - Valid: true, - }, - }); err != nil { - return fmt.Errorf("failed to create dust marker: %w", err) - } - - // Insert into swept_marker - if err := m.querier.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ - MarkerID: dustMarkerID, - SweptAt: sweptAt, - }); err != nil { - return fmt.Errorf("failed to insert swept marker: %w", err) - } - - // Update the vtxo's markers to include the dust marker - newMarkers := append(parentMarkers, dustMarkerID) - newMarkersJSON, err := json.Marshal(newMarkers) - if err != nil { - return fmt.Errorf("failed to marshal new markers: %w", err) - } - - if err := m.querier.UpdateVtxoMarkers(ctx, queries.UpdateVtxoMarkersParams{ - Markers: newMarkersJSON, - Txid: outpoint.Txid, - Vout: int32(outpoint.VOut), - }); err != nil { - return fmt.Errorf("failed to update vtxo markers: %w", err) - } - - return nil -} - func (m *markerRepository) GetVtxosByDepthRange( ctx context.Context, minDepth, maxDepth uint32, diff --git a/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql index 9ead5d349..f6b451ac9 100644 --- a/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql +++ b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql @@ -39,42 +39,28 @@ FROM intent LEFT OUTER JOIN vtxo_vw ON intent.id = vtxo_vw.intent_id; --- Backfill markers for existing VTXOs based on their depth +-- Backfill: Create a marker for every existing VTXO using its outpoint as marker ID +-- This ensures every VTXO has at least 1 marker INSERT INTO marker (id, depth, parent_markers) SELECT v.txid || ':' || v.vout, v.depth, '[]'::jsonb -FROM vtxo v -WHERE v.depth % 100 = 0; +FROM vtxo v; --- Assign markers array to VTXOs at boundary depths -UPDATE vtxo SET markers = jsonb_build_array(txid || ':' || vout) -WHERE depth % 100 = 0; +-- Assign the marker to every VTXO +UPDATE vtxo SET markers = jsonb_build_array(txid || ':' || vout); -- Migrate existing swept VTXOs to swept_marker table before dropping column --- For each swept VTXO, create a unique dust marker and insert into swept_marker -INSERT INTO marker (id, depth, parent_markers) -SELECT - v.txid || ':' || v.vout || ':dust', - v.depth, - COALESCE(v.markers, '[]'::jsonb) -FROM vtxo v -WHERE v.swept = true -ON CONFLICT (id) DO NOTHING; - +-- Insert the VTXO's marker into swept_marker INSERT INTO swept_marker (marker_id, swept_at) SELECT - v.txid || ':' || v.vout || ':dust', + v.txid || ':' || v.vout, EXTRACT(EPOCH FROM NOW())::BIGINT FROM vtxo v WHERE v.swept = true ON CONFLICT (marker_id) DO NOTHING; --- Update swept VTXOs to include the dust marker in their markers array -UPDATE vtxo SET markers = COALESCE(markers, '[]'::jsonb) || jsonb_build_array(txid || ':' || vout || ':dust') -WHERE swept = true; - -- Drop views before dropping the swept column (views depend on it via v.*) DROP VIEW IF EXISTS intent_with_inputs_vw; DROP VIEW IF EXISTS vtxo_vw; diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index 22200d25f..583b47499 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -578,26 +578,25 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) // Create marker if at boundary depth, or inherit ALL parent markers var markerIDs []string - if s.markerStore != nil { - if domain.IsAtMarkerBoundary(newDepth) { - // Create marker ID from the first output (the ark tx id + first vtxo vout) - newMarkerID := fmt.Sprintf("%s:marker:%d", txid, newDepth) - marker := domain.Marker{ - ID: newMarkerID, - Depth: newDepth, - ParentMarkerIDs: parentMarkerIDs, - } - if err := s.markerStore.AddMarker(ctx, marker); err != nil { - log.WithError(err).Warn("failed to create marker for chained vtxo") - // Continue without marker - non-fatal - } else { - log.Debugf("created marker %s at depth %d", newMarkerID, newDepth) - markerIDs = []string{newMarkerID} - } - } else if len(parentMarkerIDs) > 0 { - // Inherit ALL markers from parents at non-boundary depth - markerIDs = parentMarkerIDs + + if domain.IsAtMarkerBoundary(newDepth) { + // Create marker ID from the first output (the ark tx id + first vtxo vout) + newMarkerID := fmt.Sprintf("%s:marker:%d", txid, newDepth) + marker := domain.Marker{ + ID: newMarkerID, + Depth: newDepth, + ParentMarkerIDs: parentMarkerIDs, } + if err := s.markerStore.AddMarker(ctx, marker); err != nil { + log.WithError(err).Warn("failed to create marker for chained vtxo") + // Continue without marker - non-fatal + } else { + log.Debugf("created marker %s at depth %d", newMarkerID, newDepth) + markerIDs = []string{newMarkerID} + } + } else if len(parentMarkerIDs) > 0 { + // Inherit ALL markers from parents at non-boundary depth + markerIDs = parentMarkerIDs } newVtxos := make([]domain.Vtxo, 0, len(outs)) @@ -638,19 +637,17 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) } log.Debugf("added %d vtxos at depth %d", len(newVtxos), newDepth) - // Bob: do we need to handle dust differently? same question in sweepVtxosWithMarkers - // where we do it as well. - // Mark dust VTXOs as swept via marker + // Mark dust VTXOs as swept via their markers // Dust vtxos are below dust limit and can't be spent again in future offchain tx - // The only way to spend a swept vtxo is by collecting enough dust to cover the minSettlementVtxoAmount and then settle // Because sub-dust vtxos are using OP_RETURN output script, they can't be unilaterally exited - if s.markerStore != nil { - sweptAt := time.Now().Unix() + if len(dustVtxoOutpoints) > 0 { + dustMarkerIDs := make([]string, 0, len(dustVtxoOutpoints)) for _, outpoint := range dustVtxoOutpoints { - if err := s.markerStore.MarkDustVtxoSwept(ctx, outpoint, sweptAt); err != nil { - log.WithError(err). - Warnf("failed to mark dust vtxo %s as swept", outpoint.String()) - } + dustMarkerIDs = append(dustMarkerIDs, outpoint.String()) + } + sweptAt := time.Now().Unix() + if err := s.markerStore.BulkSweepMarkers(ctx, dustMarkerIDs, sweptAt); err != nil { + log.WithError(err).Warnf("failed to sweep %d dust vtxo markers", len(dustMarkerIDs)) } } } @@ -748,53 +745,32 @@ func (s *service) sweepVtxosWithMarkers( } // Collect all unique markers from all VTXOs + // Every VTXO is guaranteed to have at least 1 marker after migration uniqueMarkers := make(map[string]struct{}) - noMarkerVtxos := make([]domain.Outpoint, 0) - for _, vtxo := range vtxos { - if len(vtxo.MarkerIDs) > 0 { - // Collect all markers for this vtxo - for _, markerID := range vtxo.MarkerIDs { - uniqueMarkers[markerID] = struct{}{} - } - } else { - noMarkerVtxos = append(noMarkerVtxos, vtxo.Outpoint) + for _, markerID := range vtxo.MarkerIDs { + uniqueMarkers[markerID] = struct{}{} } } - var totalSwept int64 - sweptAt := time.Now().Unix() - - // Bulk sweep all markers at once - if len(uniqueMarkers) > 0 { - // Convert marker set to slice for bulk sweeping - markerIDs := make([]string, 0, len(uniqueMarkers)) - for markerID := range uniqueMarkers { - markerIDs = append(markerIDs, markerID) - } - - if err := s.markerStore.BulkSweepMarkers(ctx, markerIDs, sweptAt); err != nil { - log.WithError(err).Warn("failed to bulk sweep markers") - } else { - // Count VTXOs that have at least one marker (they're all swept now) - totalSwept = int64(len(vtxos) - len(noMarkerVtxos)) - log.Debugf("bulk swept %d markers affecting %d vtxos", len(markerIDs), totalSwept) - } + if len(uniqueMarkers) == 0 { + return 0 } - // Bob: I dont quite understand this part. If there are VTXOs without markers, does that mean they were not swept by the marker-based sweeping? Why do we need to sweep them with unique dust markers? Are these VTXOs that were missed by the marker-based sweeping, or are they a different category of VTXOs that require special handling? - // Bob: I think we cant get rid of this is we assume that every vtxo has >=1 marker. - // Bob: In the current implementation, we create a root marker for every batch VTXO at depth 0, but if there are any VTXOs that for some reason dont have markers (maybe they were created before we implemented marker-based sweeping), we need to sweep them as well. Since they dont have markers, we can create unique dust markers for each of them to mark them as swept. This way, we ensure that all VTXOs are accounted for in the sweeping process, even if they dont have markers. + // Convert marker set to slice for bulk sweeping + markerIDs := make([]string, 0, len(uniqueMarkers)) + for markerID := range uniqueMarkers { + markerIDs = append(markerIDs, markerID) + } - // Sweep VTXOs without markers by creating unique dust markers for each - for _, outpoint := range noMarkerVtxos { - if err := s.markerStore.MarkDustVtxoSwept(ctx, outpoint, sweptAt); err != nil { - log.WithError(err).Warnf("failed to sweep vtxo without marker: %s", outpoint.String()) - continue - } - totalSwept++ + sweptAt := time.Now().Unix() + if err := s.markerStore.BulkSweepMarkers(ctx, markerIDs, sweptAt); err != nil { + log.WithError(err).Warn("failed to bulk sweep markers") + return 0 } + totalSwept := int64(len(vtxos)) + log.Debugf("bulk swept %d markers affecting %d vtxos", len(markerIDs), totalSwept) return totalSwept } diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index 31f19e6a3..1b3102356 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -1226,8 +1226,18 @@ func testVtxoRepository(t *testing.T, svc ports.RepoManager) { // Mark the swept vtxo via markers (if marker store is available) if svc.Markers() != nil { + // Create a marker for the VTXO and sweep it + markerID := expiringVtxoToSweep.Outpoint.String() + err = svc.Markers().AddMarker(ctx, domain.Marker{ + ID: markerID, + Depth: 0, + }) + require.NoError(t, err) + err = svc.Markers(). + UpdateVtxoMarkers(ctx, expiringVtxoToSweep.Outpoint, []string{markerID}) + require.NoError(t, err) sweptAt := time.Now().Unix() - err = svc.Markers().MarkDustVtxoSwept(ctx, expiringVtxoToSweep.Outpoint, sweptAt) + err = svc.Markers().SweepMarker(ctx, markerID, sweptAt) require.NoError(t, err) } @@ -1284,10 +1294,23 @@ func testVtxoRepository(t *testing.T, svc ports.RepoManager) { // Mark first two vtxos as swept via markers (if marker store is available) if svc.Markers() != nil { + // Create markers for VTXOs and sweep them + marker1ID := recoverableVtxo1.Outpoint.String() + marker2ID := recoverableVtxo2.Outpoint.String() + err = svc.Markers().AddMarker(ctx, domain.Marker{ID: marker1ID, Depth: 0}) + require.NoError(t, err) + err = svc.Markers().AddMarker(ctx, domain.Marker{ID: marker2ID, Depth: 0}) + require.NoError(t, err) + err = svc.Markers(). + UpdateVtxoMarkers(ctx, recoverableVtxo1.Outpoint, []string{marker1ID}) + require.NoError(t, err) + err = svc.Markers(). + UpdateVtxoMarkers(ctx, recoverableVtxo2.Outpoint, []string{marker2ID}) + require.NoError(t, err) sweptAt := time.Now().Unix() - err = svc.Markers().MarkDustVtxoSwept(ctx, recoverableVtxo1.Outpoint, sweptAt) + err = svc.Markers().SweepMarker(ctx, marker1ID, sweptAt) require.NoError(t, err) - err = svc.Markers().MarkDustVtxoSwept(ctx, recoverableVtxo2.Outpoint, sweptAt) + err = svc.Markers().SweepMarker(ctx, marker2ID, sweptAt) require.NoError(t, err) } @@ -1787,11 +1810,22 @@ func testSweepVtxosByMarker(t *testing.T, svc ports.RepoManager) { require.NoError(t, err) } - // Mark vtxos[3] and vtxos[4] as swept via MarkDustVtxoSwept + // Mark vtxos[3] and vtxos[4] as swept via individual markers + // Create individual markers for these VTXOs and sweep them + marker3ID := vtxos[3].Outpoint.String() + marker4ID := vtxos[4].Outpoint.String() + err = svc.Markers().AddMarker(ctx, domain.Marker{ID: marker3ID, Depth: vtxos[3].Depth}) + require.NoError(t, err) + err = svc.Markers().AddMarker(ctx, domain.Marker{ID: marker4ID, Depth: vtxos[4].Depth}) + require.NoError(t, err) + err = svc.Markers().UpdateVtxoMarkers(ctx, vtxos[3].Outpoint, []string{markerID, marker3ID}) + require.NoError(t, err) + err = svc.Markers().UpdateVtxoMarkers(ctx, vtxos[4].Outpoint, []string{markerID, marker4ID}) + require.NoError(t, err) sweptAt := time.Now().Unix() - err = svc.Markers().MarkDustVtxoSwept(ctx, vtxos[3].Outpoint, sweptAt) + err = svc.Markers().SweepMarker(ctx, marker3ID, sweptAt) require.NoError(t, err) - err = svc.Markers().MarkDustVtxoSwept(ctx, vtxos[4].Outpoint, sweptAt) + err = svc.Markers().SweepMarker(ctx, marker4ID, sweptAt) require.NoError(t, err) // Verify initial state - vtxos[3] and vtxos[4] should be swept diff --git a/internal/infrastructure/db/sqlite/marker_repo.go b/internal/infrastructure/db/sqlite/marker_repo.go index 44dbd9e7b..aacbcab4b 100644 --- a/internal/infrastructure/db/sqlite/marker_repo.go +++ b/internal/infrastructure/db/sqlite/marker_repo.go @@ -312,74 +312,6 @@ func (m *markerRepository) CreateRootMarkersForVtxos( return execTx(ctx, m.db, txBody) } -func (m *markerRepository) MarkDustVtxoSwept( - ctx context.Context, - outpoint domain.Outpoint, - sweptAt int64, -) error { - // Create a unique dust marker for this vtxo - dustMarkerID := outpoint.String() + ":dust" - - // First, get the vtxo to find its depth and current markers - vtxoRow, err := m.querier.SelectVtxo(ctx, queries.SelectVtxoParams{ - Txid: outpoint.Txid, - Vout: int64(outpoint.VOut), - }) - if err != nil { - return fmt.Errorf("failed to get vtxo: %w", err) - } - - // Get current markers from the vtxo - var parentMarkers []string - if vtxoRow.VtxoVw.Markers.Valid && vtxoRow.VtxoVw.Markers.String != "" { - if err := json.Unmarshal( - []byte(vtxoRow.VtxoVw.Markers.String), - &parentMarkers, - ); err != nil { - parentMarkers = nil - } - } - - parentMarkersJSON, err := json.Marshal(parentMarkers) - if err != nil { - return fmt.Errorf("failed to marshal parent markers: %w", err) - } - - // Create the dust marker - if err := m.querier.UpsertMarker(ctx, queries.UpsertMarkerParams{ - ID: dustMarkerID, - Depth: vtxoRow.VtxoVw.Depth, - ParentMarkers: sql.NullString{String: string(parentMarkersJSON), Valid: true}, - }); err != nil { - return fmt.Errorf("failed to create dust marker: %w", err) - } - - // Insert into swept_marker - if err := m.querier.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ - MarkerID: dustMarkerID, - SweptAt: sweptAt, - }); err != nil { - return fmt.Errorf("failed to insert swept marker: %w", err) - } - - // Update the vtxo's markers to include the dust marker - newMarkers := append(parentMarkers, dustMarkerID) - newMarkersJSON, err := json.Marshal(newMarkers) - if err != nil { - return fmt.Errorf("failed to marshal new markers: %w", err) - } - - if err := m.querier.UpdateVtxoMarkers(ctx, queries.UpdateVtxoMarkersParams{ - Markers: sql.NullString{String: string(newMarkersJSON), Valid: true}, - Txid: outpoint.Txid, - Vout: int64(outpoint.VOut), - }); err != nil { - return fmt.Errorf("failed to update vtxo markers: %w", err) - } - - return nil -} - func (m *markerRepository) GetVtxosByDepthRange( ctx context.Context, minDepth, maxDepth uint32, diff --git a/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql index a3c4d07cb..0c4a457cf 100644 --- a/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql +++ b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql @@ -40,47 +40,27 @@ FROM intent LEFT OUTER JOIN vtxo_vw ON intent.id = vtxo_vw.intent_id; --- Backfill markers for existing VTXOs based on their depth --- VTXOs at depth 0, 100, 200, ... get their own markers - --- First, create markers for all existing VTXOs at marker boundary depths (depth % 100 == 0) +-- Backfill: Create a marker for every existing VTXO using its outpoint as marker ID +-- This ensures every VTXO has at least 1 marker INSERT INTO marker (id, depth, parent_markers) SELECT - v.txid || ':' || v.vout, -- Use VTXO outpoint as marker ID + v.txid || ':' || v.vout, v.depth, - '[]' -- Empty parent markers for initial backfill -FROM vtxo v -WHERE v.depth % 100 = 0; + '[]' +FROM vtxo v; --- Assign markers array to VTXOs at boundary depths -UPDATE vtxo SET markers = '["' || txid || ':' || vout || '"]' -WHERE depth % 100 = 0; +-- Assign the marker to every VTXO +UPDATE vtxo SET markers = '["' || txid || ':' || vout || '"]'; -- Migrate existing swept VTXOs to swept_marker table before dropping column --- For each swept VTXO, create a unique dust marker and insert into swept_marker -INSERT OR IGNORE INTO marker (id, depth, parent_markers) -SELECT - v.txid || ':' || v.vout || ':dust', - v.depth, - COALESCE(v.markers, '[]') -FROM vtxo v -WHERE v.swept = 1; - +-- Insert the VTXO's marker into swept_marker INSERT OR IGNORE INTO swept_marker (marker_id, swept_at) SELECT - v.txid || ':' || v.vout || ':dust', + v.txid || ':' || v.vout, strftime('%s', 'now') FROM vtxo v WHERE v.swept = 1; --- Update swept VTXOs to include the dust marker in their markers array -UPDATE vtxo SET markers = - CASE - WHEN markers IS NULL OR markers = '' THEN '["' || txid || ':' || vout || ':dust"]' - ELSE substr(markers, 1, length(markers)-1) || ',"' || txid || ':' || vout || ':dust"]' - END -WHERE swept = 1; - -- SQLite doesn't support DROP COLUMN easily, so we recreate the table -- Create new vtxo table without swept column CREATE TABLE vtxo_new ( From 2fa768902b2ae9c50ba548644a6d5db227252c48 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 12 Feb 2026 15:25:24 -0500 Subject: [PATCH 17/54] Tests --- internal/core/application/indexer_test.go | 865 ++++++++++++++ internal/core/application/service_test.go | 415 +++++++ internal/core/application/sweeper_test.go | 1242 ++++++++++++++++++++ internal/infrastructure/db/service_test.go | 627 ++++++++++ 4 files changed, 3149 insertions(+) create mode 100644 internal/core/application/indexer_test.go create mode 100644 internal/core/application/sweeper_test.go diff --git a/internal/core/application/indexer_test.go b/internal/core/application/indexer_test.go new file mode 100644 index 000000000..4b1b3bc80 --- /dev/null +++ b/internal/core/application/indexer_test.go @@ -0,0 +1,865 @@ +package application + +import ( + "context" + "fmt" + "testing" + + "github.com/arkade-os/arkd/internal/core/domain" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// Mock implementations for indexer tests + +type mockVtxoRepoForIndexer struct { + mock.Mock +} + +func (m *mockVtxoRepoForIndexer) GetVtxos( + ctx context.Context, + outpoints []domain.Outpoint, +) ([]domain.Vtxo, error) { + args := m.Called(ctx, outpoints) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).([]domain.Vtxo), args.Error(1) +} + +// Stub implementations for unused VtxoRepository methods +func (m *mockVtxoRepoForIndexer) AddVtxos(ctx context.Context, vtxos []domain.Vtxo) error { + return nil +} + +func (m *mockVtxoRepoForIndexer) SettleVtxos( + ctx context.Context, + spentVtxos map[domain.Outpoint]string, + commitmentTxid string, +) error { + return nil +} + +func (m *mockVtxoRepoForIndexer) SpendVtxos( + ctx context.Context, + spentVtxos map[domain.Outpoint]string, + arkTxid string, +) error { + return nil +} + +func (m *mockVtxoRepoForIndexer) UnrollVtxos( + ctx context.Context, + outpoints []domain.Outpoint, +) error { + return nil +} + +func (m *mockVtxoRepoForIndexer) GetAllNonUnrolledVtxos( + ctx context.Context, + pubkey string, +) ([]domain.Vtxo, []domain.Vtxo, error) { + return nil, nil, nil +} + +func (m *mockVtxoRepoForIndexer) GetAllSweepableUnrolledVtxos( + ctx context.Context, +) ([]domain.Vtxo, error) { + return nil, nil +} +func (m *mockVtxoRepoForIndexer) GetAllVtxos(ctx context.Context) ([]domain.Vtxo, error) { + return nil, nil +} + +func (m *mockVtxoRepoForIndexer) GetAllVtxosWithPubKeys( + ctx context.Context, + pubkeys []string, + after, before int64, +) ([]domain.Vtxo, error) { + return nil, nil +} + +func (m *mockVtxoRepoForIndexer) GetExpiringLiquidity( + ctx context.Context, + after, before int64, +) (uint64, error) { + return 0, nil +} +func (m *mockVtxoRepoForIndexer) GetRecoverableLiquidity(ctx context.Context) (uint64, error) { + return 0, nil +} + +func (m *mockVtxoRepoForIndexer) UpdateVtxosExpiration( + ctx context.Context, + outpoints []domain.Outpoint, + expiresAt int64, +) error { + return nil +} + +func (m *mockVtxoRepoForIndexer) GetLeafVtxosForBatch( + ctx context.Context, + txid string, +) ([]domain.Vtxo, error) { + return nil, nil +} + +func (m *mockVtxoRepoForIndexer) GetSweepableVtxosByCommitmentTxid( + ctx context.Context, + commitmentTxid string, +) ([]domain.Outpoint, error) { + return nil, nil +} + +func (m *mockVtxoRepoForIndexer) GetAllChildrenVtxos( + ctx context.Context, + txid string, +) ([]domain.Outpoint, error) { + return nil, nil +} + +func (m *mockVtxoRepoForIndexer) GetVtxoPubKeysByCommitmentTxid( + ctx context.Context, + commitmentTxid string, + withMinimumAmount uint64, +) ([]string, error) { + return nil, nil +} + +func (m *mockVtxoRepoForIndexer) GetPendingSpentVtxosWithPubKeys( + ctx context.Context, + pubkeys []string, + after, before int64, +) ([]domain.Vtxo, error) { + return nil, nil +} + +func (m *mockVtxoRepoForIndexer) GetPendingSpentVtxosWithOutpoints( + ctx context.Context, + outpoints []domain.Outpoint, +) ([]domain.Vtxo, error) { + return nil, nil +} +func (m *mockVtxoRepoForIndexer) Close() {} + +type mockMarkerRepoForIndexer struct { + mock.Mock +} + +func (m *mockMarkerRepoForIndexer) GetMarker( + ctx context.Context, + id string, +) (*domain.Marker, error) { + args := m.Called(ctx, id) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*domain.Marker), args.Error(1) +} + +func (m *mockMarkerRepoForIndexer) GetVtxoChainByMarkers( + ctx context.Context, + markerIDs []string, +) ([]domain.Vtxo, error) { + args := m.Called(ctx, markerIDs) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).([]domain.Vtxo), args.Error(1) +} + +// Stub implementations for unused MarkerRepository methods +func (m *mockMarkerRepoForIndexer) AddMarker(ctx context.Context, marker domain.Marker) error { + return nil +} + +func (m *mockMarkerRepoForIndexer) GetMarkersByDepth( + ctx context.Context, + depth uint32, +) ([]domain.Marker, error) { + return nil, nil +} + +func (m *mockMarkerRepoForIndexer) GetMarkersByDepthRange( + ctx context.Context, + minDepth, maxDepth uint32, +) ([]domain.Marker, error) { + return nil, nil +} + +func (m *mockMarkerRepoForIndexer) GetMarkersByIds( + ctx context.Context, + ids []string, +) ([]domain.Marker, error) { + return nil, nil +} + +func (m *mockMarkerRepoForIndexer) SweepMarker( + ctx context.Context, + markerID string, + sweptAt int64, +) error { + return nil +} + +func (m *mockMarkerRepoForIndexer) BulkSweepMarkers( + ctx context.Context, + markerIDs []string, + sweptAt int64, +) error { + return nil +} + +func (m *mockMarkerRepoForIndexer) SweepMarkerWithDescendants( + ctx context.Context, + markerID string, + sweptAt int64, +) (int64, error) { + return 0, nil +} + +func (m *mockMarkerRepoForIndexer) IsMarkerSwept( + ctx context.Context, + markerID string, +) (bool, error) { + return false, nil +} + +func (m *mockMarkerRepoForIndexer) GetSweptMarkers( + ctx context.Context, + markerIDs []string, +) ([]domain.SweptMarker, error) { + return nil, nil +} + +func (m *mockMarkerRepoForIndexer) UpdateVtxoMarkers( + ctx context.Context, + outpoint domain.Outpoint, + markerIDs []string, +) error { + return nil +} + +func (m *mockMarkerRepoForIndexer) GetVtxosByMarker( + ctx context.Context, + markerID string, +) ([]domain.Vtxo, error) { + return nil, nil +} + +func (m *mockMarkerRepoForIndexer) SweepVtxosByMarker( + ctx context.Context, + markerID string, +) (int64, error) { + return 0, nil +} + +func (m *mockMarkerRepoForIndexer) CreateRootMarkersForVtxos( + ctx context.Context, + vtxos []domain.Vtxo, +) error { + return nil +} + +func (m *mockMarkerRepoForIndexer) GetVtxosByDepthRange( + ctx context.Context, + minDepth, maxDepth uint32, +) ([]domain.Vtxo, error) { + return nil, nil +} + +func (m *mockMarkerRepoForIndexer) GetVtxosByArkTxid( + ctx context.Context, + arkTxid string, +) ([]domain.Vtxo, error) { + return nil, nil +} +func (m *mockMarkerRepoForIndexer) Close() {} + +type mockRepoManagerForIndexer struct { + vtxos *mockVtxoRepoForIndexer + markers *mockMarkerRepoForIndexer +} + +func (m *mockRepoManagerForIndexer) Events() domain.EventRepository { return nil } +func (m *mockRepoManagerForIndexer) Rounds() domain.RoundRepository { return nil } +func (m *mockRepoManagerForIndexer) Vtxos() domain.VtxoRepository { return m.vtxos } +func (m *mockRepoManagerForIndexer) Markers() domain.MarkerRepository { + // Must explicitly return nil to avoid Go's nil interface issue + // where a nil concrete type wrapped in an interface != nil + if m.markers == nil { + return nil + } + return m.markers +} +func (m *mockRepoManagerForIndexer) ScheduledSession() domain.ScheduledSessionRepo { return nil } +func (m *mockRepoManagerForIndexer) OffchainTxs() domain.OffchainTxRepository { return nil } +func (m *mockRepoManagerForIndexer) Convictions() domain.ConvictionRepository { return nil } +func (m *mockRepoManagerForIndexer) Fees() domain.FeeRepository { return nil } +func (m *mockRepoManagerForIndexer) Close() {} + +// TestPrefetchVtxosByMarkers_BuildsCacheFromMarkerChain verifies that prefetchVtxosByMarkers +// correctly traverses the marker hierarchy (following ParentMarkerIDs) and bulk fetches +// all VTXOs associated with those markers into a cache map. +func TestPrefetchVtxosByMarkers_BuildsCacheFromMarkerChain(t *testing.T) { + vtxoRepo := &mockVtxoRepoForIndexer{} + markerRepo := &mockMarkerRepoForIndexer{} + repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} + + indexer := &indexerService{repoManager: repoManager} + + ctx := context.Background() + startKey := Outpoint{Txid: "start-vtxo", VOut: 0} + + // Starting VTXO with markers at depth 200 + startVtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "start-vtxo", VOut: 0}, + MarkerIDs: []string{"marker-200"}, + Depth: 200, + } + + // Marker chain: marker-200 -> marker-100 -> marker-0 (root) + marker200 := &domain.Marker{ + ID: "marker-200", + Depth: 200, + ParentMarkerIDs: []string{"marker-100"}, + } + marker100 := &domain.Marker{ID: "marker-100", Depth: 100, ParentMarkerIDs: []string{"marker-0"}} + marker0 := &domain.Marker{ID: "marker-0", Depth: 0, ParentMarkerIDs: []string{}} + + // VTXOs associated with all markers in the chain + chainVtxos := []domain.Vtxo{ + {Outpoint: domain.Outpoint{Txid: "vtxo-a", VOut: 0}, Depth: 50}, + {Outpoint: domain.Outpoint{Txid: "vtxo-b", VOut: 0}, Depth: 100}, + {Outpoint: domain.Outpoint{Txid: "vtxo-c", VOut: 0}, Depth: 150}, + {Outpoint: domain.Outpoint{Txid: "vtxo-d", VOut: 0}, Depth: 200}, + } + + // Setup expectations + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{startKey}). + Return([]domain.Vtxo{startVtxo}, nil) + + markerRepo.On("GetMarker", ctx, "marker-200").Return(marker200, nil) + markerRepo.On("GetMarker", ctx, "marker-100").Return(marker100, nil) + markerRepo.On("GetMarker", ctx, "marker-0").Return(marker0, nil) + + // Expect bulk fetch with all markers in the chain + markerRepo.On("GetVtxoChainByMarkers", ctx, mock.MatchedBy(func(ids []string) bool { + // Should contain marker-200, marker-100, marker-0 + idSet := make(map[string]bool) + for _, id := range ids { + idSet[id] = true + } + return len(ids) == 3 && idSet["marker-200"] && idSet["marker-100"] && idSet["marker-0"] + })).Return(chainVtxos, nil) + + // Execute + cache := indexer.prefetchVtxosByMarkers(ctx, startKey) + + // Verify cache contains all VTXOs plus the start VTXO + require.Len(t, cache, 5) // 4 chain VTXOs + 1 start VTXO + require.Contains(t, cache, "start-vtxo:0") + require.Contains(t, cache, "vtxo-a:0") + require.Contains(t, cache, "vtxo-b:0") + require.Contains(t, cache, "vtxo-c:0") + require.Contains(t, cache, "vtxo-d:0") +} + +// TestPrefetchVtxosByMarkers_EmptyMarkersReturnsStartVtxoOnly verifies that when the +// starting VTXO has no markers, the cache only contains the starting VTXO itself. +func TestPrefetchVtxosByMarkers_EmptyMarkersReturnsStartVtxoOnly(t *testing.T) { + vtxoRepo := &mockVtxoRepoForIndexer{} + markerRepo := &mockMarkerRepoForIndexer{} + repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} + + indexer := &indexerService{repoManager: repoManager} + + ctx := context.Background() + startKey := Outpoint{Txid: "vtxo-no-markers", VOut: 0} + + // VTXO with no markers + startVtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "vtxo-no-markers", VOut: 0}, + MarkerIDs: []string{}, // Empty markers + Depth: 0, + } + + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{startKey}). + Return([]domain.Vtxo{startVtxo}, nil) + + cache := indexer.prefetchVtxosByMarkers(ctx, startKey) + + // Cache should only contain the start VTXO + require.Len(t, cache, 1) + require.Contains(t, cache, "vtxo-no-markers:0") + + // Marker repo should not be called for chain traversal + markerRepo.AssertNotCalled(t, "GetMarker", mock.Anything, mock.Anything) + markerRepo.AssertNotCalled(t, "GetVtxoChainByMarkers", mock.Anything, mock.Anything) +} + +// TestPrefetchVtxosByMarkers_NilMarkerRepoReturnsEmptyCache verifies that when the +// marker repository is nil (not configured), an empty cache is returned gracefully. +func TestPrefetchVtxosByMarkers_NilMarkerRepoReturnsEmptyCache(t *testing.T) { + vtxoRepo := &mockVtxoRepoForIndexer{} + repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: nil} + + indexer := &indexerService{repoManager: repoManager} + + ctx := context.Background() + startKey := Outpoint{Txid: "vtxo-any", VOut: 0} + + cache := indexer.prefetchVtxosByMarkers(ctx, startKey) + + // Cache should be empty when marker repo is nil + require.Empty(t, cache) + + // VTXO repo should not be called + vtxoRepo.AssertNotCalled(t, "GetVtxos", mock.Anything, mock.Anything) +} + +// TestGetVtxosFromCacheOrDB_CacheHitAvoidsDBCall verifies that when all requested +// outpoints are in the cache, no database call is made. +func TestGetVtxosFromCacheOrDB_CacheHitAvoidsDBCall(t *testing.T) { + vtxoRepo := &mockVtxoRepoForIndexer{} + markerRepo := &mockMarkerRepoForIndexer{} + repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} + + indexer := &indexerService{repoManager: repoManager} + + ctx := context.Background() + + // Pre-populated cache + cache := map[string]domain.Vtxo{ + "cached-vtxo-1:0": { + Outpoint: domain.Outpoint{Txid: "cached-vtxo-1", VOut: 0}, + Amount: 1000, + }, + "cached-vtxo-2:0": { + Outpoint: domain.Outpoint{Txid: "cached-vtxo-2", VOut: 0}, + Amount: 2000, + }, + } + + outpoints := []domain.Outpoint{ + {Txid: "cached-vtxo-1", VOut: 0}, + {Txid: "cached-vtxo-2", VOut: 0}, + } + + result, err := indexer.getVtxosFromCacheOrDB(ctx, outpoints, cache) + + require.NoError(t, err) + require.Len(t, result, 2) + + // Verify GetVtxos was never called (all cache hits) + vtxoRepo.AssertNotCalled(t, "GetVtxos", mock.Anything, mock.Anything) +} + +// TestGetVtxosFromCacheOrDB_CacheMissTriggersDBCall verifies that when outpoints +// are not in the cache, a database call is made for the missing ones only. +func TestGetVtxosFromCacheOrDB_CacheMissTriggersDBCall(t *testing.T) { + vtxoRepo := &mockVtxoRepoForIndexer{} + markerRepo := &mockMarkerRepoForIndexer{} + repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} + + indexer := &indexerService{repoManager: repoManager} + + ctx := context.Background() + + // Cache with one VTXO + cache := map[string]domain.Vtxo{ + "cached-vtxo:0": {Outpoint: domain.Outpoint{Txid: "cached-vtxo", VOut: 0}, Amount: 1000}, + } + + // Request both cached and uncached + outpoints := []domain.Outpoint{ + {Txid: "cached-vtxo", VOut: 0}, + {Txid: "uncached-vtxo", VOut: 0}, + } + + // DB should be called only for uncached outpoint + dbVtxo := domain.Vtxo{Outpoint: domain.Outpoint{Txid: "uncached-vtxo", VOut: 0}, Amount: 3000} + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: "uncached-vtxo", VOut: 0}}). + Return([]domain.Vtxo{dbVtxo}, nil) + + result, err := indexer.getVtxosFromCacheOrDB(ctx, outpoints, cache) + + require.NoError(t, err) + require.Len(t, result, 2) + + // Verify the uncached VTXO was added to cache + require.Contains(t, cache, "uncached-vtxo:0") + require.Equal(t, uint64(3000), cache["uncached-vtxo:0"].Amount) + + vtxoRepo.AssertExpectations(t) +} + +// TestGetVtxosFromCacheOrDB_AllCacheMiss verifies behavior when cache is empty +// and all outpoints must be fetched from the database. +func TestGetVtxosFromCacheOrDB_AllCacheMiss(t *testing.T) { + vtxoRepo := &mockVtxoRepoForIndexer{} + markerRepo := &mockMarkerRepoForIndexer{} + repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} + + indexer := &indexerService{repoManager: repoManager} + + ctx := context.Background() + + // Empty cache + cache := make(map[string]domain.Vtxo) + + outpoints := []domain.Outpoint{ + {Txid: "vtxo-1", VOut: 0}, + {Txid: "vtxo-2", VOut: 0}, + {Txid: "vtxo-3", VOut: 0}, + } + + dbVtxos := []domain.Vtxo{ + {Outpoint: domain.Outpoint{Txid: "vtxo-1", VOut: 0}, Amount: 100}, + {Outpoint: domain.Outpoint{Txid: "vtxo-2", VOut: 0}, Amount: 200}, + {Outpoint: domain.Outpoint{Txid: "vtxo-3", VOut: 0}, Amount: 300}, + } + + vtxoRepo.On("GetVtxos", ctx, outpoints).Return(dbVtxos, nil) + + result, err := indexer.getVtxosFromCacheOrDB(ctx, outpoints, cache) + + require.NoError(t, err) + require.Len(t, result, 3) + + // All VTXOs should now be in cache + require.Len(t, cache, 3) + require.Contains(t, cache, "vtxo-1:0") + require.Contains(t, cache, "vtxo-2:0") + require.Contains(t, cache, "vtxo-3:0") +} + +// TestGetVtxosFromCacheOrDB_DBErrorPropagated verifies that database errors +// are properly propagated to the caller. +func TestGetVtxosFromCacheOrDB_DBErrorPropagated(t *testing.T) { + vtxoRepo := &mockVtxoRepoForIndexer{} + markerRepo := &mockMarkerRepoForIndexer{} + repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} + + indexer := &indexerService{repoManager: repoManager} + + ctx := context.Background() + cache := make(map[string]domain.Vtxo) + + outpoints := []domain.Outpoint{{Txid: "vtxo-err", VOut: 0}} + + vtxoRepo.On("GetVtxos", ctx, outpoints). + Return(nil, fmt.Errorf("vtxo not found")) + + result, err := indexer.getVtxosFromCacheOrDB(ctx, outpoints, cache) + + require.Error(t, err) + require.Nil(t, result) +} + +// TestPrefetchVtxosByMarkers_HandlesMultipleParentMarkers verifies that the BFS +// traversal correctly handles VTXOs with multiple parent markers (diamond pattern +// in the marker DAG). +func TestPrefetchVtxosByMarkers_HandlesMultipleParentMarkers(t *testing.T) { + vtxoRepo := &mockVtxoRepoForIndexer{} + markerRepo := &mockMarkerRepoForIndexer{} + repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} + + indexer := &indexerService{repoManager: repoManager} + + ctx := context.Background() + startKey := Outpoint{Txid: "diamond-vtxo", VOut: 0} + + // VTXO with multiple markers (diamond pattern) + // marker-C has two parents: marker-A and marker-B, both pointing to marker-root + startVtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "diamond-vtxo", VOut: 0}, + MarkerIDs: []string{"marker-C"}, + Depth: 200, + } + + markerC := &domain.Marker{ + ID: "marker-C", + Depth: 200, + ParentMarkerIDs: []string{"marker-A", "marker-B"}, + } + markerA := &domain.Marker{ID: "marker-A", Depth: 100, ParentMarkerIDs: []string{"marker-root"}} + markerB := &domain.Marker{ID: "marker-B", Depth: 100, ParentMarkerIDs: []string{"marker-root"}} + markerRoot := &domain.Marker{ID: "marker-root", Depth: 0, ParentMarkerIDs: []string{}} + + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{startKey}). + Return([]domain.Vtxo{startVtxo}, nil) + + markerRepo.On("GetMarker", ctx, "marker-C").Return(markerC, nil) + markerRepo.On("GetMarker", ctx, "marker-A").Return(markerA, nil) + markerRepo.On("GetMarker", ctx, "marker-B").Return(markerB, nil) + markerRepo.On("GetMarker", ctx, "marker-root").Return(markerRoot, nil) + + chainVtxos := []domain.Vtxo{ + {Outpoint: domain.Outpoint{Txid: "vtxo-from-chain", VOut: 0}}, + } + + // Should collect all 4 markers despite diamond pattern + markerRepo.On("GetVtxoChainByMarkers", ctx, mock.MatchedBy(func(ids []string) bool { + idSet := make(map[string]bool) + for _, id := range ids { + idSet[id] = true + } + // Must have all 4 markers, no duplicates + return len(ids) == 4 && + idSet["marker-C"] && + idSet["marker-A"] && + idSet["marker-B"] && + idSet["marker-root"] + })).Return(chainVtxos, nil) + + cache := indexer.prefetchVtxosByMarkers(ctx, startKey) + + // Verify we got the starting VTXO plus chain VTXOs + require.Contains(t, cache, "diamond-vtxo:0") + require.Contains(t, cache, "vtxo-from-chain:0") +} + +// TestPrefetchVtxosByMarkers_GetVtxosError verifies that when GetVtxos fails +// to retrieve the starting VTXO, prefetchVtxosByMarkers returns an empty cache +// gracefully without panicking. +func TestPrefetchVtxosByMarkers_GetVtxosError(t *testing.T) { + vtxoRepo := &mockVtxoRepoForIndexer{} + markerRepo := &mockMarkerRepoForIndexer{} + repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} + + indexer := &indexerService{repoManager: repoManager} + + ctx := context.Background() + startKey := Outpoint{Txid: "vtxo-error", VOut: 0} + + // Simulate DB error when fetching start VTXO + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{startKey}). + Return(nil, fmt.Errorf("database connection lost")) + + cache := indexer.prefetchVtxosByMarkers(ctx, startKey) + + // Cache should be empty on error + require.Empty(t, cache) + + // Marker repo should not be called + markerRepo.AssertNotCalled(t, "GetMarker", mock.Anything, mock.Anything) + markerRepo.AssertNotCalled(t, "GetVtxoChainByMarkers", mock.Anything, mock.Anything) +} + +// TestPrefetchVtxosByMarkers_GetMarkerError verifies that when GetMarker +// fails for a marker in the BFS traversal, the function still returns +// partial results from successfully fetched markers. +func TestPrefetchVtxosByMarkers_GetMarkerError(t *testing.T) { + vtxoRepo := &mockVtxoRepoForIndexer{} + markerRepo := &mockMarkerRepoForIndexer{} + repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} + + indexer := &indexerService{repoManager: repoManager} + + ctx := context.Background() + startKey := Outpoint{Txid: "vtxo-partial", VOut: 0} + + // Starting VTXO with a marker at depth 200 + startVtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "vtxo-partial", VOut: 0}, + MarkerIDs: []string{"marker-200"}, + Depth: 200, + } + + // marker-200 has parent marker-100, but marker-100 lookup will fail + marker200 := &domain.Marker{ + ID: "marker-200", + Depth: 200, + ParentMarkerIDs: []string{"marker-100"}, + } + + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{startKey}). + Return([]domain.Vtxo{startVtxo}, nil) + + markerRepo.On("GetMarker", ctx, "marker-200").Return(marker200, nil) + // marker-100 lookup fails + markerRepo.On("GetMarker", ctx, "marker-100"). + Return(nil, fmt.Errorf("marker not found")) + + // GetVtxoChainByMarkers should still be called with the markers we did collect + chainVtxos := []domain.Vtxo{ + {Outpoint: domain.Outpoint{Txid: "vtxo-from-200", VOut: 0}, Depth: 180}, + } + markerRepo.On("GetVtxoChainByMarkers", ctx, mock.MatchedBy(func(ids []string) bool { + // Should have marker-200 and marker-100 (both were added to markerIDs) + idSet := make(map[string]bool) + for _, id := range ids { + idSet[id] = true + } + return len(ids) == 2 && idSet["marker-200"] && idSet["marker-100"] + })).Return(chainVtxos, nil) + + cache := indexer.prefetchVtxosByMarkers(ctx, startKey) + + // Should have the start VTXO plus chain VTXOs from marker-200 + require.Contains(t, cache, "vtxo-partial:0") + require.Contains(t, cache, "vtxo-from-200:0") +} + +// TestPrefetchVtxosByMarkers_GetVtxoChainByMarkersError verifies that when +// the bulk fetch of VTXOs by markers fails, the cache still contains at +// least the starting VTXO. +func TestPrefetchVtxosByMarkers_GetVtxoChainByMarkersError(t *testing.T) { + vtxoRepo := &mockVtxoRepoForIndexer{} + markerRepo := &mockMarkerRepoForIndexer{} + repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} + + indexer := &indexerService{repoManager: repoManager} + + ctx := context.Background() + startKey := Outpoint{Txid: "vtxo-bulk-err", VOut: 0} + + startVtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "vtxo-bulk-err", VOut: 0}, + MarkerIDs: []string{"marker-100"}, + Depth: 100, + } + + marker100 := &domain.Marker{ + ID: "marker-100", + Depth: 100, + ParentMarkerIDs: []string{}, + } + + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{startKey}). + Return([]domain.Vtxo{startVtxo}, nil) + + markerRepo.On("GetMarker", ctx, "marker-100").Return(marker100, nil) + + // Bulk fetch fails + markerRepo.On("GetVtxoChainByMarkers", ctx, mock.Anything). + Return(nil, fmt.Errorf("bulk fetch failed")) + + cache := indexer.prefetchVtxosByMarkers(ctx, startKey) + + // Cache should still contain the start VTXO + require.Len(t, cache, 1) + require.Contains(t, cache, "vtxo-bulk-err:0") +} + +// TestPrefetchVtxosByMarkers_DeepChainManyMarkers verifies that BFS traversal +// correctly handles a deep chain with 5+ markers (depth 500), collecting all +// markers without off-by-one errors or missed parents. +func TestPrefetchVtxosByMarkers_DeepChainManyMarkers(t *testing.T) { + vtxoRepo := &mockVtxoRepoForIndexer{} + markerRepo := &mockMarkerRepoForIndexer{} + repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} + + indexer := &indexerService{repoManager: repoManager} + + ctx := context.Background() + startKey := Outpoint{Txid: "deep-vtxo", VOut: 0} + + // VTXO at depth 500 with marker at depth 500 + startVtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "deep-vtxo", VOut: 0}, + MarkerIDs: []string{"marker-500"}, + Depth: 500, + } + + // Linear marker chain: 500 -> 400 -> 300 -> 200 -> 100 -> 0 + marker500 := &domain.Marker{ + ID: "marker-500", + Depth: 500, + ParentMarkerIDs: []string{"marker-400"}, + } + marker400 := &domain.Marker{ + ID: "marker-400", + Depth: 400, + ParentMarkerIDs: []string{"marker-300"}, + } + marker300 := &domain.Marker{ + ID: "marker-300", + Depth: 300, + ParentMarkerIDs: []string{"marker-200"}, + } + marker200 := &domain.Marker{ + ID: "marker-200", + Depth: 200, + ParentMarkerIDs: []string{"marker-100"}, + } + marker100 := &domain.Marker{ID: "marker-100", Depth: 100, ParentMarkerIDs: []string{"marker-0"}} + marker0 := &domain.Marker{ID: "marker-0", Depth: 0, ParentMarkerIDs: []string{}} + + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{startKey}). + Return([]domain.Vtxo{startVtxo}, nil) + + markerRepo.On("GetMarker", ctx, "marker-500").Return(marker500, nil) + markerRepo.On("GetMarker", ctx, "marker-400").Return(marker400, nil) + markerRepo.On("GetMarker", ctx, "marker-300").Return(marker300, nil) + markerRepo.On("GetMarker", ctx, "marker-200").Return(marker200, nil) + markerRepo.On("GetMarker", ctx, "marker-100").Return(marker100, nil) + markerRepo.On("GetMarker", ctx, "marker-0").Return(marker0, nil) + + // Chain VTXOs from the bulk fetch + chainVtxos := []domain.Vtxo{ + {Outpoint: domain.Outpoint{Txid: "v-450", VOut: 0}, Depth: 450}, + {Outpoint: domain.Outpoint{Txid: "v-350", VOut: 0}, Depth: 350}, + {Outpoint: domain.Outpoint{Txid: "v-250", VOut: 0}, Depth: 250}, + {Outpoint: domain.Outpoint{Txid: "v-150", VOut: 0}, Depth: 150}, + {Outpoint: domain.Outpoint{Txid: "v-050", VOut: 0}, Depth: 50}, + {Outpoint: domain.Outpoint{Txid: "v-000", VOut: 0}, Depth: 0}, + } + + // All 6 markers should be collected + markerRepo.On("GetVtxoChainByMarkers", ctx, mock.MatchedBy(func(ids []string) bool { + if len(ids) != 6 { + return false + } + idSet := make(map[string]bool) + for _, id := range ids { + idSet[id] = true + } + return idSet["marker-500"] && idSet["marker-400"] && idSet["marker-300"] && + idSet["marker-200"] && idSet["marker-100"] && idSet["marker-0"] + })).Return(chainVtxos, nil) + + cache := indexer.prefetchVtxosByMarkers(ctx, startKey) + + // 6 chain VTXOs + 1 start VTXO = 7 + require.Len(t, cache, 7) + require.Contains(t, cache, "deep-vtxo:0") + require.Contains(t, cache, "v-450:0") + require.Contains(t, cache, "v-350:0") + require.Contains(t, cache, "v-250:0") + require.Contains(t, cache, "v-150:0") + require.Contains(t, cache, "v-050:0") + require.Contains(t, cache, "v-000:0") + + // Verify every marker in the chain was visited + markerRepo.AssertCalled(t, "GetMarker", ctx, "marker-500") + markerRepo.AssertCalled(t, "GetMarker", ctx, "marker-400") + markerRepo.AssertCalled(t, "GetMarker", ctx, "marker-300") + markerRepo.AssertCalled(t, "GetMarker", ctx, "marker-200") + markerRepo.AssertCalled(t, "GetMarker", ctx, "marker-100") + markerRepo.AssertCalled(t, "GetMarker", ctx, "marker-0") +} + +// TestGetVtxosFromCacheOrDB_EmptyOutpoints verifies that an empty outpoints +// list returns an empty result without making any database call. +func TestGetVtxosFromCacheOrDB_EmptyOutpoints(t *testing.T) { + vtxoRepo := &mockVtxoRepoForIndexer{} + markerRepo := &mockMarkerRepoForIndexer{} + repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} + + indexer := &indexerService{repoManager: repoManager} + + ctx := context.Background() + cache := map[string]domain.Vtxo{ + "existing:0": {Outpoint: domain.Outpoint{Txid: "existing", VOut: 0}}, + } + + result, err := indexer.getVtxosFromCacheOrDB(ctx, []domain.Outpoint{}, cache) + + require.NoError(t, err) + require.Empty(t, result) + + // DB should never be called for empty input + vtxoRepo.AssertNotCalled(t, "GetVtxos", mock.Anything, mock.Anything) +} diff --git a/internal/core/application/service_test.go b/internal/core/application/service_test.go index cda4a5e1b..bde130c78 100644 --- a/internal/core/application/service_test.go +++ b/internal/core/application/service_test.go @@ -1,9 +1,12 @@ package application import ( + "fmt" + "sort" "testing" "time" + "github.com/arkade-os/arkd/internal/core/domain" "github.com/stretchr/testify/require" ) @@ -66,3 +69,415 @@ func parseTime(t *testing.T, value string) time.Time { require.NoError(t, err) return tm } + +// calculateMaxDepth mimics the depth calculation logic in the service event handler. +// This function exists to make the depth calculation testable independently. +func calculateMaxDepth(spentVtxos []domain.Vtxo) uint32 { + var maxDepth uint32 + for _, v := range spentVtxos { + if v.Depth > maxDepth { + maxDepth = v.Depth + } + } + return maxDepth +} + +func TestDepthCalculation(t *testing.T) { + testCases := []struct { + name string + spentVtxos []domain.Vtxo + expectedDepth uint32 + description string + }{ + { + name: "single batch vtxo at depth 0", + spentVtxos: []domain.Vtxo{{Depth: 0}}, + expectedDepth: 1, + description: "spending a batch vtxo creates vtxo at depth 1", + }, + { + name: "single vtxo at depth 50", + spentVtxos: []domain.Vtxo{{Depth: 50}}, + expectedDepth: 51, + description: "spending a chained vtxo increments depth", + }, + { + name: "multiple vtxos with same depth", + spentVtxos: []domain.Vtxo{ + {Depth: 10}, + {Depth: 10}, + {Depth: 10}, + }, + expectedDepth: 11, + description: "combining vtxos at same depth increments once", + }, + { + name: "multiple vtxos with different depths", + spentVtxos: []domain.Vtxo{ + {Depth: 5}, + {Depth: 25}, + {Depth: 15}, + }, + expectedDepth: 26, + description: "uses max depth from inputs", + }, + { + name: "vtxos spanning marker boundary", + spentVtxos: []domain.Vtxo{ + {Depth: 95}, + {Depth: 105}, + }, + expectedDepth: 106, + description: "handles depths across marker boundaries", + }, + { + name: "deep chain near marker boundary", + spentVtxos: []domain.Vtxo{ + {Depth: 99}, + }, + expectedDepth: 100, + description: "result at marker boundary (100)", + }, + { + name: "very deep chain", + spentVtxos: []domain.Vtxo{ + {Depth: 500}, + }, + expectedDepth: 501, + description: "handles deep chains beyond multiple marker intervals", + }, + { + name: "no spent vtxos (empty)", + spentVtxos: []domain.Vtxo{}, + expectedDepth: 1, + description: "empty input results in depth 1 (edge case)", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + maxDepth := calculateMaxDepth(tc.spentVtxos) + newDepth := maxDepth + 1 + require.Equal(t, tc.expectedDepth, newDepth, tc.description) + }) + } +} + +func TestDepthAtMarkerBoundary(t *testing.T) { + // Test integration of depth and marker boundary detection + testCases := []struct { + depth uint32 + isAtBoundary bool + description string + }{ + {0, true, "depth 0 is at marker boundary"}, + {1, false, "depth 1 is not at boundary"}, + {50, false, "depth 50 is not at boundary"}, + {99, false, "depth 99 is not at boundary"}, + {100, true, "depth 100 is at marker boundary"}, + {101, false, "depth 101 is not at boundary"}, + {200, true, "depth 200 is at marker boundary"}, + {500, true, "depth 500 is at marker boundary"}, + {1000, true, "depth 1000 is at marker boundary"}, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + isAtBoundary := domain.IsAtMarkerBoundary(tc.depth) + require.Equal(t, tc.isAtBoundary, isAtBoundary) + }) + } +} + +func TestDepthIncrementCreatesMarkerAtBoundary(t *testing.T) { + // Test scenario: when depth increments to a marker boundary, + // a marker should be created for that VTXO + testCases := []struct { + parentDepth uint32 + newDepth uint32 + shouldCreateMarker bool + }{ + {99, 100, true}, // crossing into boundary + {100, 101, false}, // leaving boundary + {199, 200, true}, // crossing into next boundary + {0, 1, false}, // moving away from initial boundary + {98, 99, false}, // approaching but not at boundary + } + + for _, tc := range testCases { + t.Run("", func(t *testing.T) { + // Simulate the depth increment + spentVtxos := []domain.Vtxo{{Depth: tc.parentDepth}} + maxDepth := calculateMaxDepth(spentVtxos) + newDepth := maxDepth + 1 + + require.Equal(t, tc.newDepth, newDepth) + isAtBoundary := domain.IsAtMarkerBoundary(newDepth) + require.Equal(t, tc.shouldCreateMarker, isAtBoundary) + }) + } +} + +// collectParentMarkers mimics the parent marker collection logic in the +// service's updateProjectionsAfterOffchainTxEvents handler. +// It collects ALL unique, non-empty marker IDs from the spent VTXOs. +func collectParentMarkers(spentVtxos []domain.Vtxo) []string { + parentMarkerSet := make(map[string]struct{}) + for _, v := range spentVtxos { + for _, markerID := range v.MarkerIDs { + if markerID != "" { + parentMarkerSet[markerID] = struct{}{} + } + } + } + result := make([]string, 0, len(parentMarkerSet)) + for id := range parentMarkerSet { + result = append(result, id) + } + return result +} + +// deriveMarkerIDs mimics the marker creation/inheritance decision in the +// service's updateProjectionsAfterOffchainTxEvents handler. +// At boundary depths a new marker is created; otherwise parent markers are inherited. +func deriveMarkerIDs( + newDepth uint32, + parentMarkerIDs []string, + txid string, +) (markerIDs []string, createdMarker *domain.Marker) { + if domain.IsAtMarkerBoundary(newDepth) { + newMarkerID := txid + ":marker:" + fmt.Sprintf("%d", newDepth) + marker := domain.Marker{ + ID: newMarkerID, + Depth: newDepth, + ParentMarkerIDs: parentMarkerIDs, + } + return []string{newMarkerID}, &marker + } + if len(parentMarkerIDs) > 0 { + return parentMarkerIDs, nil + } + return nil, nil +} + +func TestParentMarkerCollectionFromMultipleParents(t *testing.T) { + // When spending multiple VTXOs with different marker sets, + // the parent marker set should be the deduplicated union of all inputs' markers. + testCases := []struct { + name string + spentVtxos []domain.Vtxo + expectedMarkers []string + }{ + { + name: "single parent with one marker", + spentVtxos: []domain.Vtxo{ + {MarkerIDs: []string{"marker-A"}}, + }, + expectedMarkers: []string{"marker-A"}, + }, + { + name: "two parents with distinct markers", + spentVtxos: []domain.Vtxo{ + {MarkerIDs: []string{"marker-A"}}, + {MarkerIDs: []string{"marker-B"}}, + }, + expectedMarkers: []string{"marker-A", "marker-B"}, + }, + { + name: "three parents with overlapping markers", + spentVtxos: []domain.Vtxo{ + {MarkerIDs: []string{"marker-A", "marker-B"}}, + {MarkerIDs: []string{"marker-B", "marker-C"}}, + {MarkerIDs: []string{"marker-A", "marker-C"}}, + }, + expectedMarkers: []string{"marker-A", "marker-B", "marker-C"}, + }, + { + name: "all parents share the same marker", + spentVtxos: []domain.Vtxo{ + {MarkerIDs: []string{"root-marker"}}, + {MarkerIDs: []string{"root-marker"}}, + {MarkerIDs: []string{"root-marker"}}, + }, + expectedMarkers: []string{"root-marker"}, + }, + { + name: "no parents", + spentVtxos: []domain.Vtxo{}, + expectedMarkers: []string{}, + }, + { + name: "parent with no markers", + spentVtxos: []domain.Vtxo{ + {MarkerIDs: []string{}}, + }, + expectedMarkers: []string{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := collectParentMarkers(tc.spentVtxos) + sort.Strings(result) + sort.Strings(tc.expectedMarkers) + require.Equal(t, tc.expectedMarkers, result) + }) + } +} + +func TestParentMarkerCollectionSkipsEmptyMarkerIDs(t *testing.T) { + // Empty string marker IDs should be filtered out. + spentVtxos := []domain.Vtxo{ + {MarkerIDs: []string{"marker-A", "", "marker-B"}}, + {MarkerIDs: []string{"", ""}}, + {MarkerIDs: []string{"marker-C", ""}}, + } + + result := collectParentMarkers(spentVtxos) + sort.Strings(result) + require.Equal(t, []string{"marker-A", "marker-B", "marker-C"}, result) +} + +func TestMarkerInheritanceAtNonBoundary(t *testing.T) { + // When the new depth is NOT at a marker boundary, the child VTXO + // should inherit ALL parent marker IDs (no new marker created). + testCases := []struct { + name string + parentDepths []uint32 + parentMarkerSets [][]string + expectedDepth uint32 + expectedMarkers []string + description string + }{ + { + name: "single parent at depth 0, child at depth 1", + parentDepths: []uint32{0}, + parentMarkerSets: [][]string{{"root-marker-1"}}, + expectedDepth: 1, + expectedMarkers: []string{"root-marker-1"}, + description: "child inherits single parent marker", + }, + { + name: "single parent at depth 50, child at depth 51", + parentDepths: []uint32{50}, + parentMarkerSets: [][]string{{"marker-A", "marker-B"}}, + expectedDepth: 51, + expectedMarkers: []string{"marker-A", "marker-B"}, + description: "child inherits multiple parent markers", + }, + { + name: "two parents at different depths, child not at boundary", + parentDepths: []uint32{30, 40}, + parentMarkerSets: [][]string{{"marker-X"}, {"marker-Y"}}, + expectedDepth: 41, + expectedMarkers: []string{"marker-X", "marker-Y"}, + description: "child inherits union of all parent markers", + }, + { + name: "three parents with overlapping markers", + parentDepths: []uint32{10, 20, 15}, + parentMarkerSets: [][]string{{"m1", "m2"}, {"m2", "m3"}, {"m1"}}, + expectedDepth: 21, + expectedMarkers: []string{"m1", "m2", "m3"}, + description: "child inherits deduplicated union", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + spentVtxos := make([]domain.Vtxo, len(tc.parentDepths)) + for i, depth := range tc.parentDepths { + spentVtxos[i] = domain.Vtxo{ + Depth: depth, + MarkerIDs: tc.parentMarkerSets[i], + } + } + + maxDepth := calculateMaxDepth(spentVtxos) + newDepth := maxDepth + 1 + require.Equal(t, tc.expectedDepth, newDepth) + + // Should NOT be at a marker boundary + require.False(t, domain.IsAtMarkerBoundary(newDepth), + "depth %d should not be at marker boundary for this test", newDepth) + + parentMarkers := collectParentMarkers(spentVtxos) + markerIDs, createdMarker := deriveMarkerIDs(newDepth, parentMarkers, "some-txid") + + // No new marker should be created + require.Nil(t, createdMarker, tc.description) + // Should inherit all parent markers + sort.Strings(markerIDs) + sort.Strings(tc.expectedMarkers) + require.Equal(t, tc.expectedMarkers, markerIDs, tc.description) + }) + } +} + +func TestMarkerCreationAtBoundary(t *testing.T) { + // When the new depth IS at a marker boundary, a new marker should be + // created with the collected parent markers as its ParentMarkerIDs. + testCases := []struct { + name string + parentDepths []uint32 + parentMarkerSets [][]string + expectedDepth uint32 + description string + }{ + { + name: "parent at depth 99, child at depth 100", + parentDepths: []uint32{99}, + parentMarkerSets: [][]string{{"root-marker"}}, + expectedDepth: 100, + description: "first non-root boundary", + }, + { + name: "parent at depth 199, child at depth 200", + parentDepths: []uint32{199}, + parentMarkerSets: [][]string{{"marker-100", "root-marker"}}, + expectedDepth: 200, + description: "second boundary with two parent markers", + }, + { + name: "multiple parents converging at boundary", + parentDepths: []uint32{95, 99}, + parentMarkerSets: [][]string{{"marker-A"}, {"marker-B"}}, + expectedDepth: 100, + description: "boundary with multiple parent VTXOs", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + spentVtxos := make([]domain.Vtxo, len(tc.parentDepths)) + for i, depth := range tc.parentDepths { + spentVtxos[i] = domain.Vtxo{ + Depth: depth, + MarkerIDs: tc.parentMarkerSets[i], + } + } + + maxDepth := calculateMaxDepth(spentVtxos) + newDepth := maxDepth + 1 + require.Equal(t, tc.expectedDepth, newDepth) + + // Should be at a marker boundary + require.True(t, domain.IsAtMarkerBoundary(newDepth), + "depth %d should be at marker boundary for this test", newDepth) + + parentMarkers := collectParentMarkers(spentVtxos) + markerIDs, createdMarker := deriveMarkerIDs(newDepth, parentMarkers, "test-txid") + + // A new marker should be created + require.NotNil(t, createdMarker, tc.description) + require.Equal(t, newDepth, createdMarker.Depth) + // The new marker's ParentMarkerIDs should match collected parent markers + sort.Strings(createdMarker.ParentMarkerIDs) + sort.Strings(parentMarkers) + require.Equal(t, parentMarkers, createdMarker.ParentMarkerIDs) + // The child VTXO should get ONLY the new marker ID, not parent markers + require.Len(t, markerIDs, 1) + require.Equal(t, createdMarker.ID, markerIDs[0]) + }) + } +} diff --git a/internal/core/application/sweeper_test.go b/internal/core/application/sweeper_test.go new file mode 100644 index 000000000..73ccdd83a --- /dev/null +++ b/internal/core/application/sweeper_test.go @@ -0,0 +1,1242 @@ +package application + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/arkade-os/arkd/internal/core/domain" + "github.com/arkade-os/arkd/internal/core/ports" + arklib "github.com/arkade-os/arkd/pkg/ark-lib" + "github.com/arkade-os/arkd/pkg/ark-lib/tree" + "github.com/btcsuite/btcd/btcec/v2" + "github.com/btcsuite/btcd/btcutil/psbt" + "github.com/btcsuite/btcd/wire" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// Mock implementations for sweeper tests + +type mockWalletService struct { + mock.Mock +} + +func (m *mockWalletService) BroadcastTransaction( + ctx context.Context, + txs ...string, +) (string, error) { + args := m.Called(ctx, txs) + return args.String(0), args.Error(1) +} + +func (m *mockWalletService) GetTransaction(ctx context.Context, txid string) (string, error) { + args := m.Called(ctx, txid) + return args.String(0), args.Error(1) +} + +// Stub implementations for unused WalletService methods +func (m *mockWalletService) GetReadyUpdate(ctx context.Context) (<-chan struct{}, error) { + return nil, nil +} +func (m *mockWalletService) GenSeed(ctx context.Context) (string, error) { return "", nil } +func (m *mockWalletService) Create(ctx context.Context, seed, password string) error { + return nil +} +func (m *mockWalletService) Restore(ctx context.Context, seed, password string) error { + return nil +} +func (m *mockWalletService) Unlock(ctx context.Context, password string) error { return nil } +func (m *mockWalletService) Lock(ctx context.Context) error { return nil } +func (m *mockWalletService) Status(ctx context.Context) (ports.WalletStatus, error) { + return nil, nil +} +func (m *mockWalletService) GetNetwork(ctx context.Context) (*arklib.Network, error) { + return nil, nil +} +func (m *mockWalletService) GetForfeitPubkey(ctx context.Context) (*btcec.PublicKey, error) { + return nil, nil +} +func (m *mockWalletService) DeriveConnectorAddress(ctx context.Context) (string, error) { + return "", nil +} +func (m *mockWalletService) DeriveAddresses(ctx context.Context, num int) ([]string, error) { + return nil, nil +} + +func (m *mockWalletService) SignTransaction( + ctx context.Context, + tx string, + extractRawTx bool, +) (string, error) { + return "", nil +} + +func (m *mockWalletService) SignTransactionTapscript( + ctx context.Context, + tx string, + inputIndexes []int, +) (string, error) { + return "", nil +} + +func (m *mockWalletService) SelectUtxos( + ctx context.Context, + asset string, + amount uint64, + confirmedOnly bool, +) ([]ports.TxInput, uint64, error) { + return nil, 0, nil +} +func (m *mockWalletService) EstimateFees(ctx context.Context, pset string) (uint64, error) { + return 0, nil +} +func (m *mockWalletService) FeeRate(ctx context.Context) (uint64, error) { return 0, nil } + +func (m *mockWalletService) ListConnectorUtxos( + ctx context.Context, + addr string, +) ([]ports.TxInput, error) { + return nil, nil +} +func (m *mockWalletService) MainAccountBalance(ctx context.Context) (uint64, uint64, error) { + return 0, 0, nil +} +func (m *mockWalletService) ConnectorsAccountBalance(ctx context.Context) (uint64, uint64, error) { + return 0, 0, nil +} +func (m *mockWalletService) LockConnectorUtxos(ctx context.Context, utxos []domain.Outpoint) error { + return nil +} +func (m *mockWalletService) GetDustAmount(ctx context.Context) (uint64, error) { return 0, nil } + +func (m *mockWalletService) GetOutpointStatus( + ctx context.Context, + outpoint domain.Outpoint, +) (bool, error) { + return false, nil +} + +func (m *mockWalletService) GetCurrentBlockTime( + ctx context.Context, +) (*ports.BlockTimestamp, error) { + return nil, nil +} + +func (m *mockWalletService) Withdraw( + ctx context.Context, + address string, + amount uint64, + all bool, +) (string, error) { + return "", nil +} +func (m *mockWalletService) LoadSignerKey(ctx context.Context, prvkey string) error { return nil } +func (m *mockWalletService) Close() {} +func (m *mockWalletService) WatchScripts(ctx context.Context, scripts []string) error { + return nil +} +func (m *mockWalletService) UnwatchScripts(ctx context.Context, scripts []string) error { + return nil +} + +func (m *mockWalletService) GetNotificationChannel( + ctx context.Context, +) <-chan map[string][]ports.VtxoWithValue { + return nil +} + +func (m *mockWalletService) IsTransactionConfirmed( + ctx context.Context, + txid string, +) (bool, *ports.BlockTimestamp, error) { + return false, nil, nil +} +func (m *mockWalletService) RescanUtxos(ctx context.Context, outpoints []wire.OutPoint) error { + return nil +} + +type mockVtxoRepository struct { + mock.Mock +} + +func (m *mockVtxoRepository) GetAllChildrenVtxos( + ctx context.Context, + txid string, +) ([]domain.Outpoint, error) { + args := m.Called(ctx, txid) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).([]domain.Outpoint), args.Error(1) +} + +func (m *mockVtxoRepository) GetVtxos( + ctx context.Context, + outpoints []domain.Outpoint, +) ([]domain.Vtxo, error) { + args := m.Called(ctx, outpoints) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).([]domain.Vtxo), args.Error(1) +} + +// Stub implementations for unused VtxoRepository methods +func (m *mockVtxoRepository) AddVtxos(ctx context.Context, vtxos []domain.Vtxo) error { return nil } + +func (m *mockVtxoRepository) SettleVtxos( + ctx context.Context, + spentVtxos map[domain.Outpoint]string, + commitmentTxid string, +) error { + return nil +} + +func (m *mockVtxoRepository) SpendVtxos( + ctx context.Context, + spentVtxos map[domain.Outpoint]string, + arkTxid string, +) error { + return nil +} +func (m *mockVtxoRepository) UnrollVtxos(ctx context.Context, outpoints []domain.Outpoint) error { + return nil +} + +func (m *mockVtxoRepository) GetAllNonUnrolledVtxos( + ctx context.Context, + pubkey string, +) ([]domain.Vtxo, []domain.Vtxo, error) { + return nil, nil, nil +} + +func (m *mockVtxoRepository) GetAllSweepableUnrolledVtxos( + ctx context.Context, +) ([]domain.Vtxo, error) { + return nil, nil +} +func (m *mockVtxoRepository) GetAllVtxos(ctx context.Context) ([]domain.Vtxo, error) { + return nil, nil +} + +func (m *mockVtxoRepository) GetAllVtxosWithPubKeys( + ctx context.Context, + pubkeys []string, + after, before int64, +) ([]domain.Vtxo, error) { + return nil, nil +} + +func (m *mockVtxoRepository) GetExpiringLiquidity( + ctx context.Context, + after, before int64, +) (uint64, error) { + return 0, nil +} +func (m *mockVtxoRepository) GetRecoverableLiquidity(ctx context.Context) (uint64, error) { + return 0, nil +} + +func (m *mockVtxoRepository) UpdateVtxosExpiration( + ctx context.Context, + outpoints []domain.Outpoint, + expiresAt int64, +) error { + return nil +} + +func (m *mockVtxoRepository) GetLeafVtxosForBatch( + ctx context.Context, + txid string, +) ([]domain.Vtxo, error) { + return nil, nil +} + +func (m *mockVtxoRepository) GetSweepableVtxosByCommitmentTxid( + ctx context.Context, + commitmentTxid string, +) ([]domain.Outpoint, error) { + return nil, nil +} + +func (m *mockVtxoRepository) GetVtxoPubKeysByCommitmentTxid( + ctx context.Context, + commitmentTxid string, + withMinimumAmount uint64, +) ([]string, error) { + return nil, nil +} + +func (m *mockVtxoRepository) GetPendingSpentVtxosWithPubKeys( + ctx context.Context, + pubkeys []string, + after, before int64, +) ([]domain.Vtxo, error) { + return nil, nil +} + +func (m *mockVtxoRepository) GetPendingSpentVtxosWithOutpoints( + ctx context.Context, + outpoints []domain.Outpoint, +) ([]domain.Vtxo, error) { + return nil, nil +} +func (m *mockVtxoRepository) Close() {} + +type mockMarkerRepository struct { + mock.Mock +} + +func (m *mockMarkerRepository) BulkSweepMarkers( + ctx context.Context, + markerIDs []string, + sweptAt int64, +) error { + args := m.Called(ctx, markerIDs, sweptAt) + return args.Error(0) +} + +// Stub implementations for unused MarkerRepository methods +func (m *mockMarkerRepository) AddMarker(ctx context.Context, marker domain.Marker) error { + return nil +} +func (m *mockMarkerRepository) GetMarker(ctx context.Context, id string) (*domain.Marker, error) { + return nil, nil +} + +func (m *mockMarkerRepository) GetMarkersByDepth( + ctx context.Context, + depth uint32, +) ([]domain.Marker, error) { + return nil, nil +} + +func (m *mockMarkerRepository) GetMarkersByDepthRange( + ctx context.Context, + minDepth, maxDepth uint32, +) ([]domain.Marker, error) { + return nil, nil +} + +func (m *mockMarkerRepository) GetMarkersByIds( + ctx context.Context, + ids []string, +) ([]domain.Marker, error) { + return nil, nil +} + +func (m *mockMarkerRepository) SweepMarker( + ctx context.Context, + markerID string, + sweptAt int64, +) error { + return nil +} + +func (m *mockMarkerRepository) SweepMarkerWithDescendants( + ctx context.Context, + markerID string, + sweptAt int64, +) (int64, error) { + return 0, nil +} +func (m *mockMarkerRepository) IsMarkerSwept(ctx context.Context, markerID string) (bool, error) { + return false, nil +} + +func (m *mockMarkerRepository) GetSweptMarkers( + ctx context.Context, + markerIDs []string, +) ([]domain.SweptMarker, error) { + return nil, nil +} + +func (m *mockMarkerRepository) UpdateVtxoMarkers( + ctx context.Context, + outpoint domain.Outpoint, + markerIDs []string, +) error { + return nil +} + +func (m *mockMarkerRepository) GetVtxosByMarker( + ctx context.Context, + markerID string, +) ([]domain.Vtxo, error) { + return nil, nil +} + +func (m *mockMarkerRepository) SweepVtxosByMarker( + ctx context.Context, + markerID string, +) (int64, error) { + return 0, nil +} + +func (m *mockMarkerRepository) CreateRootMarkersForVtxos( + ctx context.Context, + vtxos []domain.Vtxo, +) error { + return nil +} + +func (m *mockMarkerRepository) GetVtxosByDepthRange( + ctx context.Context, + minDepth, maxDepth uint32, +) ([]domain.Vtxo, error) { + return nil, nil +} + +func (m *mockMarkerRepository) GetVtxosByArkTxid( + ctx context.Context, + arkTxid string, +) ([]domain.Vtxo, error) { + return nil, nil +} + +func (m *mockMarkerRepository) GetVtxoChainByMarkers( + ctx context.Context, + markerIDs []string, +) ([]domain.Vtxo, error) { + return nil, nil +} +func (m *mockMarkerRepository) Close() {} + +type mockTxBuilder struct { + mock.Mock +} + +func (m *mockTxBuilder) BuildSweepTx(inputs []ports.TxInput) (string, string, error) { + args := m.Called(inputs) + return args.String(0), args.String(1), args.Error(2) +} + +// Stub implementations for unused TxBuilder methods +func (m *mockTxBuilder) BuildCommitmentTx( + signerPubkey *btcec.PublicKey, intents domain.Intents, + boardingInputs []ports.BoardingInput, cosigners [][]string, +) (string, *tree.TxTree, string, *tree.TxTree, error) { + return "", nil, "", nil, nil +} + +func (m *mockTxBuilder) VerifyForfeitTxs( + vtxos []domain.Vtxo, + connectors tree.FlatTxTree, + txs []string, +) (map[domain.Outpoint]ports.ValidForfeitTx, error) { + return nil, nil +} + +func (m *mockTxBuilder) GetSweepableBatchOutputs( + vtxoTree *tree.TxTree, +) (*arklib.RelativeLocktime, *ports.TxInput, error) { + return nil, nil, nil +} +func (m *mockTxBuilder) FinalizeAndExtract(tx string) (string, error) { return "", nil } + +func (m *mockTxBuilder) VerifyVtxoTapscriptSigs( + tx string, + mustIncludeSignerSig bool, +) (bool, *psbt.Packet, error) { + return false, nil, nil +} + +func (m *mockTxBuilder) VerifyBoardingTapscriptSigs( + signedTx string, + commitmentTx string, +) (map[uint32]ports.SignedBoardingInput, error) { + return nil, nil +} + +type mockRepoManager struct { + vtxos *mockVtxoRepository + markers *mockMarkerRepository +} + +func (m *mockRepoManager) Events() domain.EventRepository { return nil } +func (m *mockRepoManager) Rounds() domain.RoundRepository { return nil } +func (m *mockRepoManager) Vtxos() domain.VtxoRepository { return m.vtxos } +func (m *mockRepoManager) Markers() domain.MarkerRepository { return m.markers } +func (m *mockRepoManager) ScheduledSession() domain.ScheduledSessionRepo { return nil } +func (m *mockRepoManager) OffchainTxs() domain.OffchainTxRepository { return nil } +func (m *mockRepoManager) Convictions() domain.ConvictionRepository { return nil } +func (m *mockRepoManager) Fees() domain.FeeRepository { return nil } +func (m *mockRepoManager) Close() {} + +type mockScheduler struct{} + +func (m *mockScheduler) Start() {} +func (m *mockScheduler) Stop() {} +func (m *mockScheduler) Unit() ports.TimeUnit { return ports.UnixTime } +func (m *mockScheduler) AfterNow(expiry int64) bool { return false } +func (m *mockScheduler) ScheduleTaskOnce(at int64, task func()) error { return nil } + +// TestCreateCheckpointSweepTask_BulkSweepsMarkers verifies that when a checkpoint +// is swept, the sweeper correctly collects all unique marker IDs from the affected +// VTXOs and calls BulkSweepMarkers with the deduplicated set. This tests the core +// optimization where multiple VTXOs sharing markers result in fewer marker sweep +// operations (3 VTXOs with overlapping markers should yield only 3 unique markers). +func TestCreateCheckpointSweepTask_BulkSweepsMarkers(t *testing.T) { + // Setup mocks + wallet := &mockWalletService{} + vtxoRepo := &mockVtxoRepository{} + markerRepo := &mockMarkerRepository{} + repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} + builder := &mockTxBuilder{} + scheduler := &mockScheduler{} + + // Create sweeper instance + s := newSweeper(wallet, repoManager, builder, scheduler, "") + + // Test data + checkpointTxid := "checkpoint123" + vtxoOutpoint := domain.Outpoint{Txid: "vtxo123", VOut: 0} + + // Child VTXOs that will be returned by GetAllChildrenVtxos + childOutpoints := []domain.Outpoint{ + {Txid: "child1", VOut: 0}, + {Txid: "child2", VOut: 0}, + {Txid: "child3", VOut: 0}, + } + + // VTXOs with markers - note some share markers to test deduplication + vtxosWithMarkers := []domain.Vtxo{ + { + Outpoint: childOutpoints[0], + MarkerIDs: []string{"marker-A", "marker-B"}, + Depth: 50, + }, + { + Outpoint: childOutpoints[1], + MarkerIDs: []string{"marker-B", "marker-C"}, // marker-B is shared + Depth: 75, + }, + { + Outpoint: childOutpoints[2], + MarkerIDs: []string{"marker-A"}, // marker-A is shared + Depth: 100, + }, + } + + // Setup mock expectations + toSweep := ports.TxInput{ + Txid: checkpointTxid, + Index: 0, + Value: 10000, + } + + builder.On("BuildSweepTx", []ports.TxInput{toSweep}). + Return("sweeptxid123", "sweeptx_hex", nil) + + wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). + Return("sweeptxid123", nil) + + vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). + Return(childOutpoints, nil) + + vtxoRepo.On("GetVtxos", mock.Anything, childOutpoints). + Return(vtxosWithMarkers, nil) + + // Expect BulkSweepMarkers to be called with deduplicated markers + // Should have: marker-A, marker-B, marker-C (3 unique markers) + markerRepo.On("BulkSweepMarkers", mock.Anything, mock.MatchedBy(func(markerIDs []string) bool { + // Verify we have exactly 3 unique markers + if len(markerIDs) != 3 { + return false + } + // Verify all expected markers are present + markerSet := make(map[string]bool) + for _, id := range markerIDs { + markerSet[id] = true + } + return markerSet["marker-A"] && markerSet["marker-B"] && markerSet["marker-C"] + }), mock.AnythingOfType("int64")).Return(nil) + + // Execute the sweep task + task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) + err := task() + + // Verify + require.NoError(t, err) + wallet.AssertExpectations(t) + vtxoRepo.AssertExpectations(t) + markerRepo.AssertExpectations(t) + builder.AssertExpectations(t) +} + +// TestCreateCheckpointSweepTask_NoMarkersSkipsSweep verifies that when VTXOs have +// no markers (empty MarkerIDs slice), the sweeper does not call BulkSweepMarkers. +// This is an edge case that could occur with legacy VTXOs or during error recovery, +// and ensures the sweeper handles it gracefully without attempting empty bulk operations. +func TestCreateCheckpointSweepTask_NoMarkersSkipsSweep(t *testing.T) { + // Setup mocks + wallet := &mockWalletService{} + vtxoRepo := &mockVtxoRepository{} + markerRepo := &mockMarkerRepository{} + repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} + builder := &mockTxBuilder{} + scheduler := &mockScheduler{} + + // Create sweeper instance + s := newSweeper(wallet, repoManager, builder, scheduler, "") + + // Test data + checkpointTxid := "checkpoint456" + vtxoOutpoint := domain.Outpoint{Txid: "vtxo456", VOut: 0} + + // Child VTXOs with no markers (empty MarkerIDs) + childOutpoints := []domain.Outpoint{ + {Txid: "child1", VOut: 0}, + } + + vtxosWithoutMarkers := []domain.Vtxo{ + { + Outpoint: childOutpoints[0], + MarkerIDs: []string{}, // No markers + Depth: 0, + }, + } + + // Setup mock expectations + toSweep := ports.TxInput{ + Txid: checkpointTxid, + Index: 0, + Value: 10000, + } + + builder.On("BuildSweepTx", []ports.TxInput{toSweep}). + Return("sweeptxid456", "sweeptx_hex", nil) + + wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). + Return("sweeptxid456", nil) + + vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). + Return(childOutpoints, nil) + + vtxoRepo.On("GetVtxos", mock.Anything, childOutpoints). + Return(vtxosWithoutMarkers, nil) + + // BulkSweepMarkers should NOT be called since there are no markers + + // Execute the sweep task + task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) + err := task() + + // Verify + require.NoError(t, err) + wallet.AssertExpectations(t) + vtxoRepo.AssertExpectations(t) + // Verify BulkSweepMarkers was never called + markerRepo.AssertNotCalled(t, "BulkSweepMarkers", mock.Anything, mock.Anything, mock.Anything) +} + +// TestCreateCheckpointSweepTask_SingleMarkerPerVtxo verifies the typical post-migration +// state where each VTXO has exactly one marker (its own outpoint as the marker ID). +// This represents the common case after the database migration that assigns a unique +// marker to every existing VTXO, ensuring backward compatibility with the new marker system. +func TestCreateCheckpointSweepTask_SingleMarkerPerVtxo(t *testing.T) { + // Test case: each VTXO has exactly one marker (post-migration state) + wallet := &mockWalletService{} + vtxoRepo := &mockVtxoRepository{} + markerRepo := &mockMarkerRepository{} + repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} + builder := &mockTxBuilder{} + scheduler := &mockScheduler{} + + s := newSweeper(wallet, repoManager, builder, scheduler, "") + + checkpointTxid := "checkpoint789" + vtxoOutpoint := domain.Outpoint{Txid: "vtxo789", VOut: 0} + + // Each VTXO has its own unique marker (typical post-migration state) + childOutpoints := []domain.Outpoint{ + {Txid: "child1", VOut: 0}, + {Txid: "child2", VOut: 0}, + } + + vtxosWithUniqueMarkers := []domain.Vtxo{ + { + Outpoint: childOutpoints[0], + MarkerIDs: []string{"child1:0"}, // Marker ID matches outpoint + Depth: 0, + }, + { + Outpoint: childOutpoints[1], + MarkerIDs: []string{"child2:0"}, + Depth: 0, + }, + } + + toSweep := ports.TxInput{ + Txid: checkpointTxid, + Index: 0, + Value: 20000, + } + + builder.On("BuildSweepTx", []ports.TxInput{toSweep}). + Return("sweeptxid789", "sweeptx_hex", nil) + + wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). + Return("sweeptxid789", nil) + + vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). + Return(childOutpoints, nil) + + vtxoRepo.On("GetVtxos", mock.Anything, childOutpoints). + Return(vtxosWithUniqueMarkers, nil) + + // Expect exactly 2 unique markers + markerRepo.On("BulkSweepMarkers", mock.Anything, mock.MatchedBy(func(markerIDs []string) bool { + if len(markerIDs) != 2 { + return false + } + markerSet := make(map[string]bool) + for _, id := range markerIDs { + markerSet[id] = true + } + return markerSet["child1:0"] && markerSet["child2:0"] + }), mock.AnythingOfType("int64")).Return(nil) + + task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) + err := task() + + require.NoError(t, err) + wallet.AssertExpectations(t) + vtxoRepo.AssertExpectations(t) + markerRepo.AssertExpectations(t) +} + +// TestCreateCheckpointSweepTask_ManyVtxosWithSharedMarkers verifies that the bulk sweep +// optimization works correctly for deep VTXO chains where many VTXOs share the same +// markers. This simulates a chain spanning depths 0-196 where all VTXOs share a root +// marker, and VTXOs at depth >= 100 also share an additional marker. Despite having +// 50 VTXOs, only 2 unique markers should be swept, demonstrating the efficiency gain. +func TestCreateCheckpointSweepTask_ManyVtxosWithSharedMarkers(t *testing.T) { + // Test case: many VTXOs share markers (chain with depth > 100) + wallet := &mockWalletService{} + vtxoRepo := &mockVtxoRepository{} + markerRepo := &mockMarkerRepository{} + repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} + builder := &mockTxBuilder{} + scheduler := &mockScheduler{} + + s := newSweeper(wallet, repoManager, builder, scheduler, "") + + checkpointTxid := "checkpoint_deep" + vtxoOutpoint := domain.Outpoint{Txid: "vtxo_deep", VOut: 0} + + // Simulate a deep chain where many VTXOs share the same root marker + childOutpoints := make([]domain.Outpoint, 50) + vtxosWithSharedMarkers := make([]domain.Vtxo, 50) + + for i := 0; i < 50; i++ { + childOutpoints[i] = domain.Outpoint{Txid: "child" + string(rune('A'+i)), VOut: 0} + // All VTXOs at depth < 100 share the root marker + // VTXOs at depth >= 100 also have a depth-100 marker + depth := uint32(i * 4) // depths: 0, 4, 8, ... 196 (spans beyond 100) + markers := []string{"root-marker"} + if depth >= 100 { + markers = append(markers, "marker-100") + } + vtxosWithSharedMarkers[i] = domain.Vtxo{ + Outpoint: childOutpoints[i], + MarkerIDs: markers, + Depth: depth, + } + } + + toSweep := ports.TxInput{ + Txid: checkpointTxid, + Index: 0, + Value: 500000, + } + + builder.On("BuildSweepTx", []ports.TxInput{toSweep}). + Return("sweeptxid_deep", "sweeptx_hex", nil) + + wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). + Return("sweeptxid_deep", nil) + + vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). + Return(childOutpoints, nil) + + vtxoRepo.On("GetVtxos", mock.Anything, childOutpoints). + Return(vtxosWithSharedMarkers, nil) + + // Even with 50 VTXOs, we should only have 2 unique markers + markerRepo.On("BulkSweepMarkers", mock.Anything, mock.MatchedBy(func(markerIDs []string) bool { + if len(markerIDs) != 2 { + return false + } + markerSet := make(map[string]bool) + for _, id := range markerIDs { + markerSet[id] = true + } + return markerSet["root-marker"] && markerSet["marker-100"] + }), mock.AnythingOfType("int64")).Return(nil) + + task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) + err := task() + + require.NoError(t, err) + markerRepo.AssertExpectations(t) +} + +// TestCreateCheckpointSweepTask_SweptAtTimestamp verifies that the sweptAt timestamp +// passed to BulkSweepMarkers is accurate and falls within the execution window. +// This ensures that swept marker records have correct timestamps for auditing and +// debugging purposes, and that the timestamp is generated at execution time rather +// than being a stale or incorrect value. +func TestCreateCheckpointSweepTask_SweptAtTimestamp(t *testing.T) { + // Test that the sweptAt timestamp is reasonable (within a few seconds of now) + wallet := &mockWalletService{} + vtxoRepo := &mockVtxoRepository{} + markerRepo := &mockMarkerRepository{} + repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} + builder := &mockTxBuilder{} + scheduler := &mockScheduler{} + + s := newSweeper(wallet, repoManager, builder, scheduler, "") + + checkpointTxid := "checkpoint_timestamp" + vtxoOutpoint := domain.Outpoint{Txid: "vtxo_timestamp", VOut: 0} + + childOutpoints := []domain.Outpoint{{Txid: "child_ts", VOut: 0}} + vtxos := []domain.Vtxo{{ + Outpoint: childOutpoints[0], + MarkerIDs: []string{"marker-ts"}, + Depth: 0, + }} + + toSweep := ports.TxInput{Txid: checkpointTxid, Index: 0, Value: 1000} + + builder.On("BuildSweepTx", []ports.TxInput{toSweep}). + Return("sweeptxid_ts", "sweeptx_hex", nil) + + wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). + Return("sweeptxid_ts", nil) + + vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). + Return(childOutpoints, nil) + + vtxoRepo.On("GetVtxos", mock.Anything, childOutpoints). + Return(vtxos, nil) + + // Capture the sweptAt timestamp + beforeExec := time.Now().Unix() + var capturedSweptAt int64 + + markerRepo.On("BulkSweepMarkers", mock.Anything, mock.Anything, mock.MatchedBy(func(sweptAt int64) bool { + capturedSweptAt = sweptAt + return true + })). + Return(nil) + + task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) + err := task() + afterExec := time.Now().Unix() + + require.NoError(t, err) + // Verify timestamp is within the execution window + require.GreaterOrEqual(t, capturedSweptAt, beforeExec) + require.LessOrEqual(t, capturedSweptAt, afterExec) +} + +// TestCreateCheckpointSweepTask_BulkSweepMarkersError verifies that when BulkSweepMarkers +// returns an error, the sweep task propagates the error back to the caller. This ensures +// that marker sweep failures are not silently ignored and can be properly handled by +// the calling code for retry logic or alerting. +func TestCreateCheckpointSweepTask_BulkSweepMarkersError(t *testing.T) { + wallet := &mockWalletService{} + vtxoRepo := &mockVtxoRepository{} + markerRepo := &mockMarkerRepository{} + repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} + builder := &mockTxBuilder{} + scheduler := &mockScheduler{} + + s := newSweeper(wallet, repoManager, builder, scheduler, "") + + checkpointTxid := "checkpoint_error" + vtxoOutpoint := domain.Outpoint{Txid: "vtxo_error", VOut: 0} + + childOutpoints := []domain.Outpoint{{Txid: "child_err", VOut: 0}} + vtxos := []domain.Vtxo{{ + Outpoint: childOutpoints[0], + MarkerIDs: []string{"marker-err"}, + Depth: 0, + }} + + toSweep := ports.TxInput{Txid: checkpointTxid, Index: 0, Value: 1000} + + builder.On("BuildSweepTx", []ports.TxInput{toSweep}). + Return("sweeptxid_err", "sweeptx_hex", nil) + + wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). + Return("sweeptxid_err", nil) + + vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). + Return(childOutpoints, nil) + + vtxoRepo.On("GetVtxos", mock.Anything, childOutpoints). + Return(vtxos, nil) + + // Simulate a database error during bulk sweep + dbError := fmt.Errorf("database connection failed") + markerRepo.On("BulkSweepMarkers", mock.Anything, mock.Anything, mock.Anything). + Return(dbError) + + task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) + err := task() + + // Verify the error is propagated + require.Error(t, err) + require.Contains(t, err.Error(), "database connection failed") +} + +// TestCreateCheckpointSweepTask_GetVtxosError verifies that when GetVtxos fails to +// retrieve the VTXOs associated with child outpoints, the error is properly propagated. +// This tests the error handling path before marker collection even begins. +func TestCreateCheckpointSweepTask_GetVtxosError(t *testing.T) { + wallet := &mockWalletService{} + vtxoRepo := &mockVtxoRepository{} + markerRepo := &mockMarkerRepository{} + repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} + builder := &mockTxBuilder{} + scheduler := &mockScheduler{} + + s := newSweeper(wallet, repoManager, builder, scheduler, "") + + checkpointTxid := "checkpoint_vtxo_err" + vtxoOutpoint := domain.Outpoint{Txid: "vtxo_vtxo_err", VOut: 0} + + childOutpoints := []domain.Outpoint{{Txid: "child_vtxo_err", VOut: 0}} + + toSweep := ports.TxInput{Txid: checkpointTxid, Index: 0, Value: 1000} + + builder.On("BuildSweepTx", []ports.TxInput{toSweep}). + Return("sweeptxid_vtxo_err", "sweeptx_hex", nil) + + wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). + Return("sweeptxid_vtxo_err", nil) + + vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). + Return(childOutpoints, nil) + + // Simulate error when fetching VTXOs + vtxoRepo.On("GetVtxos", mock.Anything, childOutpoints). + Return(nil, fmt.Errorf("vtxo not found in database")) + + task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) + err := task() + + // Verify the error is propagated + require.Error(t, err) + require.Contains(t, err.Error(), "vtxo not found") + + // BulkSweepMarkers should never be called since we failed earlier + markerRepo.AssertNotCalled(t, "BulkSweepMarkers", mock.Anything, mock.Anything, mock.Anything) +} + +// TestCreateCheckpointSweepTask_GetAllChildrenVtxosError verifies that when +// GetAllChildrenVtxos fails to retrieve child outpoints, the error is propagated. +// This tests the earliest error handling path in the sweep task. +func TestCreateCheckpointSweepTask_GetAllChildrenVtxosError(t *testing.T) { + wallet := &mockWalletService{} + vtxoRepo := &mockVtxoRepository{} + markerRepo := &mockMarkerRepository{} + repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} + builder := &mockTxBuilder{} + scheduler := &mockScheduler{} + + s := newSweeper(wallet, repoManager, builder, scheduler, "") + + checkpointTxid := "checkpoint_children_err" + vtxoOutpoint := domain.Outpoint{Txid: "vtxo_children_err", VOut: 0} + + toSweep := ports.TxInput{Txid: checkpointTxid, Index: 0, Value: 1000} + + builder.On("BuildSweepTx", []ports.TxInput{toSweep}). + Return("sweeptxid_children_err", "sweeptx_hex", nil) + + wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). + Return("sweeptxid_children_err", nil) + + // Simulate error when fetching children + vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). + Return(nil, fmt.Errorf("failed to query children vtxos")) + + task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) + err := task() + + // Verify the error is propagated + require.Error(t, err) + require.Contains(t, err.Error(), "failed to query children") + + // Neither GetVtxos nor BulkSweepMarkers should be called + vtxoRepo.AssertNotCalled(t, "GetVtxos", mock.Anything, mock.Anything) + markerRepo.AssertNotCalled(t, "BulkSweepMarkers", mock.Anything, mock.Anything, mock.Anything) +} + +// TestCreateCheckpointSweepTask_BuildSweepTxError verifies that when BuildSweepTx +// fails to create the sweep transaction, the error is propagated and no marker +// operations are attempted. This tests the very first error handling path. +func TestCreateCheckpointSweepTask_BuildSweepTxError(t *testing.T) { + wallet := &mockWalletService{} + vtxoRepo := &mockVtxoRepository{} + markerRepo := &mockMarkerRepository{} + repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} + builder := &mockTxBuilder{} + scheduler := &mockScheduler{} + + s := newSweeper(wallet, repoManager, builder, scheduler, "") + + checkpointTxid := "checkpoint_build_err" + vtxoOutpoint := domain.Outpoint{Txid: "vtxo_build_err", VOut: 0} + + toSweep := ports.TxInput{Txid: checkpointTxid, Index: 0, Value: 1000} + + // Simulate error when building sweep tx + builder.On("BuildSweepTx", []ports.TxInput{toSweep}). + Return("", "", fmt.Errorf("insufficient funds for sweep")) + + task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) + err := task() + + // Verify the error is propagated + require.Error(t, err) + require.Contains(t, err.Error(), "insufficient funds") + + // No other operations should be called + wallet.AssertNotCalled(t, "BroadcastTransaction", mock.Anything, mock.Anything) + vtxoRepo.AssertNotCalled(t, "GetAllChildrenVtxos", mock.Anything, mock.Anything) + markerRepo.AssertNotCalled(t, "BulkSweepMarkers", mock.Anything, mock.Anything, mock.Anything) +} + +// TestCreateCheckpointSweepTask_BroadcastError verifies that when BroadcastTransaction +// fails, the error is propagated and marker sweep operations are not attempted. +// This ensures we don't mark VTXOs as swept if the sweep transaction wasn't actually broadcast. +func TestCreateCheckpointSweepTask_BroadcastError(t *testing.T) { + wallet := &mockWalletService{} + vtxoRepo := &mockVtxoRepository{} + markerRepo := &mockMarkerRepository{} + repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} + builder := &mockTxBuilder{} + scheduler := &mockScheduler{} + + s := newSweeper(wallet, repoManager, builder, scheduler, "") + + checkpointTxid := "checkpoint_broadcast_err" + vtxoOutpoint := domain.Outpoint{Txid: "vtxo_broadcast_err", VOut: 0} + + toSweep := ports.TxInput{Txid: checkpointTxid, Index: 0, Value: 1000} + + builder.On("BuildSweepTx", []ports.TxInput{toSweep}). + Return("sweeptxid_broadcast_err", "sweeptx_hex", nil) + + // Simulate broadcast failure + wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). + Return("", fmt.Errorf("network timeout")) + + task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) + err := task() + + // Verify the error is propagated + require.Error(t, err) + require.Contains(t, err.Error(), "network timeout") + + // Marker operations should not be attempted since broadcast failed + vtxoRepo.AssertNotCalled(t, "GetAllChildrenVtxos", mock.Anything, mock.Anything) + markerRepo.AssertNotCalled(t, "BulkSweepMarkers", mock.Anything, mock.Anything, mock.Anything) +} + +// TestCreateCheckpointSweepTask_NoChildrenVtxos verifies that when +// GetAllChildrenVtxos returns an empty slice (no children under the unrolled +// vtxo), the sweeper does not attempt to fetch VTXOs or sweep markers. +func TestCreateCheckpointSweepTask_NoChildrenVtxos(t *testing.T) { + wallet := &mockWalletService{} + vtxoRepo := &mockVtxoRepository{} + markerRepo := &mockMarkerRepository{} + repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} + builder := &mockTxBuilder{} + scheduler := &mockScheduler{} + + s := newSweeper(wallet, repoManager, builder, scheduler, "") + + checkpointTxid := "checkpoint_no_children" + vtxoOutpoint := domain.Outpoint{Txid: "vtxo_no_children", VOut: 0} + + toSweep := ports.TxInput{Txid: checkpointTxid, Index: 0, Value: 5000} + + builder.On("BuildSweepTx", []ports.TxInput{toSweep}). + Return("sweeptxid_nc", "sweeptx_hex", nil) + + wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). + Return("sweeptxid_nc", nil) + + // No children found + vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). + Return([]domain.Outpoint{}, nil) + + // GetVtxos called with empty slice returns empty + vtxoRepo.On("GetVtxos", mock.Anything, []domain.Outpoint{}). + Return([]domain.Vtxo{}, nil) + + task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) + err := task() + + require.NoError(t, err) + wallet.AssertExpectations(t) + vtxoRepo.AssertExpectations(t) + // BulkSweepMarkers should NOT be called since there are no VTXOs/markers + markerRepo.AssertNotCalled(t, "BulkSweepMarkers", mock.Anything, mock.Anything, mock.Anything) +} + +// TestCreateCheckpointSweepTask_DuplicateMarkersAcrossVtxos verifies that when +// all VTXOs share the exact same marker set (100% overlap), only the unique +// markers are passed to BulkSweepMarkers. For example, 5 VTXOs each carrying +// {"marker-X", "marker-Y"} should result in exactly 2 markers being swept. +func TestCreateCheckpointSweepTask_DuplicateMarkersAcrossVtxos(t *testing.T) { + wallet := &mockWalletService{} + vtxoRepo := &mockVtxoRepository{} + markerRepo := &mockMarkerRepository{} + repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} + builder := &mockTxBuilder{} + scheduler := &mockScheduler{} + + s := newSweeper(wallet, repoManager, builder, scheduler, "") + + checkpointTxid := "checkpoint_dup" + vtxoOutpoint := domain.Outpoint{Txid: "vtxo_dup", VOut: 0} + + // 5 children, all sharing the identical marker set + childOutpoints := []domain.Outpoint{ + {Txid: "dup1", VOut: 0}, + {Txid: "dup2", VOut: 0}, + {Txid: "dup3", VOut: 0}, + {Txid: "dup4", VOut: 0}, + {Txid: "dup5", VOut: 0}, + } + + identicalMarkers := []string{"marker-X", "marker-Y"} + vtxosWithDupMarkers := make([]domain.Vtxo, len(childOutpoints)) + for i, op := range childOutpoints { + vtxosWithDupMarkers[i] = domain.Vtxo{ + Outpoint: op, + MarkerIDs: identicalMarkers, + Depth: 50, + } + } + + toSweep := ports.TxInput{Txid: checkpointTxid, Index: 0, Value: 25000} + + builder.On("BuildSweepTx", []ports.TxInput{toSweep}). + Return("sweeptxid_dup", "sweeptx_hex", nil) + + wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). + Return("sweeptxid_dup", nil) + + vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). + Return(childOutpoints, nil) + + vtxoRepo.On("GetVtxos", mock.Anything, childOutpoints). + Return(vtxosWithDupMarkers, nil) + + // Despite 5 VTXOs, only 2 unique markers should be swept + markerRepo.On("BulkSweepMarkers", mock.Anything, mock.MatchedBy(func(markerIDs []string) bool { + if len(markerIDs) != 2 { + return false + } + markerSet := make(map[string]bool) + for _, id := range markerIDs { + markerSet[id] = true + } + return markerSet["marker-X"] && markerSet["marker-Y"] + }), mock.AnythingOfType("int64")).Return(nil) + + task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) + err := task() + + require.NoError(t, err) + markerRepo.AssertExpectations(t) +} + +// TestCreateCheckpointSweepTask_LargeMarkerSet verifies that the map-based +// deduplication works correctly at scale: 120 VTXOs carrying a mix of 60 +// unique markers. This ensures no scaling issues with the map allocation +// or iteration, and that the deduplicated set is passed correctly to +// BulkSweepMarkers. +func TestCreateCheckpointSweepTask_LargeMarkerSet(t *testing.T) { + wallet := &mockWalletService{} + vtxoRepo := &mockVtxoRepository{} + markerRepo := &mockMarkerRepository{} + repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} + builder := &mockTxBuilder{} + scheduler := &mockScheduler{} + + s := newSweeper(wallet, repoManager, builder, scheduler, "") + + checkpointTxid := "checkpoint_large" + vtxoOutpoint := domain.Outpoint{Txid: "vtxo_large", VOut: 0} + + // 120 VTXOs, each with 2 markers drawn from a pool of 60 + childOutpoints := make([]domain.Outpoint, 120) + vtxosLarge := make([]domain.Vtxo, 120) + expectedMarkers := make(map[string]bool) + + for i := range 120 { + txid := fmt.Sprintf("large-child-%d", i) + childOutpoints[i] = domain.Outpoint{Txid: txid, VOut: 0} + + // Each VTXO gets two markers: marker-{i%60} and marker-{(i+1)%60} + m1 := fmt.Sprintf("marker-%d", i%60) + m2 := fmt.Sprintf("marker-%d", (i+1)%60) + expectedMarkers[m1] = true + expectedMarkers[m2] = true + + vtxosLarge[i] = domain.Vtxo{ + Outpoint: childOutpoints[i], + MarkerIDs: []string{m1, m2}, + Depth: uint32(i * 2), + } + } + + toSweep := ports.TxInput{Txid: checkpointTxid, Index: 0, Value: 1200000} + + builder.On("BuildSweepTx", []ports.TxInput{toSweep}). + Return("sweeptxid_large", "sweeptx_hex", nil) + + wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). + Return("sweeptxid_large", nil) + + vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). + Return(childOutpoints, nil) + + vtxoRepo.On("GetVtxos", mock.Anything, childOutpoints). + Return(vtxosLarge, nil) + + // Should have exactly 60 unique markers (marker-0 through marker-59) + markerRepo.On("BulkSweepMarkers", mock.Anything, mock.MatchedBy(func(markerIDs []string) bool { + if len(markerIDs) != 60 { + return false + } + seen := make(map[string]bool) + for _, id := range markerIDs { + if seen[id] { + return false // duplicate found — dedup failed + } + seen[id] = true + if !expectedMarkers[id] { + return false // unexpected marker + } + } + return true + }), mock.AnythingOfType("int64")).Return(nil) + + task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) + err := task() + + require.NoError(t, err) + markerRepo.AssertExpectations(t) +} diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index 1b3102356..479d690e1 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -193,6 +193,13 @@ func TestService(t *testing.T) { testMarkerDepthRangeQueries(t, svc) testMarkerChainTraversal(t, svc) testGetVtxoChainWithMarkerOptimization(t, svc) + testBulkSweepMarkersConcurrent(t, svc) + testCreateRootMarkersForVtxos(t, svc) + testMarkerCreationAtBoundaryDepth(t, svc) + testMarkerInheritanceAtNonBoundary(t, svc) + testDustVtxoMarkersSweptImmediately(t, svc) + testSweepVtxosWithMarkersEmptyInput(t, svc) + testSweepVtxosWithMarkersNoMarkersOnVtxos(t, svc) testOffchainTxRepository(t, svc) testScheduledSessionRepository(t, svc) testConvictionRepository(t, svc) @@ -2320,6 +2327,626 @@ func testGetVtxoChainWithMarkerOptimization(t *testing.T, svc ports.RepoManager) }) } +// testBulkSweepMarkersConcurrent tests that BulkSweepMarkers is thread-safe +// when multiple goroutines attempt to sweep the same markers concurrently. +// This verifies: +// 1. No race conditions occur with concurrent sweeps +// 2. Idempotency is maintained (same markers can be swept multiple times safely) +// 3. All markers end up in the correct swept state +func testBulkSweepMarkersConcurrent(t *testing.T, svc ports.RepoManager) { + t.Run("test_bulk_sweep_markers_concurrent", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + + // Create 20 markers to sweep concurrently + numMarkers := 20 + markers := make([]domain.Marker, numMarkers) + markerIDs := make([]string, numMarkers) + for i := 0; i < numMarkers; i++ { + markers[i] = domain.Marker{ + ID: "concurrent_marker_" + randomString(16), + Depth: uint32(i * 100), + ParentMarkerIDs: nil, + } + if i > 0 { + markers[i].ParentMarkerIDs = []string{markers[i-1].ID} + } + markerIDs[i] = markers[i].ID + } + + // Add all markers + for _, m := range markers { + err := svc.Markers().AddMarker(ctx, m) + require.NoError(t, err) + } + + // Verify none are swept initially + for _, id := range markerIDs { + isSwept, err := svc.Markers().IsMarkerSwept(ctx, id) + require.NoError(t, err) + require.False(t, isSwept, "Marker %s should not be swept initially", id) + } + + // Launch concurrent goroutines to sweep the same markers + numGoroutines := 10 + sweptAt := time.Now().UnixMilli() + + var wg sync.WaitGroup + errChan := make(chan error, numGoroutines) + + for i := 0; i < numGoroutines; i++ { + wg.Add(1) + go func(goroutineID int) { + defer wg.Done() + // Each goroutine sweeps all markers with slightly different timestamp + err := svc.Markers().BulkSweepMarkers(ctx, markerIDs, sweptAt+int64(goroutineID)) + if err != nil { + errChan <- err + } + }(i) + } + + wg.Wait() + close(errChan) + + // Check for errors from goroutines + for err := range errChan { + require.NoError(t, err, "BulkSweepMarkers should not error on concurrent calls") + } + + // Verify all markers are now swept + for _, id := range markerIDs { + isSwept, err := svc.Markers().IsMarkerSwept(ctx, id) + require.NoError(t, err) + require.True(t, isSwept, "Marker %s should be swept after concurrent operations", id) + } + + // Verify swept markers can be retrieved + sweptMarkers, err := svc.Markers().GetSweptMarkers(ctx, markerIDs) + require.NoError(t, err) + require.Len(t, sweptMarkers, numMarkers) + }) + + t.Run("test_bulk_sweep_overlapping_marker_sets", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + + // Create 30 markers + numMarkers := 30 + markers := make([]domain.Marker, numMarkers) + markerIDs := make([]string, numMarkers) + for i := 0; i < numMarkers; i++ { + markers[i] = domain.Marker{ + ID: "overlap_marker_" + randomString(16), + Depth: uint32(i * 50), + ParentMarkerIDs: nil, + } + markerIDs[i] = markers[i].ID + } + + // Add all markers + for _, m := range markers { + err := svc.Markers().AddMarker(ctx, m) + require.NoError(t, err) + } + + // Create overlapping subsets + // Set A: markers 0-19 + // Set B: markers 10-29 + // Overlap: markers 10-19 + setA := markerIDs[0:20] + setB := markerIDs[10:30] + + sweptAt := time.Now().UnixMilli() + + var wg sync.WaitGroup + errChan := make(chan error, 2) + + // Sweep set A and set B concurrently + wg.Add(2) + go func() { + defer wg.Done() + if err := svc.Markers().BulkSweepMarkers(ctx, setA, sweptAt); err != nil { + errChan <- err + } + }() + go func() { + defer wg.Done() + if err := svc.Markers().BulkSweepMarkers(ctx, setB, sweptAt+1); err != nil { + errChan <- err + } + }() + + wg.Wait() + close(errChan) + + // Check for errors + for err := range errChan { + require.NoError(t, err, "BulkSweepMarkers should handle overlapping sets") + } + + // Verify all markers are swept + for _, id := range markerIDs { + isSwept, err := svc.Markers().IsMarkerSwept(ctx, id) + require.NoError(t, err) + require.True(t, isSwept, "Marker %s should be swept", id) + } + }) + + t.Run("test_bulk_sweep_empty_and_non_empty_concurrent", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + + // Create 5 markers + markers := make([]domain.Marker, 5) + markerIDs := make([]string, 5) + for i := 0; i < 5; i++ { + markers[i] = domain.Marker{ + ID: "empty_nonempty_marker_" + randomString(16), + Depth: uint32(i * 100), + ParentMarkerIDs: nil, + } + markerIDs[i] = markers[i].ID + err := svc.Markers().AddMarker(ctx, markers[i]) + require.NoError(t, err) + } + + sweptAt := time.Now().UnixMilli() + + var wg sync.WaitGroup + errChan := make(chan error, 4) + + // Mix of empty and non-empty sweeps concurrently + wg.Add(4) + go func() { + defer wg.Done() + if err := svc.Markers().BulkSweepMarkers(ctx, markerIDs, sweptAt); err != nil { + errChan <- err + } + }() + go func() { + defer wg.Done() + // Empty slice should not error + if err := svc.Markers().BulkSweepMarkers(ctx, []string{}, sweptAt); err != nil { + errChan <- err + } + }() + go func() { + defer wg.Done() + if err := svc.Markers().BulkSweepMarkers(ctx, markerIDs[0:2], sweptAt); err != nil { + errChan <- err + } + }() + go func() { + defer wg.Done() + // Empty slice again + if err := svc.Markers().BulkSweepMarkers(ctx, []string{}, sweptAt); err != nil { + errChan <- err + } + }() + + wg.Wait() + close(errChan) + + for err := range errChan { + require.NoError(t, err) + } + + // All markers should be swept + for _, id := range markerIDs { + isSwept, err := svc.Markers().IsMarkerSwept(ctx, id) + require.NoError(t, err) + require.True(t, isSwept, "Marker %s should be swept", id) + } + }) + + t.Run("test_bulk_sweep_idempotency_rapid_fire", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + + // Create a single marker and sweep it many times concurrently + marker := domain.Marker{ + ID: "rapid_fire_marker_" + randomString(16), + Depth: 0, + ParentMarkerIDs: nil, + } + err := svc.Markers().AddMarker(ctx, marker) + require.NoError(t, err) + + sweptAt := time.Now().UnixMilli() + + // Launch 50 concurrent sweeps on the same marker + numSweeps := 50 + var wg sync.WaitGroup + errChan := make(chan error, numSweeps) + + for i := 0; i < numSweeps; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + if err := svc.Markers(). + BulkSweepMarkers(ctx, []string{marker.ID}, sweptAt+int64(idx)); err != nil { + errChan <- err + } + }(i) + } + + wg.Wait() + close(errChan) + + for err := range errChan { + require.NoError(t, err, "Rapid-fire sweeps should all succeed") + } + + // Verify marker is swept and only one record exists + isSwept, err := svc.Markers().IsMarkerSwept(ctx, marker.ID) + require.NoError(t, err) + require.True(t, isSwept) + + // Get swept markers should return exactly 1 entry + sweptMarkers, err := svc.Markers().GetSweptMarkers(ctx, []string{marker.ID}) + require.NoError(t, err) + require.Len(t, sweptMarkers, 1) + }) +} + +func testCreateRootMarkersForVtxos(t *testing.T, svc ports.RepoManager) { + t.Run("test_create_root_markers_for_vtxos", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + commitmentTxid := randomString(32) + + // Create batch VTXOs at depth 0 with MarkerIDs = outpoint.String() + vtxo1 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 1000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 0, + MarkerIDs: nil, // will be set to outpoint.String() by convention + } + vtxo2 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 2000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 0, + MarkerIDs: nil, + } + vtxo3 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 1}, + PubKey: pubkey, + Amount: 3000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 0, + MarkerIDs: nil, + } + + // Set MarkerIDs to outpoint.String() as the service does for batch VTXOs + vtxo1.MarkerIDs = []string{vtxo1.Outpoint.String()} + vtxo2.MarkerIDs = []string{vtxo2.Outpoint.String()} + vtxo3.MarkerIDs = []string{vtxo3.Outpoint.String()} + + vtxos := []domain.Vtxo{vtxo1, vtxo2, vtxo3} + + // Add VTXOs first + err := svc.Vtxos().AddVtxos(ctx, vtxos) + require.NoError(t, err) + + // Create root markers + err = svc.Markers().CreateRootMarkersForVtxos(ctx, vtxos) + require.NoError(t, err) + + // Verify each VTXO got a root marker with ID = outpoint.String() + for _, vtxo := range vtxos { + expectedMarkerID := vtxo.Outpoint.String() + marker, err := svc.Markers().GetMarker(ctx, expectedMarkerID) + require.NoError(t, err) + require.NotNil(t, marker, "root marker should exist for vtxo %s", expectedMarkerID) + require.Equal(t, expectedMarkerID, marker.ID) + require.Equal(t, uint32(0), marker.Depth) + require.Empty(t, marker.ParentMarkerIDs, "root markers should have no parents") + } + + // Idempotency: calling again should not error + err = svc.Markers().CreateRootMarkersForVtxos(ctx, vtxos) + require.NoError(t, err) + }) +} + +func testMarkerCreationAtBoundaryDepth(t *testing.T, svc ports.RepoManager) { + t.Run("test_marker_creation_at_boundary_depth", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + commitmentTxid := randomString(32) + + // Create parent VTXOs at depth 99 with root markers + parentMarkerID := "root_boundary_" + randomString(16) + parentMarker := domain.Marker{ + ID: parentMarkerID, + Depth: 0, + ParentMarkerIDs: nil, + } + err := svc.Markers().AddMarker(ctx, parentMarker) + require.NoError(t, err) + + parentVtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 5000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 99, + MarkerIDs: []string{parentMarkerID}, + } + err = svc.Vtxos().AddVtxos(ctx, []domain.Vtxo{parentVtxo}) + require.NoError(t, err) + + // Simulate offchain tx: child at depth 100 (marker boundary) + newDepth := uint32(100) + require.True(t, domain.IsAtMarkerBoundary(newDepth)) + + // Collect parent markers (mimics service logic) + parentMarkerIDs := parentVtxo.MarkerIDs + + // Create new marker at boundary + newMarkerID := "boundary_marker_" + randomString(16) + newMarker := domain.Marker{ + ID: newMarkerID, + Depth: newDepth, + ParentMarkerIDs: parentMarkerIDs, + } + err = svc.Markers().AddMarker(ctx, newMarker) + require.NoError(t, err) + + // Create child VTXO with the new marker + childVtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 4500, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid, randomString(32)}, + Depth: newDepth, + MarkerIDs: []string{newMarkerID}, + } + err = svc.Vtxos().AddVtxos(ctx, []domain.Vtxo{childVtxo}) + require.NoError(t, err) + + // Verify the marker was created correctly + retrieved, err := svc.Markers().GetMarker(ctx, newMarkerID) + require.NoError(t, err) + require.NotNil(t, retrieved) + require.Equal(t, newDepth, retrieved.Depth) + require.ElementsMatch(t, parentMarkerIDs, retrieved.ParentMarkerIDs) + + // Verify the child VTXO has only the new marker (not parent markers) + childVtxos, err := svc.Vtxos().GetVtxos(ctx, []domain.Outpoint{childVtxo.Outpoint}) + require.NoError(t, err) + require.Len(t, childVtxos, 1) + require.Equal(t, []string{newMarkerID}, childVtxos[0].MarkerIDs) + require.Equal(t, newDepth, childVtxos[0].Depth) + }) +} + +func testMarkerInheritanceAtNonBoundary(t *testing.T, svc ports.RepoManager) { + t.Run("test_marker_inheritance_at_non_boundary", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + commitmentTxid := randomString(32) + + // Create two parent VTXOs at depth 50 with different markers + markerA := "inherit_marker_A_" + randomString(16) + markerB := "inherit_marker_B_" + randomString(16) + + err := svc.Markers().AddMarker(ctx, domain.Marker{ + ID: markerA, Depth: 0, ParentMarkerIDs: nil, + }) + require.NoError(t, err) + err = svc.Markers().AddMarker(ctx, domain.Marker{ + ID: markerB, Depth: 0, ParentMarkerIDs: nil, + }) + require.NoError(t, err) + + parent1 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 3000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 50, + MarkerIDs: []string{markerA}, + } + parent2 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 2000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 50, + MarkerIDs: []string{markerB}, + } + + err = svc.Vtxos().AddVtxos(ctx, []domain.Vtxo{parent1, parent2}) + require.NoError(t, err) + + // Child at depth 51 (NOT a boundary) should inherit both parent markers + newDepth := uint32(51) + require.False(t, domain.IsAtMarkerBoundary(newDepth)) + + inheritedMarkers := []string{markerA, markerB} + childVtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 4500, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid, randomString(32)}, + Depth: newDepth, + MarkerIDs: inheritedMarkers, + } + err = svc.Vtxos().AddVtxos(ctx, []domain.Vtxo{childVtxo}) + require.NoError(t, err) + + // Verify the child VTXO inherited both parent markers + childVtxos, err := svc.Vtxos().GetVtxos(ctx, []domain.Outpoint{childVtxo.Outpoint}) + require.NoError(t, err) + require.Len(t, childVtxos, 1) + require.ElementsMatch(t, inheritedMarkers, childVtxos[0].MarkerIDs) + require.Equal(t, newDepth, childVtxos[0].Depth) + + // No new marker should have been created for this depth + // (verify by checking there's no marker with this child's txid) + nonExistent, err := svc.Markers().GetMarker(ctx, childVtxo.Outpoint.String()) + require.NoError(t, err) + require.Nil(t, nonExistent) + }) +} + +func testDustVtxoMarkersSweptImmediately(t *testing.T, svc ports.RepoManager) { + t.Run("test_dust_vtxo_markers_swept_immediately", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + + // Create markers that represent dust VTXOs (outpoint-based IDs) + dustOutpoint1 := domain.Outpoint{Txid: randomString(32), VOut: 0} + dustOutpoint2 := domain.Outpoint{Txid: randomString(32), VOut: 1} + + dustMarkerID1 := dustOutpoint1.String() + dustMarkerID2 := dustOutpoint2.String() + + // Add root markers for these dust VTXOs + err := svc.Markers().AddMarker(ctx, domain.Marker{ + ID: dustMarkerID1, Depth: 0, ParentMarkerIDs: nil, + }) + require.NoError(t, err) + err = svc.Markers().AddMarker(ctx, domain.Marker{ + ID: dustMarkerID2, Depth: 0, ParentMarkerIDs: nil, + }) + require.NoError(t, err) + + // Verify they are NOT swept initially + isSwept, err := svc.Markers().IsMarkerSwept(ctx, dustMarkerID1) + require.NoError(t, err) + require.False(t, isSwept) + + isSwept, err = svc.Markers().IsMarkerSwept(ctx, dustMarkerID2) + require.NoError(t, err) + require.False(t, isSwept) + + // Simulate the dust sweep that happens in updateProjectionsAfterOffchainTxEvents: + // BulkSweepMarkers is called immediately for dust VTXOs + sweptAt := time.Now().Unix() + err = svc.Markers().BulkSweepMarkers(ctx, []string{dustMarkerID1, dustMarkerID2}, sweptAt) + require.NoError(t, err) + + // Verify both dust markers are now swept + isSwept, err = svc.Markers().IsMarkerSwept(ctx, dustMarkerID1) + require.NoError(t, err) + require.True(t, isSwept, "dust marker 1 should be swept immediately") + + isSwept, err = svc.Markers().IsMarkerSwept(ctx, dustMarkerID2) + require.NoError(t, err) + require.True(t, isSwept, "dust marker 2 should be swept immediately") + + // Verify swept records have correct timestamp + sweptMarkers, err := svc.Markers(). + GetSweptMarkers(ctx, []string{dustMarkerID1, dustMarkerID2}) + require.NoError(t, err) + require.Len(t, sweptMarkers, 2) + for _, sm := range sweptMarkers { + require.Equal(t, sweptAt, sm.SweptAt) + } + }) +} + +func testSweepVtxosWithMarkersEmptyInput(t *testing.T, svc ports.RepoManager) { + t.Run("test_sweep_vtxos_with_markers_empty_input", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + + // Simulate what sweepVtxosWithMarkers does with empty input: + // it should return early without touching the DB. + vtxoOutpoints := []domain.Outpoint{} + + // Empty outpoints → nothing to fetch, nothing to sweep + require.Empty(t, vtxoOutpoints) + + // BulkSweepMarkers with empty slice should not error + err := svc.Markers().BulkSweepMarkers(ctx, []string{}, time.Now().Unix()) + require.NoError(t, err) + }) +} + +func testSweepVtxosWithMarkersNoMarkersOnVtxos(t *testing.T, svc ports.RepoManager) { + t.Run("test_sweep_vtxos_with_markers_no_markers_on_vtxos", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + commitmentTxid := randomString(32) + + // Create VTXOs with empty MarkerIDs (legacy / edge case) + vtxo1 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 1000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 0, + MarkerIDs: []string{}, // empty + } + vtxo2 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 2000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 0, + MarkerIDs: nil, // nil + } + + err := svc.Vtxos().AddVtxos(ctx, []domain.Vtxo{vtxo1, vtxo2}) + require.NoError(t, err) + + // Simulate sweepVtxosWithMarkers logic: + // fetch VTXOs, collect markers, if no markers → return 0 + vtxos, err := svc.Vtxos().GetVtxos(ctx, []domain.Outpoint{vtxo1.Outpoint, vtxo2.Outpoint}) + require.NoError(t, err) + require.Len(t, vtxos, 2) + + // Collect unique markers (should be empty) + uniqueMarkers := make(map[string]struct{}) + for _, vtxo := range vtxos { + for _, markerID := range vtxo.MarkerIDs { + uniqueMarkers[markerID] = struct{}{} + } + } + + // No markers to sweep → would return 0 in sweepVtxosWithMarkers + require.Empty(t, uniqueMarkers, "VTXOs with no markers should yield empty marker set") + }) +} + func testScheduledSessionRepository(t *testing.T, svc ports.RepoManager) { t.Run("test_scheduled_session_repository", func(t *testing.T) { ctx := context.Background() From bb6d7dcc4ef497962239a10ea8c14d9b5e3f7197 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 12 Feb 2026 15:51:22 -0500 Subject: [PATCH 18/54] more tests --- internal/core/application/indexer_test.go | 85 +++++ internal/core/application/service_test.go | 89 +++++ internal/core/application/utils_test.go | 166 +++++++++ internal/infrastructure/db/service_test.go | 336 ++++++++++++++++++ .../interface/grpc/handlers/parser_test.go | 212 +++++++++++ 5 files changed, 888 insertions(+) create mode 100644 internal/core/application/utils_test.go create mode 100644 internal/interface/grpc/handlers/parser_test.go diff --git a/internal/core/application/indexer_test.go b/internal/core/application/indexer_test.go index 4b1b3bc80..b6ba91889 100644 --- a/internal/core/application/indexer_test.go +++ b/internal/core/application/indexer_test.go @@ -863,3 +863,88 @@ func TestGetVtxosFromCacheOrDB_EmptyOutpoints(t *testing.T) { // DB should never be called for empty input vtxoRepo.AssertNotCalled(t, "GetVtxos", mock.Anything, mock.Anything) } + +// TestPrefetchVtxosByMarkers_CycleInMarkerDAG verifies that the BFS in +// prefetchVtxosByMarkers terminates when there is a cycle in the marker DAG +// (marker-A → parent marker-B → parent marker-A). +func TestPrefetchVtxosByMarkers_CycleInMarkerDAG(t *testing.T) { + vtxoRepo := &mockVtxoRepoForIndexer{} + markerRepo := &mockMarkerRepoForIndexer{} + repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} + + indexer := &indexerService{repoManager: repoManager} + + ctx := context.Background() + startKey := Outpoint{Txid: "cycle-vtxo", VOut: 0} + + // Starting VTXO references marker-A + startVtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "cycle-vtxo", VOut: 0}, + MarkerIDs: []string{"marker-A"}, + Depth: 200, + } + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: "cycle-vtxo", VOut: 0}}). + Return([]domain.Vtxo{startVtxo}, nil) + + // marker-A points to marker-B as parent + markerRepo.On("GetMarker", ctx, "marker-A").Return(&domain.Marker{ + ID: "marker-A", + Depth: 200, + ParentMarkerIDs: []string{"marker-B"}, + }, nil) + + // marker-B points BACK to marker-A (cycle!) + markerRepo.On("GetMarker", ctx, "marker-B").Return(&domain.Marker{ + ID: "marker-B", + Depth: 100, + ParentMarkerIDs: []string{"marker-A"}, + }, nil) + + // Both markers should be collected despite the cycle + markerRepo.On("GetVtxoChainByMarkers", ctx, mock.MatchedBy(func(ids []string) bool { + if len(ids) != 2 { + return false + } + idSet := make(map[string]bool) + for _, id := range ids { + idSet[id] = true + } + return idSet["marker-A"] && idSet["marker-B"] + })).Return([]domain.Vtxo{ + {Outpoint: domain.Outpoint{Txid: "chain-vtxo-1", VOut: 0}, Depth: 150}, + }, nil) + + cache := indexer.prefetchVtxosByMarkers(ctx, startKey) + + // Should terminate and contain the start VTXO + chain VTXO + require.Len(t, cache, 2) + require.Contains(t, cache, "cycle-vtxo:0") + require.Contains(t, cache, "chain-vtxo-1:0") + + // Each marker should be visited exactly once + markerRepo.AssertNumberOfCalls(t, "GetMarker", 2) +} + +// TestPrefetchVtxosByMarkers_StartVtxoNotFound verifies that when the starting +// VTXO is not found in the database, an empty cache is returned. +func TestPrefetchVtxosByMarkers_StartVtxoNotFound(t *testing.T) { + vtxoRepo := &mockVtxoRepoForIndexer{} + markerRepo := &mockMarkerRepoForIndexer{} + repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} + + indexer := &indexerService{repoManager: repoManager} + + ctx := context.Background() + startKey := Outpoint{Txid: "nonexistent", VOut: 0} + + // GetVtxos returns empty slice (VTXO not found) + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: "nonexistent", VOut: 0}}). + Return([]domain.Vtxo{}, nil) + + cache := indexer.prefetchVtxosByMarkers(ctx, startKey) + + require.Empty(t, cache) + // Marker repo should never be touched + markerRepo.AssertNotCalled(t, "GetMarker", mock.Anything, mock.Anything) + markerRepo.AssertNotCalled(t, "GetVtxoChainByMarkers", mock.Anything, mock.Anything) +} diff --git a/internal/core/application/service_test.go b/internal/core/application/service_test.go index bde130c78..297d2e76c 100644 --- a/internal/core/application/service_test.go +++ b/internal/core/application/service_test.go @@ -481,3 +481,92 @@ func TestMarkerCreationAtBoundary(t *testing.T) { }) } } + +// TestAllNewVtxosGetSameDepth verifies that when a single offchain tx produces +// multiple output VTXOs, all of them receive the same depth (max parent depth + 1) +// and the same marker IDs. This mirrors the logic in updateProjectionsAfterOffchainTxEvents +// where newDepth is computed once and applied to all new VTXOs from the same tx. +func TestAllNewVtxosGetSameDepth(t *testing.T) { + testCases := []struct { + name string + parentDepths []uint32 + parentMarkerSets [][]string + numOutputVtxos int + expectedDepth uint32 + expectedMarkerLen int + description string + }{ + { + name: "3 outputs from single parent at depth 0", + parentDepths: []uint32{0}, + parentMarkerSets: [][]string{{"root-marker-1"}}, + numOutputVtxos: 3, + expectedDepth: 1, + expectedMarkerLen: 1, + description: "all 3 outputs get depth 1 and inherit root marker", + }, + { + name: "5 outputs from two parents at different depths", + parentDepths: []uint32{30, 50}, + parentMarkerSets: [][]string{{"marker-A"}, {"marker-B", "marker-C"}}, + numOutputVtxos: 5, + expectedDepth: 51, + expectedMarkerLen: 3, + description: "all 5 outputs get depth 51 (max+1) and inherit union of markers", + }, + { + name: "2 outputs at marker boundary", + parentDepths: []uint32{99}, + parentMarkerSets: [][]string{{"root-marker"}}, + numOutputVtxos: 2, + expectedDepth: 100, + expectedMarkerLen: 1, + description: "both outputs get depth 100 and the same new marker", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + spentVtxos := make([]domain.Vtxo, len(tc.parentDepths)) + for i, depth := range tc.parentDepths { + spentVtxos[i] = domain.Vtxo{ + Depth: depth, + MarkerIDs: tc.parentMarkerSets[i], + } + } + + maxDepth := calculateMaxDepth(spentVtxos) + newDepth := maxDepth + 1 + require.Equal(t, tc.expectedDepth, newDepth) + + parentMarkers := collectParentMarkers(spentVtxos) + markerIDs, _ := deriveMarkerIDs(newDepth, parentMarkers, "tx-with-multiple-outputs") + + // Simulate creating multiple output VTXOs — each gets the same depth and markers + outputs := make([]domain.Vtxo, tc.numOutputVtxos) + for i := 0; i < tc.numOutputVtxos; i++ { + outputs[i] = domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "tx-with-multiple-outputs", VOut: uint32(i)}, + Depth: newDepth, + MarkerIDs: markerIDs, + } + } + + // All outputs must have the same depth + for i, v := range outputs { + require.Equal(t, tc.expectedDepth, v.Depth, + "output %d has wrong depth", i) + } + + // All outputs must have the same marker IDs + for i := 1; i < len(outputs); i++ { + sort.Strings(outputs[0].MarkerIDs) + sort.Strings(outputs[i].MarkerIDs) + require.Equal(t, outputs[0].MarkerIDs, outputs[i].MarkerIDs, + "output %d has different markers than output 0", i) + } + + require.Len(t, outputs[0].MarkerIDs, tc.expectedMarkerLen, tc.description) + }) + } +} diff --git a/internal/core/application/utils_test.go b/internal/core/application/utils_test.go new file mode 100644 index 000000000..e47e8620f --- /dev/null +++ b/internal/core/application/utils_test.go @@ -0,0 +1,166 @@ +package application + +import ( + "testing" + + "github.com/arkade-os/arkd/internal/core/domain" + "github.com/arkade-os/arkd/pkg/ark-lib/tree" + "github.com/btcsuite/btcd/btcec/v2" + "github.com/btcsuite/btcd/btcec/v2/schnorr" + "github.com/btcsuite/btcd/btcutil/psbt" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/wire" + "github.com/stretchr/testify/require" +) + +// makeP2TRLeafTx creates a valid base64-encoded PSBT with P2TR outputs +// for the given schnorr public keys and amounts. +func makeP2TRLeafTx(t *testing.T, outputs []struct { + pubkey *btcec.PublicKey + amount int64 +}) string { + t.Helper() + hash, err := chainhash.NewHashFromStr( + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + ) + require.NoError(t, err) + + txOuts := make([]*wire.TxOut, 0, len(outputs)) + for _, out := range outputs { + pkScript := make([]byte, 34) + pkScript[0] = 0x51 // OP_1 + pkScript[1] = 0x20 // 32-byte push + copy(pkScript[2:], schnorr.SerializePubKey(out.pubkey)) + + txOuts = append(txOuts, &wire.TxOut{ + Value: out.amount, + PkScript: pkScript, + }) + } + + ptx, err := psbt.New( + []*wire.OutPoint{{Hash: *hash, Index: 0}}, + txOuts, + 3, + 0, + []uint32{wire.MaxTxInSequenceNum}, + ) + require.NoError(t, err) + + b64, err := ptx.B64Encode() + require.NoError(t, err) + return b64 +} + +func TestGetNewVtxosFromRound_MarkerIDsAndDepth(t *testing.T) { + // Generate two distinct keys for two outputs + privKey1, err := btcec.NewPrivateKey() + require.NoError(t, err) + privKey2, err := btcec.NewPrivateKey() + require.NoError(t, err) + + pub1 := privKey1.PubKey() + pub2 := privKey2.PubKey() + + leafTx := makeP2TRLeafTx(t, []struct { + pubkey *btcec.PublicKey + amount int64 + }{ + {pubkey: pub1, amount: 50000}, + {pubkey: pub2, amount: 30000}, + }) + + round := &domain.Round{ + CommitmentTxid: "test-commitment-txid", + VtxoTreeExpiration: 3600, + EndingTimestamp: 1700000000, + Stage: domain.Stage{Code: int(domain.RoundFinalizationStage), Ended: true}, + VtxoTree: tree.FlatTxTree{ + { + Txid: "leaf-tx-id", + Tx: leafTx, + Children: nil, // leaf node + }, + }, + } + + vtxos := getNewVtxosFromRound(round) + + require.Len(t, vtxos, 2) + + for i, vtxo := range vtxos { + // All batch VTXOs must have Depth = 0 + require.Equal(t, uint32(0), vtxo.Depth, "vtxo %d should have depth 0", i) + + // MarkerIDs must be exactly []string{outpoint.String()} + expectedMarkerID := vtxo.Outpoint.String() + require.Equal(t, []string{expectedMarkerID}, vtxo.MarkerIDs, + "vtxo %d MarkerIDs should be [outpoint.String()]", i) + + // CommitmentTxids should reference the round's commitment + require.Equal(t, []string{"test-commitment-txid"}, vtxo.CommitmentTxids) + require.Equal(t, "test-commitment-txid", vtxo.RootCommitmentTxid) + + // Amount must match + if i == 0 { + require.Equal(t, uint64(50000), vtxo.Amount) + } else { + require.Equal(t, uint64(30000), vtxo.Amount) + } + + // PubKey must be non-empty + require.NotEmpty(t, vtxo.PubKey) + } + + // VOut should be sequential (0, 1) + require.Equal(t, uint32(0), vtxos[0].VOut) + require.Equal(t, uint32(1), vtxos[1].VOut) + + // Both should have the same txid (from the same PSBT) + require.Equal(t, vtxos[0].Txid, vtxos[1].Txid) +} + +func TestGetNewVtxosFromRound_EmptyVtxoTree(t *testing.T) { + round := &domain.Round{ + CommitmentTxid: "empty-round", + VtxoTree: nil, + } + + vtxos := getNewVtxosFromRound(round) + require.Nil(t, vtxos) +} + +func TestGetNewVtxosFromRound_SingleOutput(t *testing.T) { + privKey, err := btcec.NewPrivateKey() + require.NoError(t, err) + + leafTx := makeP2TRLeafTx(t, []struct { + pubkey *btcec.PublicKey + amount int64 + }{ + {pubkey: privKey.PubKey(), amount: 100000}, + }) + + round := &domain.Round{ + CommitmentTxid: "single-output-commitment", + VtxoTreeExpiration: 7200, + EndingTimestamp: 1700000000, + Stage: domain.Stage{Code: int(domain.RoundFinalizationStage), Ended: true}, + VtxoTree: tree.FlatTxTree{ + { + Txid: "single-leaf", + Tx: leafTx, + Children: nil, + }, + }, + } + + vtxos := getNewVtxosFromRound(round) + require.Len(t, vtxos, 1) + + vtxo := vtxos[0] + require.Equal(t, uint32(0), vtxo.Depth) + require.Equal(t, []string{vtxo.Outpoint.String()}, vtxo.MarkerIDs) + require.Equal(t, uint64(100000), vtxo.Amount) + require.Equal(t, uint32(0), vtxo.VOut) +} diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index 479d690e1..0d9f1d892 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -200,6 +200,10 @@ func TestService(t *testing.T) { testDustVtxoMarkersSweptImmediately(t, svc) testSweepVtxosWithMarkersEmptyInput(t, svc) testSweepVtxosWithMarkersNoMarkersOnVtxos(t, svc) + testVtxoMarkerIDsRoundTrip(t, svc) + testGetVtxosByArkTxidMultipleOutputs(t, svc) + testCreateRootMarkersForEmptyVtxos(t, svc) + testSweepVtxosWithMarkersIntegration(t, svc) testOffchainTxRepository(t, svc) testScheduledSessionRepository(t, svc) testConvictionRepository(t, svc) @@ -2947,6 +2951,198 @@ func testSweepVtxosWithMarkersNoMarkersOnVtxos(t *testing.T, svc ports.RepoManag }) } +func testVtxoMarkerIDsRoundTrip(t *testing.T, svc ports.RepoManager) { + t.Run("test_vtxo_marker_ids_round_trip", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + commitmentTxid := randomString(32) + + // VTXOs with various MarkerIDs configurations + vtxoSingle := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 1000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 0, + MarkerIDs: []string{"single-marker"}, + } + vtxoMulti := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 2000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 150, + MarkerIDs: []string{"marker-A", "marker-B", "marker-C"}, + } + vtxoEmpty := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 3000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 0, + MarkerIDs: []string{}, + } + vtxoNil := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 4000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 0, + MarkerIDs: nil, + } + vtxoDeep := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: randomString(32), VOut: 0}, + PubKey: pubkey, + Amount: 5000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Depth: 500, + MarkerIDs: []string{"marker-500", "marker-400"}, + } + + allVtxos := []domain.Vtxo{vtxoSingle, vtxoMulti, vtxoEmpty, vtxoNil, vtxoDeep} + err := svc.Vtxos().AddVtxos(ctx, allVtxos) + require.NoError(t, err) + + // Retrieve all and verify + outpoints := make([]domain.Outpoint, len(allVtxos)) + for i, v := range allVtxos { + outpoints[i] = v.Outpoint + } + retrieved, err := svc.Vtxos().GetVtxos(ctx, outpoints) + require.NoError(t, err) + require.Len(t, retrieved, 5) + + byOutpoint := make(map[string]domain.Vtxo) + for _, v := range retrieved { + byOutpoint[v.Outpoint.String()] = v + } + + // Single marker + got := byOutpoint[vtxoSingle.Outpoint.String()] + require.Equal(t, uint32(0), got.Depth) + require.Equal(t, []string{"single-marker"}, got.MarkerIDs) + + // Multiple markers — order may vary, use ElementsMatch + got = byOutpoint[vtxoMulti.Outpoint.String()] + require.Equal(t, uint32(150), got.Depth) + require.ElementsMatch(t, []string{"marker-A", "marker-B", "marker-C"}, got.MarkerIDs) + + // Empty markers — should come back as empty or nil (both acceptable) + got = byOutpoint[vtxoEmpty.Outpoint.String()] + require.Equal(t, uint32(0), got.Depth) + require.Empty(t, got.MarkerIDs) + + // Nil markers — should come back as empty or nil + got = byOutpoint[vtxoNil.Outpoint.String()] + require.Empty(t, got.MarkerIDs) + + // Deep VTXO with two markers + got = byOutpoint[vtxoDeep.Outpoint.String()] + require.Equal(t, uint32(500), got.Depth) + require.ElementsMatch(t, []string{"marker-500", "marker-400"}, got.MarkerIDs) + }) +} + +func testGetVtxosByArkTxidMultipleOutputs(t *testing.T, svc ports.RepoManager) { + t.Run("test_get_vtxos_by_ark_txid_multiple_outputs", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + commitmentTxid := randomString(32) + + // An ark txid producing multiple VTXOs (different vouts) at the same depth + arkTxid := randomString(32) + sharedMarkers := []string{"shared-marker-" + randomString(8)} + sharedDepth := uint32(100) + + vtxoOut0 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: arkTxid, VOut: 0}, + PubKey: pubkey, + Amount: 1000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Preconfirmed: true, + ArkTxid: arkTxid, + Depth: sharedDepth, + MarkerIDs: sharedMarkers, + } + vtxoOut1 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: arkTxid, VOut: 1}, + PubKey: pubkey2, + Amount: 2000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Preconfirmed: true, + ArkTxid: arkTxid, + Depth: sharedDepth, + MarkerIDs: sharedMarkers, + } + vtxoOut2 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: arkTxid, VOut: 2}, + PubKey: pubkey, + Amount: 500, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + Preconfirmed: true, + ArkTxid: arkTxid, + Depth: sharedDepth, + MarkerIDs: sharedMarkers, + } + + err := svc.Vtxos().AddVtxos(ctx, []domain.Vtxo{vtxoOut0, vtxoOut1, vtxoOut2}) + require.NoError(t, err) + + // Query by ark txid + results, err := svc.Markers().GetVtxosByArkTxid(ctx, arkTxid) + require.NoError(t, err) + require.Len(t, results, 3) + + // Verify all outputs are returned with correct depth and markers + for _, v := range results { + require.Equal(t, arkTxid, v.Txid) + require.Equal(t, sharedDepth, v.Depth) + require.ElementsMatch(t, sharedMarkers, v.MarkerIDs) + } + + // Verify all vouts are present + vouts := make([]uint32, len(results)) + for i, v := range results { + vouts[i] = v.VOut + } + require.ElementsMatch(t, []uint32{0, 1, 2}, vouts) + + // Non-existent ark txid returns empty + empty, err := svc.Markers().GetVtxosByArkTxid(ctx, "nonexistent") + require.NoError(t, err) + require.Empty(t, empty) + }) +} + +func testCreateRootMarkersForEmptyVtxos(t *testing.T, svc ports.RepoManager) { + t.Run("test_create_root_markers_for_empty_vtxos", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + + // Empty slice should not error and have no side effects + err := svc.Markers().CreateRootMarkersForVtxos(ctx, []domain.Vtxo{}) + require.NoError(t, err) + + // Nil slice should also not error + err = svc.Markers().CreateRootMarkersForVtxos(ctx, nil) + require.NoError(t, err) + }) +} + func testScheduledSessionRepository(t *testing.T, svc ports.RepoManager) { t.Run("test_scheduled_session_repository", func(t *testing.T) { ctx := context.Background() @@ -3465,6 +3661,146 @@ func randomTx() string { return b64 } +// testSweepVtxosWithMarkersIntegration tests the full marker-based sweep flow: +// create VTXOs with markers, then bulk sweep the markers and verify VTXOs +// appear as swept via the marker-based view. +func testSweepVtxosWithMarkersIntegration(t *testing.T, svc ports.RepoManager) { + t.Run("test_sweep_vtxos_with_markers_integration", func(t *testing.T) { + ctx := context.Background() + + // Create a finalized round so VTXOs have a valid commitment txid + roundId := uuid.New().String() + commitmentTxid := randomString(32) + now := time.Now() + round := domain.NewRoundFromEvents([]domain.Event{ + domain.RoundStarted{ + RoundEvent: domain.RoundEvent{Id: roundId, Type: domain.EventTypeRoundStarted}, + Timestamp: now.Unix(), + }, + domain.RoundFinalizationStarted{ + RoundEvent: domain.RoundEvent{Id: roundId, Type: domain.EventTypeRoundFinalizationStarted}, + CommitmentTxid: commitmentTxid, + CommitmentTx: emptyTx, + VtxoTree: vtxoTree, + Connectors: connectorsTree, + VtxoTreeExpiration: 3600, + }, + domain.RoundFinalized{ + RoundEvent: domain.RoundEvent{Id: roundId, Type: domain.EventTypeRoundFinalized}, + FinalCommitmentTx: emptyTx, + Timestamp: now.Unix(), + }, + }) + require.NoError(t, svc.Rounds().AddOrUpdateRound(ctx, *round)) + + // Create 3 VTXOs, two sharing a marker and one with its own + txidA := randomString(32) + txidB := randomString(32) + txidC := randomString(32) + sharedMarkerID := "shared-marker-sweep-" + randomString(8) + uniqueMarkerID := "unique-marker-sweep-" + randomString(8) + + vtxosToAdd := []domain.Vtxo{ + { + Outpoint: domain.Outpoint{Txid: txidA, VOut: 0}, + PubKey: pubkey, + Amount: 1000, + CommitmentTxids: []string{commitmentTxid}, + RootCommitmentTxid: commitmentTxid, + CreatedAt: time.Now().Unix(), + ExpiresAt: time.Now().Add(time.Hour).Unix(), + Depth: 50, + MarkerIDs: []string{sharedMarkerID}, + }, + { + Outpoint: domain.Outpoint{Txid: txidB, VOut: 0}, + PubKey: pubkey, + Amount: 2000, + CommitmentTxids: []string{commitmentTxid}, + RootCommitmentTxid: commitmentTxid, + CreatedAt: time.Now().Unix(), + ExpiresAt: time.Now().Add(time.Hour).Unix(), + Depth: 50, + MarkerIDs: []string{sharedMarkerID}, + }, + { + Outpoint: domain.Outpoint{Txid: txidC, VOut: 0}, + PubKey: pubkey, + Amount: 3000, + CommitmentTxids: []string{commitmentTxid}, + RootCommitmentTxid: commitmentTxid, + CreatedAt: time.Now().Unix(), + ExpiresAt: time.Now().Add(time.Hour).Unix(), + Depth: 75, + MarkerIDs: []string{uniqueMarkerID}, + }, + } + require.NoError(t, svc.Vtxos().AddVtxos(ctx, vtxosToAdd)) + + // Create the markers + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: sharedMarkerID, + Depth: 50, + })) + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: uniqueMarkerID, + Depth: 75, + })) + + // Associate VTXOs with their markers + require.NoError(t, svc.Markers().UpdateVtxoMarkers(ctx, + domain.Outpoint{Txid: txidA, VOut: 0}, []string{sharedMarkerID})) + require.NoError(t, svc.Markers().UpdateVtxoMarkers(ctx, + domain.Outpoint{Txid: txidB, VOut: 0}, []string{sharedMarkerID})) + require.NoError(t, svc.Markers().UpdateVtxoMarkers(ctx, + domain.Outpoint{Txid: txidC, VOut: 0}, []string{uniqueMarkerID})) + + // Verify VTXOs are not swept before + fetchedBefore, err := svc.Vtxos().GetVtxos(ctx, []domain.Outpoint{ + {Txid: txidA, VOut: 0}, {Txid: txidB, VOut: 0}, {Txid: txidC, VOut: 0}, + }) + require.NoError(t, err) + require.Len(t, fetchedBefore, 3) + for _, v := range fetchedBefore { + require.False(t, v.Swept, "vtxo %s should not be swept yet", v.Txid) + } + + // Simulate sweepVtxosWithMarkers: collect unique markers, then bulk sweep + uniqueMarkers := make(map[string]struct{}) + for _, vtxo := range vtxosToAdd { + for _, markerID := range vtxo.MarkerIDs { + uniqueMarkers[markerID] = struct{}{} + } + } + markerIDs := make([]string, 0, len(uniqueMarkers)) + for markerID := range uniqueMarkers { + markerIDs = append(markerIDs, markerID) + } + require.Len(t, markerIDs, 2, "should deduplicate to 2 unique markers") + + sweptAt := time.Now().Unix() + require.NoError(t, svc.Markers().BulkSweepMarkers(ctx, markerIDs, sweptAt)) + + // Verify all VTXOs now appear as swept + fetchedAfter, err := svc.Vtxos().GetVtxos(ctx, []domain.Outpoint{ + {Txid: txidA, VOut: 0}, {Txid: txidB, VOut: 0}, {Txid: txidC, VOut: 0}, + }) + require.NoError(t, err) + require.Len(t, fetchedAfter, 3) + for _, v := range fetchedAfter { + require.True(t, v.Swept, "vtxo %s should be swept", v.Txid) + } + + // Verify both markers are recorded as swept + sweptMarkers, err := svc.Markers().GetSweptMarkers(ctx, markerIDs) + require.NoError(t, err) + require.Len(t, sweptMarkers, 2) + for _, sm := range sweptMarkers { + require.Equal(t, sweptAt, sm.SweptAt) + } + }) +} + type sortVtxos []domain.Vtxo func (a sortVtxos) String() string { diff --git a/internal/interface/grpc/handlers/parser_test.go b/internal/interface/grpc/handlers/parser_test.go new file mode 100644 index 000000000..486a99192 --- /dev/null +++ b/internal/interface/grpc/handlers/parser_test.go @@ -0,0 +1,212 @@ +package handlers + +import ( + "testing" + + "github.com/arkade-os/arkd/internal/core/application" + "github.com/arkade-os/arkd/internal/core/domain" + "github.com/stretchr/testify/require" +) + +func TestVtxoListToProto_DepthAndNewFields(t *testing.T) { + vtxos := vtxoList{ + { + Outpoint: domain.Outpoint{Txid: "aaa", VOut: 0}, + PubKey: "25a43cecfa0e1b1a4f72d64ad15f4cfa7a84d0723e8511c969aa543638ea9967", + Amount: 50000, + CommitmentTxids: []string{"commit-1"}, + Spent: false, + ExpiresAt: 1700000000, + SpentBy: "spender-tx", + Swept: false, + Preconfirmed: true, + Unrolled: false, + CreatedAt: 1699000000, + SettledBy: "settler-tx", + ArkTxid: "ark-tx-1", + Depth: 42, + }, + { + Outpoint: domain.Outpoint{Txid: "bbb", VOut: 1}, + PubKey: "33ffb3dee353b1a9ebe4ced64b946238d0a4ac364f275d771da6ad2445d07ae0", + Amount: 100000, + CommitmentTxids: []string{"commit-2", "commit-3"}, + Spent: true, + ExpiresAt: 1700100000, + SpentBy: "", + Swept: true, + Preconfirmed: false, + Unrolled: true, + CreatedAt: 1699100000, + SettledBy: "", + ArkTxid: "", + Depth: 200, + }, + { + Outpoint: domain.Outpoint{Txid: "ccc", VOut: 2}, + PubKey: "25a43cecfa0e1b1a4f72d64ad15f4cfa7a84d0723e8511c969aa543638ea9967", + Amount: 0, + Depth: 0, + }, + } + + protos := vtxos.toProto() + require.Len(t, protos, 3) + + // First VTXO: all fields populated + p0 := protos[0] + require.Equal(t, "aaa", p0.Outpoint.Txid) + require.Equal(t, uint32(0), p0.Outpoint.Vout) + require.Equal(t, uint64(50000), p0.Amount) + require.Equal(t, []string{"commit-1"}, p0.CommitmentTxids) + require.False(t, p0.IsSpent) + require.Equal(t, int64(1700000000), p0.ExpiresAt) + require.Equal(t, "spender-tx", p0.SpentBy) + require.False(t, p0.IsSwept) + require.True(t, p0.IsPreconfirmed) + require.False(t, p0.IsUnrolled) + require.Equal(t, int64(1699000000), p0.CreatedAt) + require.Equal(t, "settler-tx", p0.SettledBy) + require.Equal(t, "ark-tx-1", p0.ArkTxid) + require.Equal(t, uint32(42), p0.Depth) + require.Equal(t, "512025a43cecfa0e1b1a4f72d64ad15f4cfa7a84d0723e8511c969aa543638ea9967", p0.Script) + + // Second VTXO: different depth, spent/swept/unrolled flags + p1 := protos[1] + require.Equal(t, "bbb", p1.Outpoint.Txid) + require.Equal(t, uint32(1), p1.Outpoint.Vout) + require.Equal(t, uint32(200), p1.Depth) + require.True(t, p1.IsSpent) + require.True(t, p1.IsSwept) + require.True(t, p1.IsUnrolled) + require.Equal(t, []string{"commit-2", "commit-3"}, p1.CommitmentTxids) + + // Third VTXO: zero depth (batch vtxo) + p2 := protos[2] + require.Equal(t, uint32(0), p2.Depth) + require.Equal(t, uint64(0), p2.Amount) +} + +func TestNewIndexerVtxo_DepthMapping(t *testing.T) { + vtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "idx-tx", VOut: 3}, + PubKey: "25a43cecfa0e1b1a4f72d64ad15f4cfa7a84d0723e8511c969aa543638ea9967", + Amount: 75000, + CommitmentTxids: []string{"commit-a"}, + CreatedAt: 1699500000, + ExpiresAt: 1700500000, + Preconfirmed: true, + Swept: false, + Unrolled: false, + Spent: false, + SpentBy: "spender", + SettledBy: "settler", + ArkTxid: "ark-tx-idx", + Depth: 150, + } + + proto := newIndexerVtxo(vtxo) + + require.Equal(t, "idx-tx", proto.Outpoint.Txid) + require.Equal(t, uint32(3), proto.Outpoint.Vout) + require.Equal(t, uint64(75000), proto.Amount) + require.Equal(t, int64(1699500000), proto.CreatedAt) + require.Equal(t, int64(1700500000), proto.ExpiresAt) + require.True(t, proto.IsPreconfirmed) + require.False(t, proto.IsSwept) + require.False(t, proto.IsUnrolled) + require.False(t, proto.IsSpent) + require.Equal(t, "spender", proto.SpentBy) + require.Equal(t, "settler", proto.SettledBy) + require.Equal(t, "ark-tx-idx", proto.ArkTxid) + require.Equal(t, uint32(150), proto.Depth) + require.Equal(t, []string{"commit-a"}, proto.CommitmentTxids) + require.Equal(t, "512025a43cecfa0e1b1a4f72d64ad15f4cfa7a84d0723e8511c969aa543638ea9967", proto.Script) +} + +func TestNewIndexerVtxo_ZeroDepth(t *testing.T) { + vtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "batch-tx", VOut: 0}, + PubKey: "25a43cecfa0e1b1a4f72d64ad15f4cfa7a84d0723e8511c969aa543638ea9967", + Depth: 0, + } + + proto := newIndexerVtxo(vtxo) + require.Equal(t, uint32(0), proto.Depth) +} + +func TestTxEventToProto_DepthPreserved(t *testing.T) { + event := txEvent{ + TxData: application.TxData{ + Tx: "raw-tx-data", + Txid: "event-txid", + }, + SpentVtxos: []domain.Vtxo{ + { + Outpoint: domain.Outpoint{Txid: "spent-1", VOut: 0}, + PubKey: "25a43cecfa0e1b1a4f72d64ad15f4cfa7a84d0723e8511c969aa543638ea9967", + Depth: 99, + Amount: 10000, + }, + }, + SpendableVtxos: []domain.Vtxo{ + { + Outpoint: domain.Outpoint{Txid: "new-1", VOut: 0}, + PubKey: "33ffb3dee353b1a9ebe4ced64b946238d0a4ac364f275d771da6ad2445d07ae0", + Depth: 100, + Amount: 9000, + }, + { + Outpoint: domain.Outpoint{Txid: "new-1", VOut: 1}, + PubKey: "33ffb3dee353b1a9ebe4ced64b946238d0a4ac364f275d771da6ad2445d07ae0", + Depth: 100, + Amount: 500, + }, + }, + CheckpointTxs: map[string]application.TxData{ + "cp-1": {Txid: "cp-txid-1", Tx: "cp-raw-1"}, + }, + } + + proto := event.toProto() + + require.Equal(t, "event-txid", proto.Txid) + require.Equal(t, "raw-tx-data", proto.Tx) + + // Spent VTXOs preserve depth + require.Len(t, proto.SpentVtxos, 1) + require.Equal(t, uint32(99), proto.SpentVtxos[0].Depth) + require.Equal(t, "spent-1", proto.SpentVtxos[0].Outpoint.Txid) + + // Spendable VTXOs preserve depth + require.Len(t, proto.SpendableVtxos, 2) + require.Equal(t, uint32(100), proto.SpendableVtxos[0].Depth) + require.Equal(t, uint32(100), proto.SpendableVtxos[1].Depth) + + // Checkpoint txs mapped correctly + require.Len(t, proto.CheckpointTxs, 1) + require.Equal(t, "cp-txid-1", proto.CheckpointTxs["cp-1"].Txid) + require.Equal(t, "cp-raw-1", proto.CheckpointTxs["cp-1"].Tx) +} + +func TestTxEventToProto_EmptyCheckpointTxs(t *testing.T) { + event := txEvent{ + TxData: application.TxData{ + Txid: "simple-event", + }, + SpentVtxos: []domain.Vtxo{ + { + Outpoint: domain.Outpoint{Txid: "s1", VOut: 0}, + PubKey: "25a43cecfa0e1b1a4f72d64ad15f4cfa7a84d0723e8511c969aa543638ea9967", + Depth: 0, + }, + }, + SpendableVtxos: []domain.Vtxo{}, + } + + proto := event.toProto() + require.Nil(t, proto.CheckpointTxs) + require.Len(t, proto.SpentVtxos, 1) + require.Equal(t, uint32(0), proto.SpentVtxos[0].Depth) + require.Empty(t, proto.SpendableVtxos) +} From 35c1944fc421402a6ef887ea882f3d0c4e2ee4c3 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 12 Feb 2026 16:05:34 -0500 Subject: [PATCH 19/54] 20k depth test --- internal/core/application/indexer_test.go | 77 ++++++++++ internal/core/application/service_test.go | 92 ++++++++++++ internal/core/application/utils_test.go | 4 +- internal/infrastructure/db/service_test.go | 138 +++++++++++++++++- .../interface/grpc/handlers/parser_test.go | 12 +- 5 files changed, 317 insertions(+), 6 deletions(-) diff --git a/internal/core/application/indexer_test.go b/internal/core/application/indexer_test.go index b6ba91889..e7bcf6ec1 100644 --- a/internal/core/application/indexer_test.go +++ b/internal/core/application/indexer_test.go @@ -948,3 +948,80 @@ func TestPrefetchVtxosByMarkers_StartVtxoNotFound(t *testing.T) { markerRepo.AssertNotCalled(t, "GetMarker", mock.Anything, mock.Anything) markerRepo.AssertNotCalled(t, "GetVtxoChainByMarkers", mock.Anything, mock.Anything) } + +// TestPrefetchVtxosByMarkers_Depth20k verifies that the BFS traversal in +// prefetchVtxosByMarkers correctly handles a VTXO at depth 20000 with a chain +// of 200 markers (one every 100 depths). This is the target maximum depth. +func TestPrefetchVtxosByMarkers_Depth20k(t *testing.T) { + vtxoRepo := &mockVtxoRepoForIndexer{} + markerRepo := &mockMarkerRepoForIndexer{} + repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} + + indexer := &indexerService{repoManager: repoManager} + + ctx := context.Background() + startKey := Outpoint{Txid: "deep-20k-vtxo", VOut: 0} + + const maxDepth = 20000 + const markerInterval = 100 + const numMarkers = maxDepth / markerInterval // 200 markers + + // Starting VTXO at depth 20000 with marker at depth 20000 + startVtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "deep-20k-vtxo", VOut: 0}, + MarkerIDs: []string{fmt.Sprintf("marker-%d", maxDepth)}, + Depth: maxDepth, + } + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: "deep-20k-vtxo", VOut: 0}}). + Return([]domain.Vtxo{startVtxo}, nil) + + // Build the 200-marker chain: marker-20000 -> marker-19900 -> ... -> marker-100 -> marker-0 + for depth := uint32(maxDepth); depth > 0; depth -= markerInterval { + parentDepth := depth - markerInterval + markerID := fmt.Sprintf("marker-%d", depth) + parentMarkerID := fmt.Sprintf("marker-%d", parentDepth) + markerRepo.On("GetMarker", ctx, markerID).Return(&domain.Marker{ + ID: markerID, + Depth: depth, + ParentMarkerIDs: []string{parentMarkerID}, + }, nil) + } + // Root marker at depth 0 has no parents + markerRepo.On("GetMarker", ctx, "marker-0").Return(&domain.Marker{ + ID: "marker-0", + Depth: 0, + ParentMarkerIDs: []string{}, + }, nil) + + // Generate VTXOs that would be returned by GetVtxoChainByMarkers + // One VTXO per marker interval midpoint to simulate a populated chain + chainVtxos := make([]domain.Vtxo, 0, numMarkers) + for i := 0; i < numMarkers; i++ { + chainVtxos = append(chainVtxos, domain.Vtxo{ + Outpoint: domain.Outpoint{ + Txid: fmt.Sprintf("chain-vtxo-%d", i), + VOut: 0, + }, + Depth: uint32(i*markerInterval + 50), // midpoint of each interval + }) + } + + // All 201 markers (0, 100, 200, ..., 20000) should be collected + markerRepo.On("GetVtxoChainByMarkers", ctx, mock.MatchedBy(func(ids []string) bool { + return len(ids) == numMarkers+1 // 201 markers total + })).Return(chainVtxos, nil) + + cache := indexer.prefetchVtxosByMarkers(ctx, startKey) + + // 200 chain VTXOs + 1 start VTXO = 201 + require.Len(t, cache, numMarkers+1) + require.Contains(t, cache, "deep-20k-vtxo:0") + + // Verify a sample of chain VTXOs are in cache + require.Contains(t, cache, "chain-vtxo-0:0") + require.Contains(t, cache, "chain-vtxo-99:0") + require.Contains(t, cache, "chain-vtxo-199:0") + + // All 201 markers should have been visited via GetMarker (200 non-root + 1 root) + markerRepo.AssertNumberOfCalls(t, "GetMarker", numMarkers+1) +} diff --git a/internal/core/application/service_test.go b/internal/core/application/service_test.go index 297d2e76c..6b65bdb7e 100644 --- a/internal/core/application/service_test.go +++ b/internal/core/application/service_test.go @@ -570,3 +570,95 @@ func TestAllNewVtxosGetSameDepth(t *testing.T) { }) } } + +// TestDepth20k_MarkerBoundaryAndInheritance verifies marker behavior at the +// target maximum depth of 20000. Tests boundary transitions, inheritance with +// large marker sets, and depth calculation with deeply chained VTXOs. +func TestDepth20k_MarkerBoundaryAndInheritance(t *testing.T) { + t.Run("depth 19999 inherits markers, depth 20000 creates new marker", func(t *testing.T) { + // Parent at depth 19999 => child at 20000 (boundary) + parent := domain.Vtxo{Depth: 19999, MarkerIDs: []string{"marker-19900"}} + parentMarkers := collectParentMarkers([]domain.Vtxo{parent}) + + newDepth := calculateMaxDepth([]domain.Vtxo{parent}) + 1 + require.Equal(t, uint32(20000), newDepth) + require.True(t, domain.IsAtMarkerBoundary(newDepth)) + + markerIDs, createdMarker := deriveMarkerIDs(newDepth, parentMarkers, "tx-at-20k") + require.NotNil(t, createdMarker, "marker should be created at depth 20000") + require.Equal(t, uint32(20000), createdMarker.Depth) + require.Equal(t, []string{"marker-19900"}, createdMarker.ParentMarkerIDs) + require.Len(t, markerIDs, 1) + require.Equal(t, createdMarker.ID, markerIDs[0]) + }) + + t.Run("depth 20001 inherits markers from boundary parent", func(t *testing.T) { + parent := domain.Vtxo{Depth: 20000, MarkerIDs: []string{"marker-20000"}} + parentMarkers := collectParentMarkers([]domain.Vtxo{parent}) + + newDepth := calculateMaxDepth([]domain.Vtxo{parent}) + 1 + require.Equal(t, uint32(20001), newDepth) + require.False(t, domain.IsAtMarkerBoundary(newDepth)) + + markerIDs, createdMarker := deriveMarkerIDs(newDepth, parentMarkers, "tx-at-20001") + require.Nil(t, createdMarker, "no marker at non-boundary depth") + require.Equal(t, []string{"marker-20000"}, markerIDs) + }) + + t.Run("VTXO with 200 inherited markers from deep chain", func(t *testing.T) { + // Simulate a VTXO at depth 19950 that has accumulated 200 marker IDs + // from a chain where markers were created at every boundary + markers := make([]string, 200) + for i := range markers { + markers[i] = fmt.Sprintf("marker-%d", i*100) + } + + parent := domain.Vtxo{Depth: 19950, MarkerIDs: markers} + collected := collectParentMarkers([]domain.Vtxo{parent}) + sort.Strings(collected) + sort.Strings(markers) + require.Equal(t, markers, collected, "all 200 markers should be collected") + }) + + t.Run("multiple deep parents merge 200+ markers correctly", func(t *testing.T) { + // Two parents deep in the chain with overlapping markers + markersA := make([]string, 100) + markersB := make([]string, 150) + for i := range markersA { + markersA[i] = fmt.Sprintf("marker-%d", i*100) // 0, 100, ..., 9900 + } + for i := range markersB { + markersB[i] = fmt.Sprintf("marker-%d", i*100) // 0, 100, ..., 14900 + } + + parents := []domain.Vtxo{ + {Depth: 10000, MarkerIDs: markersA}, + {Depth: 15000, MarkerIDs: markersB}, + } + collected := collectParentMarkers(parents) + + // Union should be 150 unique markers (0..14900) + require.Len(t, collected, 150) + + newDepth := calculateMaxDepth(parents) + 1 + require.Equal(t, uint32(15001), newDepth) + require.False(t, domain.IsAtMarkerBoundary(newDepth)) + + markerIDs, createdMarker := deriveMarkerIDs(newDepth, collected, "merge-tx") + require.Nil(t, createdMarker) + require.Len(t, markerIDs, 150, "child inherits all 150 unique markers") + }) + + t.Run("depth calculation with max uint32 near boundary", func(t *testing.T) { + // Verify depth arithmetic doesn't overflow for large values + parent := domain.Vtxo{Depth: 20000, MarkerIDs: []string{"marker-20000"}} + newDepth := calculateMaxDepth([]domain.Vtxo{parent}) + 1 + require.Equal(t, uint32(20001), newDepth) + + // Depth 20100 should also be a boundary + require.True(t, domain.IsAtMarkerBoundary(20100)) + require.True(t, domain.IsAtMarkerBoundary(20200)) + require.False(t, domain.IsAtMarkerBoundary(20001)) + require.False(t, domain.IsAtMarkerBoundary(20099)) + }) +} diff --git a/internal/core/application/utils_test.go b/internal/core/application/utils_test.go index e47e8620f..7b84c821e 100644 --- a/internal/core/application/utils_test.go +++ b/internal/core/application/utils_test.go @@ -73,7 +73,7 @@ func TestGetNewVtxosFromRound_MarkerIDsAndDepth(t *testing.T) { round := &domain.Round{ CommitmentTxid: "test-commitment-txid", VtxoTreeExpiration: 3600, - EndingTimestamp: 1700000000, + EndingTimestamp: 1700000000, Stage: domain.Stage{Code: int(domain.RoundFinalizationStage), Ended: true}, VtxoTree: tree.FlatTxTree{ { @@ -144,7 +144,7 @@ func TestGetNewVtxosFromRound_SingleOutput(t *testing.T) { round := &domain.Round{ CommitmentTxid: "single-output-commitment", VtxoTreeExpiration: 7200, - EndingTimestamp: 1700000000, + EndingTimestamp: 1700000000, Stage: domain.Stage{Code: int(domain.RoundFinalizationStage), Ended: true}, VtxoTree: tree.FlatTxTree{ { diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index 0d9f1d892..5dc9953a7 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "encoding/hex" "encoding/json" + "fmt" "os" "reflect" "sort" @@ -204,6 +205,7 @@ func TestService(t *testing.T) { testGetVtxosByArkTxidMultipleOutputs(t, svc) testCreateRootMarkersForEmptyVtxos(t, svc) testSweepVtxosWithMarkersIntegration(t, svc) + testDeepChain20kMarkers(t, svc) testOffchainTxRepository(t, svc) testScheduledSessionRepository(t, svc) testConvictionRepository(t, svc) @@ -3661,6 +3663,132 @@ func randomTx() string { return b64 } +// testDeepChain20kMarkers creates a 200-marker chain (depth 0 to 20000) in the +// database, associates VTXOs at various depths, verifies GetVtxoChainByMarkers +// retrieves all VTXOs across the full chain, and then bulk sweeps all markers. +// This validates the system can handle the target maximum depth of 20000. +func testDeepChain20kMarkers(t *testing.T, svc ports.RepoManager) { + t.Run("test_deep_chain_20k_markers", func(t *testing.T) { + ctx := context.Background() + + const maxDepth = 20000 + const markerInterval = 100 + const numMarkers = maxDepth/markerInterval + 1 // 201 markers (0, 100, ..., 20000) + + // Create a round for VTXO commitment references + roundId := uuid.New().String() + commitmentTxid := randomString(32) + round := domain.NewRoundFromEvents([]domain.Event{ + domain.RoundStarted{ + RoundEvent: domain.RoundEvent{Id: roundId, Type: domain.EventTypeRoundStarted}, + Timestamp: time.Now().Unix(), + }, + domain.RoundFinalizationStarted{ + RoundEvent: domain.RoundEvent{ + Id: roundId, + Type: domain.EventTypeRoundFinalizationStarted, + }, + CommitmentTxid: commitmentTxid, + CommitmentTx: emptyTx, + VtxoTree: vtxoTree, + Connectors: connectorsTree, + VtxoTreeExpiration: 3600, + }, + domain.RoundFinalized{ + RoundEvent: domain.RoundEvent{ + Id: roundId, + Type: domain.EventTypeRoundFinalized, + }, + FinalCommitmentTx: emptyTx, + Timestamp: time.Now().Unix(), + }, + }) + require.NoError(t, svc.Rounds().AddOrUpdateRound(ctx, *round)) + + // Build the 201-marker chain: marker-0 (root) -> marker-100 -> ... -> marker-20000 + allMarkerIDs := make([]string, 0, numMarkers) + for depth := uint32(0); depth <= maxDepth; depth += markerInterval { + markerID := fmt.Sprintf("deep20k-%s-marker-%d", roundId[:8], depth) + allMarkerIDs = append(allMarkerIDs, markerID) + + var parentMarkerIDs []string + if depth > 0 { + parentMarkerIDs = []string{ + fmt.Sprintf("deep20k-%s-marker-%d", roundId[:8], depth-markerInterval), + } + } + + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: markerID, + Depth: depth, + ParentMarkerIDs: parentMarkerIDs, + })) + } + require.Len(t, allMarkerIDs, numMarkers) + + // Create VTXOs at selected depths across the chain: every 1000th depth + // Each VTXO is associated with the marker at the nearest boundary below it + vtxosToAdd := make([]domain.Vtxo, 0) + vtxoOutpoints := make([]domain.Outpoint, 0) + for depth := uint32(0); depth <= maxDepth; depth += 1000 { + txid := fmt.Sprintf("deep20k-%s-vtxo-%d", roundId[:8], depth) + outpoint := domain.Outpoint{Txid: txid, VOut: 0} + // Nearest marker at or below this depth + nearestMarkerDepth := (depth / markerInterval) * markerInterval + markerID := fmt.Sprintf("deep20k-%s-marker-%d", roundId[:8], nearestMarkerDepth) + + vtxosToAdd = append(vtxosToAdd, domain.Vtxo{ + Outpoint: outpoint, + PubKey: pubkey, + Amount: 1000, + CommitmentTxids: []string{commitmentTxid}, + RootCommitmentTxid: commitmentTxid, + CreatedAt: time.Now().Unix(), + ExpiresAt: time.Now().Add(time.Hour).Unix(), + Depth: depth, + MarkerIDs: []string{markerID}, + }) + vtxoOutpoints = append(vtxoOutpoints, outpoint) + } + require.NoError(t, svc.Vtxos().AddVtxos(ctx, vtxosToAdd)) + + // Associate each VTXO with its marker + for _, vtxo := range vtxosToAdd { + require.NoError(t, svc.Markers().UpdateVtxoMarkers(ctx, vtxo.Outpoint, vtxo.MarkerIDs)) + } + + // Verify: GetVtxoChainByMarkers with ALL markers returns ALL VTXOs + chainVtxos, err := svc.Markers().GetVtxoChainByMarkers(ctx, allMarkerIDs) + require.NoError(t, err) + require.Len(t, chainVtxos, len(vtxosToAdd), + "GetVtxoChainByMarkers should return all %d VTXOs across 200 markers", len(vtxosToAdd)) + + // Verify: VTXOs are not swept initially + fetchedVtxos, err := svc.Vtxos().GetVtxos(ctx, vtxoOutpoints) + require.NoError(t, err) + for _, v := range fetchedVtxos { + require.False(t, v.Swept, "vtxo at depth %d should not be swept yet", v.Depth) + } + + // Bulk sweep ALL 201 markers at once + sweptAt := time.Now().Unix() + require.NoError(t, svc.Markers().BulkSweepMarkers(ctx, allMarkerIDs, sweptAt)) + + // Verify: all VTXOs now appear as swept + fetchedAfter, err := svc.Vtxos().GetVtxos(ctx, vtxoOutpoints) + require.NoError(t, err) + for _, v := range fetchedAfter { + require.True(t, v.Swept, "vtxo at depth %d should be swept after bulk sweep", v.Depth) + } + + // Verify: all markers are recorded as swept + sweptMarkers, err := svc.Markers().GetSweptMarkers(ctx, allMarkerIDs) + require.NoError(t, err) + require.Len(t, sweptMarkers, numMarkers, + "all %d markers should be swept", numMarkers) + }) +} + // testSweepVtxosWithMarkersIntegration tests the full marker-based sweep flow: // create VTXOs with markers, then bulk sweep the markers and verify VTXOs // appear as swept via the marker-based view. @@ -3678,7 +3806,10 @@ func testSweepVtxosWithMarkersIntegration(t *testing.T, svc ports.RepoManager) { Timestamp: now.Unix(), }, domain.RoundFinalizationStarted{ - RoundEvent: domain.RoundEvent{Id: roundId, Type: domain.EventTypeRoundFinalizationStarted}, + RoundEvent: domain.RoundEvent{ + Id: roundId, + Type: domain.EventTypeRoundFinalizationStarted, + }, CommitmentTxid: commitmentTxid, CommitmentTx: emptyTx, VtxoTree: vtxoTree, @@ -3686,7 +3817,10 @@ func testSweepVtxosWithMarkersIntegration(t *testing.T, svc ports.RepoManager) { VtxoTreeExpiration: 3600, }, domain.RoundFinalized{ - RoundEvent: domain.RoundEvent{Id: roundId, Type: domain.EventTypeRoundFinalized}, + RoundEvent: domain.RoundEvent{ + Id: roundId, + Type: domain.EventTypeRoundFinalized, + }, FinalCommitmentTx: emptyTx, Timestamp: now.Unix(), }, diff --git a/internal/interface/grpc/handlers/parser_test.go b/internal/interface/grpc/handlers/parser_test.go index 486a99192..c1186bc1d 100644 --- a/internal/interface/grpc/handlers/parser_test.go +++ b/internal/interface/grpc/handlers/parser_test.go @@ -69,7 +69,11 @@ func TestVtxoListToProto_DepthAndNewFields(t *testing.T) { require.Equal(t, "settler-tx", p0.SettledBy) require.Equal(t, "ark-tx-1", p0.ArkTxid) require.Equal(t, uint32(42), p0.Depth) - require.Equal(t, "512025a43cecfa0e1b1a4f72d64ad15f4cfa7a84d0723e8511c969aa543638ea9967", p0.Script) + require.Equal( + t, + "512025a43cecfa0e1b1a4f72d64ad15f4cfa7a84d0723e8511c969aa543638ea9967", + p0.Script, + ) // Second VTXO: different depth, spent/swept/unrolled flags p1 := protos[1] @@ -121,7 +125,11 @@ func TestNewIndexerVtxo_DepthMapping(t *testing.T) { require.Equal(t, "ark-tx-idx", proto.ArkTxid) require.Equal(t, uint32(150), proto.Depth) require.Equal(t, []string{"commit-a"}, proto.CommitmentTxids) - require.Equal(t, "512025a43cecfa0e1b1a4f72d64ad15f4cfa7a84d0723e8511c969aa543638ea9967", proto.Script) + require.Equal( + t, + "512025a43cecfa0e1b1a4f72d64ad15f4cfa7a84d0723e8511c969aa543638ea9967", + proto.Script, + ) } func TestNewIndexerVtxo_ZeroDepth(t *testing.T) { From 38a7945ac8cc7a2b86a3b88f22da8dce5eebf400 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 12 Feb 2026 16:35:33 -0500 Subject: [PATCH 20/54] more integration tests in service_test.go --- internal/infrastructure/db/service_test.go | 632 +++++++++++++++++++++ 1 file changed, 632 insertions(+) diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index 5dc9953a7..25b0c4a98 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -206,6 +206,11 @@ func TestService(t *testing.T) { testCreateRootMarkersForEmptyVtxos(t, svc) testSweepVtxosWithMarkersIntegration(t, svc) testDeepChain20kMarkers(t, svc) + testPartialMarkerSweep(t, svc) + testListVtxosMarkerSweptFiltering(t, svc) + testSweepableUnrolledExcludesMarkerSwept(t, svc) + testConvergentMultiParentMarkerDAG(t, svc) + testSweepMarkerWithDescendantsDeepChain(t, svc) testOffchainTxRepository(t, svc) testScheduledSessionRepository(t, svc) testConvictionRepository(t, svc) @@ -3935,6 +3940,633 @@ func testSweepVtxosWithMarkersIntegration(t *testing.T, svc ports.RepoManager) { }) } +func testPartialMarkerSweep(t *testing.T, svc ports.RepoManager) { + t.Run("test_partial_marker_sweep", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + suffix := randomString(16) + + // Create a finalized round + roundId := uuid.New().String() + commitmentTxid := randomString(32) + round := domain.NewRoundFromEvents([]domain.Event{ + domain.RoundStarted{ + RoundEvent: domain.RoundEvent{Id: roundId, Type: domain.EventTypeRoundStarted}, + Timestamp: time.Now().Unix(), + }, + domain.RoundFinalizationStarted{ + RoundEvent: domain.RoundEvent{ + Id: roundId, + Type: domain.EventTypeRoundFinalizationStarted, + }, + CommitmentTxid: commitmentTxid, + CommitmentTx: emptyTx, + VtxoTree: vtxoTree, + Connectors: connectorsTree, + VtxoTreeExpiration: 3600, + }, + domain.RoundFinalized{ + RoundEvent: domain.RoundEvent{ + Id: roundId, + Type: domain.EventTypeRoundFinalized, + }, + FinalCommitmentTx: emptyTx, + Timestamp: time.Now().Unix(), + }, + }) + require.NoError(t, svc.Rounds().AddOrUpdateRound(ctx, *round)) + + // 3 markers: marker-0 (depth 0) -> marker-100 (depth 100) -> marker-200 (depth 200) + marker0ID := "partial-m0-" + suffix + marker100ID := "partial-m100-" + suffix + marker200ID := "partial-m200-" + suffix + + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: marker0ID, Depth: 0, + })) + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: marker100ID, Depth: 100, ParentMarkerIDs: []string{marker0ID}, + })) + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: marker200ID, Depth: 200, ParentMarkerIDs: []string{marker100ID}, + })) + + // 6 VTXOs: 2 per marker + type vtxoSpec struct { + txid string + depth uint32 + markerID string + } + specs := []vtxoSpec{ + {txid: "partial-v25-" + suffix, depth: 25, markerID: marker0ID}, + {txid: "partial-v75-" + suffix, depth: 75, markerID: marker0ID}, + {txid: "partial-v125-" + suffix, depth: 125, markerID: marker100ID}, + {txid: "partial-v175-" + suffix, depth: 175, markerID: marker100ID}, + {txid: "partial-v225-" + suffix, depth: 225, markerID: marker200ID}, + {txid: "partial-v250-" + suffix, depth: 250, markerID: marker200ID}, + } + + vtxosToAdd := make([]domain.Vtxo, len(specs)) + for i, s := range specs { + vtxosToAdd[i] = domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: s.txid, VOut: 0}, + PubKey: pubkey, + Amount: 1000, + CommitmentTxids: []string{commitmentTxid}, + RootCommitmentTxid: commitmentTxid, + CreatedAt: time.Now().Unix(), + ExpiresAt: time.Now().Add(time.Hour).Unix(), + Depth: s.depth, + MarkerIDs: []string{s.markerID}, + } + } + require.NoError(t, svc.Vtxos().AddVtxos(ctx, vtxosToAdd)) + + // Associate VTXOs with their markers + for _, s := range specs { + require.NoError(t, svc.Markers().UpdateVtxoMarkers(ctx, + domain.Outpoint{Txid: s.txid, VOut: 0}, []string{s.markerID})) + } + + // Sweep only marker-100 and marker-200 (NOT marker-0) + sweptAt := time.Now().Unix() + require.NoError(t, svc.Markers().BulkSweepMarkers(ctx, + []string{marker100ID, marker200ID}, sweptAt)) + + // Fetch all 6 VTXOs and check swept status + outpoints := make([]domain.Outpoint, len(specs)) + for i, s := range specs { + outpoints[i] = domain.Outpoint{Txid: s.txid, VOut: 0} + } + fetched, err := svc.Vtxos().GetVtxos(ctx, outpoints) + require.NoError(t, err) + require.Len(t, fetched, 6) + + for _, v := range fetched { + switch v.Txid { + case specs[0].txid, specs[1].txid: + // depth 25, 75 → marker-0 → NOT swept + require.False( + t, + v.Swept, + "vtxo %s (depth %d, marker-0) should NOT be swept", + v.Txid, + v.Depth, + ) + case specs[2].txid, specs[3].txid, specs[4].txid, specs[5].txid: + // depth 125, 175, 225, 250 → marker-100 or marker-200 → swept + require.True(t, v.Swept, "vtxo %s (depth %d) should be swept", v.Txid, v.Depth) + default: + t.Fatalf("unexpected vtxo txid: %s", v.Txid) + } + } + + // Verify IsMarkerSwept + isSwept, err := svc.Markers().IsMarkerSwept(ctx, marker0ID) + require.NoError(t, err) + require.False(t, isSwept, "marker-0 should NOT be swept") + + isSwept, err = svc.Markers().IsMarkerSwept(ctx, marker100ID) + require.NoError(t, err) + require.True(t, isSwept, "marker-100 should be swept") + + isSwept, err = svc.Markers().IsMarkerSwept(ctx, marker200ID) + require.NoError(t, err) + require.True(t, isSwept, "marker-200 should be swept") + }) +} + +func testListVtxosMarkerSweptFiltering(t *testing.T, svc ports.RepoManager) { + t.Run("test_list_vtxos_marker_swept_filtering", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + suffix := randomString(16) + testPubkey := "listfilter-pk-" + suffix + + // Create a finalized round + roundId := uuid.New().String() + commitmentTxid := randomString(32) + round := domain.NewRoundFromEvents([]domain.Event{ + domain.RoundStarted{ + RoundEvent: domain.RoundEvent{Id: roundId, Type: domain.EventTypeRoundStarted}, + Timestamp: time.Now().Unix(), + }, + domain.RoundFinalizationStarted{ + RoundEvent: domain.RoundEvent{ + Id: roundId, + Type: domain.EventTypeRoundFinalizationStarted, + }, + CommitmentTxid: commitmentTxid, + CommitmentTx: emptyTx, + VtxoTree: vtxoTree, + Connectors: connectorsTree, + VtxoTreeExpiration: 3600, + }, + domain.RoundFinalized{ + RoundEvent: domain.RoundEvent{ + Id: roundId, + Type: domain.EventTypeRoundFinalized, + }, + FinalCommitmentTx: emptyTx, + Timestamp: time.Now().Unix(), + }, + }) + require.NoError(t, svc.Rounds().AddOrUpdateRound(ctx, *round)) + + // 2 markers + markerAID := "listfilt-mA-" + suffix + markerBID := "listfilt-mB-" + suffix + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: markerAID, Depth: 0, + })) + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: markerBID, Depth: 0, + })) + + // 4 VTXOs: 2 with marker-A, 2 with marker-B (not unrolled, not spent) + txidA1 := "listfilt-a1-" + suffix + txidA2 := "listfilt-a2-" + suffix + txidB1 := "listfilt-b1-" + suffix + txidB2 := "listfilt-b2-" + suffix + + vtxosToAdd := []domain.Vtxo{ + { + Outpoint: domain.Outpoint{Txid: txidA1, VOut: 0}, + PubKey: testPubkey, + Amount: 1000, + CommitmentTxids: []string{commitmentTxid}, + RootCommitmentTxid: commitmentTxid, + CreatedAt: time.Now().Unix(), + ExpiresAt: time.Now().Add(time.Hour).Unix(), + Depth: 10, + MarkerIDs: []string{markerAID}, + }, + { + Outpoint: domain.Outpoint{Txid: txidA2, VOut: 0}, + PubKey: testPubkey, + Amount: 2000, + CommitmentTxids: []string{commitmentTxid}, + RootCommitmentTxid: commitmentTxid, + CreatedAt: time.Now().Unix(), + ExpiresAt: time.Now().Add(time.Hour).Unix(), + Depth: 20, + MarkerIDs: []string{markerAID}, + }, + { + Outpoint: domain.Outpoint{Txid: txidB1, VOut: 0}, + PubKey: testPubkey, + Amount: 3000, + CommitmentTxids: []string{commitmentTxid}, + RootCommitmentTxid: commitmentTxid, + CreatedAt: time.Now().Unix(), + ExpiresAt: time.Now().Add(time.Hour).Unix(), + Depth: 30, + MarkerIDs: []string{markerBID}, + }, + { + Outpoint: domain.Outpoint{Txid: txidB2, VOut: 0}, + PubKey: testPubkey, + Amount: 4000, + CommitmentTxids: []string{commitmentTxid}, + RootCommitmentTxid: commitmentTxid, + CreatedAt: time.Now().Unix(), + ExpiresAt: time.Now().Add(time.Hour).Unix(), + Depth: 40, + MarkerIDs: []string{markerBID}, + }, + } + require.NoError(t, svc.Vtxos().AddVtxos(ctx, vtxosToAdd)) + + for _, v := range vtxosToAdd { + require.NoError(t, svc.Markers().UpdateVtxoMarkers(ctx, v.Outpoint, v.MarkerIDs)) + } + + // Sweep only marker-A + sweptAt := time.Now().Unix() + require.NoError(t, svc.Markers().BulkSweepMarkers(ctx, []string{markerAID}, sweptAt)) + + // Call GetAllNonUnrolledVtxos + unspent, spent, err := svc.Vtxos().GetAllNonUnrolledVtxos(ctx, testPubkey) + require.NoError(t, err) + + // Unspent should be exactly the 2 VTXOs with marker-B + unspentTxids := make(map[string]bool) + for _, v := range unspent { + unspentTxids[v.Txid] = true + } + require.Len(t, unspent, 2, "expected 2 unspent vtxos (marker-B)") + require.True(t, unspentTxids[txidB1], "vtxo B1 should be unspent") + require.True(t, unspentTxids[txidB2], "vtxo B2 should be unspent") + + // Spent should be exactly the 2 VTXOs with marker-A (swept via marker) + spentTxids := make(map[string]bool) + for _, v := range spent { + spentTxids[v.Txid] = true + } + require.True(t, spentTxids[txidA1], "vtxo A1 should be in spent list (swept)") + require.True(t, spentTxids[txidA2], "vtxo A2 should be in spent list (swept)") + }) +} + +func testSweepableUnrolledExcludesMarkerSwept(t *testing.T, svc ports.RepoManager) { + t.Run("test_sweepable_unrolled_excludes_marker_swept", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + suffix := randomString(16) + + // Create a finalized round + roundId := uuid.New().String() + commitmentTxid := randomString(32) + round := domain.NewRoundFromEvents([]domain.Event{ + domain.RoundStarted{ + RoundEvent: domain.RoundEvent{Id: roundId, Type: domain.EventTypeRoundStarted}, + Timestamp: time.Now().Unix(), + }, + domain.RoundFinalizationStarted{ + RoundEvent: domain.RoundEvent{ + Id: roundId, + Type: domain.EventTypeRoundFinalizationStarted, + }, + CommitmentTxid: commitmentTxid, + CommitmentTx: emptyTx, + VtxoTree: vtxoTree, + Connectors: connectorsTree, + VtxoTreeExpiration: 3600, + }, + domain.RoundFinalized{ + RoundEvent: domain.RoundEvent{ + Id: roundId, + Type: domain.EventTypeRoundFinalized, + }, + FinalCommitmentTx: emptyTx, + Timestamp: time.Now().Unix(), + }, + }) + require.NoError(t, svc.Rounds().AddOrUpdateRound(ctx, *round)) + + // 2 markers + markerXID := "sweepable-mX-" + suffix + markerYID := "sweepable-mY-" + suffix + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: markerXID, Depth: 0, + })) + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: markerYID, Depth: 0, + })) + + // 3 VTXOs: VTXO-1 with marker-X, VTXO-2 and VTXO-3 with marker-Y + txid1 := "sweepable-v1-" + suffix + txid2 := "sweepable-v2-" + suffix + txid3 := "sweepable-v3-" + suffix + + vtxosToAdd := []domain.Vtxo{ + { + Outpoint: domain.Outpoint{Txid: txid1, VOut: 0}, + PubKey: pubkey, + Amount: 1000, + CommitmentTxids: []string{commitmentTxid}, + RootCommitmentTxid: commitmentTxid, + CreatedAt: time.Now().Unix(), + ExpiresAt: time.Now().Add(time.Hour).Unix(), + Depth: 10, + MarkerIDs: []string{markerXID}, + }, + { + Outpoint: domain.Outpoint{Txid: txid2, VOut: 0}, + PubKey: pubkey, + Amount: 2000, + CommitmentTxids: []string{commitmentTxid}, + RootCommitmentTxid: commitmentTxid, + CreatedAt: time.Now().Unix(), + ExpiresAt: time.Now().Add(time.Hour).Unix(), + Depth: 20, + MarkerIDs: []string{markerYID}, + }, + { + Outpoint: domain.Outpoint{Txid: txid3, VOut: 0}, + PubKey: pubkey, + Amount: 3000, + CommitmentTxids: []string{commitmentTxid}, + RootCommitmentTxid: commitmentTxid, + CreatedAt: time.Now().Unix(), + ExpiresAt: time.Now().Add(time.Hour).Unix(), + Depth: 30, + MarkerIDs: []string{markerYID}, + }, + } + require.NoError(t, svc.Vtxos().AddVtxos(ctx, vtxosToAdd)) + + for _, v := range vtxosToAdd { + require.NoError(t, svc.Markers().UpdateVtxoMarkers(ctx, v.Outpoint, v.MarkerIDs)) + } + + // Mark all as spent + spentVtxos := map[domain.Outpoint]string{ + {Txid: txid1, VOut: 0}: "spentby-" + suffix, + {Txid: txid2, VOut: 0}: "spentby-" + suffix, + {Txid: txid3, VOut: 0}: "spentby-" + suffix, + } + require.NoError(t, svc.Vtxos().SpendVtxos(ctx, spentVtxos, "arktx-"+suffix)) + + // Mark all as unrolled + unrollOutpoints := []domain.Outpoint{ + {Txid: txid1, VOut: 0}, + {Txid: txid2, VOut: 0}, + {Txid: txid3, VOut: 0}, + } + require.NoError(t, svc.Vtxos().UnrollVtxos(ctx, unrollOutpoints)) + + // Sweep only marker-X + sweptAt := time.Now().Unix() + require.NoError(t, svc.Markers().BulkSweepMarkers(ctx, []string{markerXID}, sweptAt)) + + // Call GetAllSweepableUnrolledVtxos + sweepable, err := svc.Vtxos().GetAllSweepableUnrolledVtxos(ctx) + require.NoError(t, err) + + // Result should contain VTXO-2 and VTXO-3 only (not VTXO-1 which is swept) + sweepableTxids := make(map[string]bool) + for _, v := range sweepable { + sweepableTxids[v.Txid] = true + } + require.True(t, sweepableTxids[txid2], "vtxo-2 (marker-Y, not swept) should be sweepable") + require.True(t, sweepableTxids[txid3], "vtxo-3 (marker-Y, not swept) should be sweepable") + require.False(t, sweepableTxids[txid1], "vtxo-1 (marker-X, swept) should NOT be sweepable") + }) +} + +func testConvergentMultiParentMarkerDAG(t *testing.T, svc ports.RepoManager) { + t.Run("test_convergent_multi_parent_marker_dag", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + suffix := randomString(16) + + // Create a finalized round + roundId := uuid.New().String() + commitmentTxid := randomString(32) + round := domain.NewRoundFromEvents([]domain.Event{ + domain.RoundStarted{ + RoundEvent: domain.RoundEvent{Id: roundId, Type: domain.EventTypeRoundStarted}, + Timestamp: time.Now().Unix(), + }, + domain.RoundFinalizationStarted{ + RoundEvent: domain.RoundEvent{ + Id: roundId, + Type: domain.EventTypeRoundFinalizationStarted, + }, + CommitmentTxid: commitmentTxid, + CommitmentTx: emptyTx, + VtxoTree: vtxoTree, + Connectors: connectorsTree, + VtxoTreeExpiration: 3600, + }, + domain.RoundFinalized{ + RoundEvent: domain.RoundEvent{ + Id: roundId, + Type: domain.EventTypeRoundFinalized, + }, + FinalCommitmentTx: emptyTx, + Timestamp: time.Now().Unix(), + }, + }) + require.NoError(t, svc.Rounds().AddOrUpdateRound(ctx, *round)) + + // Build convergent DAG: + // root-A (depth 0) root-B (depth 0) + // \ / + // mid-A (depth 100) mid-B (depth 100) + // \ / + // merge (depth 200, parents: [mid-A, mid-B]) + // | + // leaf (depth 300, parent: [merge]) + rootAID := "dag-rootA-" + suffix + rootBID := "dag-rootB-" + suffix + midAID := "dag-midA-" + suffix + midBID := "dag-midB-" + suffix + mergeID := "dag-merge-" + suffix + leafID := "dag-leaf-" + suffix + + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: rootAID, Depth: 0, + })) + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: rootBID, Depth: 0, + })) + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: midAID, Depth: 100, ParentMarkerIDs: []string{rootAID}, + })) + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: midBID, Depth: 100, ParentMarkerIDs: []string{rootBID}, + })) + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: mergeID, Depth: 200, ParentMarkerIDs: []string{midAID, midBID}, + })) + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: leafID, Depth: 300, ParentMarkerIDs: []string{mergeID}, + })) + + // 6 VTXOs, one per marker at intermediate depths + type vtxoSpec struct { + txid string + depth uint32 + markerID string + } + specs := []vtxoSpec{ + {txid: "dag-vrA-" + suffix, depth: 50, markerID: rootAID}, + {txid: "dag-vrB-" + suffix, depth: 50, markerID: rootBID}, + {txid: "dag-vmA-" + suffix, depth: 150, markerID: midAID}, + {txid: "dag-vmB-" + suffix, depth: 150, markerID: midBID}, + {txid: "dag-vmerge-" + suffix, depth: 250, markerID: mergeID}, + {txid: "dag-vleaf-" + suffix, depth: 350, markerID: leafID}, + } + + vtxosToAdd := make([]domain.Vtxo, len(specs)) + for i, s := range specs { + vtxosToAdd[i] = domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: s.txid, VOut: 0}, + PubKey: pubkey, + Amount: 1000, + CommitmentTxids: []string{commitmentTxid}, + RootCommitmentTxid: commitmentTxid, + CreatedAt: time.Now().Unix(), + ExpiresAt: time.Now().Add(time.Hour).Unix(), + Depth: s.depth, + MarkerIDs: []string{s.markerID}, + } + } + require.NoError(t, svc.Vtxos().AddVtxos(ctx, vtxosToAdd)) + + for _, s := range specs { + require.NoError(t, svc.Markers().UpdateVtxoMarkers(ctx, + domain.Outpoint{Txid: s.txid, VOut: 0}, []string{s.markerID})) + } + + allMarkerIDs := []string{rootAID, rootBID, midAID, midBID, mergeID, leafID} + + // GetVtxoChainByMarkers with all 6 markers → returns all 6 VTXOs + chainAll, err := svc.Markers().GetVtxoChainByMarkers(ctx, allMarkerIDs) + require.NoError(t, err) + require.Len(t, chainAll, 6, "all 6 markers should return all 6 VTXOs") + + // GetVtxoChainByMarkers with just [merge] → returns only VTXO-merge + chainMerge, err := svc.Markers().GetVtxoChainByMarkers(ctx, []string{mergeID}) + require.NoError(t, err) + require.Len(t, chainMerge, 1, "merge marker should return 1 VTXO") + require.Equal(t, specs[4].txid, chainMerge[0].Txid) + + // Sweep only root-A → only VTXO-rA is swept; others unswept + sweptAt := time.Now().Unix() + require.NoError(t, svc.Markers().BulkSweepMarkers(ctx, []string{rootAID}, sweptAt)) + + outpoints := make([]domain.Outpoint, len(specs)) + for i, s := range specs { + outpoints[i] = domain.Outpoint{Txid: s.txid, VOut: 0} + } + fetched, err := svc.Vtxos().GetVtxos(ctx, outpoints) + require.NoError(t, err) + require.Len(t, fetched, 6) + + for _, v := range fetched { + if v.Txid == specs[0].txid { + require.True(t, v.Swept, "vtxo root-A should be swept") + } else { + require.False( + t, + v.Swept, + "vtxo %s should NOT be swept after sweeping only root-A", + v.Txid, + ) + } + } + + // Sweep merge → VTXO-merge becomes swept; VTXO-leaf still unswept + require.NoError(t, svc.Markers().BulkSweepMarkers(ctx, []string{mergeID}, sweptAt)) + + fetched2, err := svc.Vtxos().GetVtxos(ctx, outpoints) + require.NoError(t, err) + require.Len(t, fetched2, 6) + + for _, v := range fetched2 { + switch v.Txid { + case specs[0].txid: // root-A + require.True(t, v.Swept, "vtxo root-A should still be swept") + case specs[4].txid: // merge + require.True(t, v.Swept, "vtxo merge should be swept") + case specs[5].txid: // leaf + require.False(t, v.Swept, "vtxo leaf should NOT be swept (different marker)") + default: + // root-B, mid-A, mid-B remain unswept + require.False(t, v.Swept, "vtxo %s should NOT be swept", v.Txid) + } + } + }) +} + +func testSweepMarkerWithDescendantsDeepChain(t *testing.T, svc ports.RepoManager) { + t.Run("test_sweep_marker_with_descendants_deep_chain", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + suffix := randomString(16) + + const maxDepth = 20000 + const markerInterval = 100 + const numMarkers = maxDepth/markerInterval + 1 // 201 + + // Build linear chain: marker-0 → marker-100 → ... → marker-20000 + allMarkerIDs := make([]string, 0, numMarkers) + + for depth := uint32(0); depth <= maxDepth; depth += markerInterval { + markerID := fmt.Sprintf("descdep-%s-m%d", suffix, depth) + allMarkerIDs = append(allMarkerIDs, markerID) + + var parentMarkerIDs []string + if depth > 0 { + parentMarkerIDs = []string{ + fmt.Sprintf("descdep-%s-m%d", suffix, depth-markerInterval), + } + } + + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: markerID, + Depth: depth, + ParentMarkerIDs: parentMarkerIDs, + })) + } + require.Len(t, allMarkerIDs, numMarkers) + rootID := allMarkerIDs[0] + + // SweepMarkerWithDescendants from root + sweptAt := time.Now().Unix() + count, err := svc.Markers().SweepMarkerWithDescendants(ctx, rootID, sweptAt) + require.NoError(t, err) + require.Equal(t, int64(numMarkers), count, + "should sweep all %d markers", numMarkers) + + // Spot-check: root (0), middle (10000), leaf (20000) → all true + for _, depth := range []uint32{0, 10000, 20000} { + markerID := fmt.Sprintf("descdep-%s-m%d", suffix, depth) + isSwept, err := svc.Markers().IsMarkerSwept(ctx, markerID) + require.NoError(t, err) + require.True(t, isSwept, "marker at depth %d should be swept", depth) + } + + // Idempotency: second call returns 0 + count, err = svc.Markers().SweepMarkerWithDescendants(ctx, rootID, sweptAt) + require.NoError(t, err) + require.Equal(t, int64(0), count, "second call should be idempotent (0 new sweeps)") + }) +} + type sortVtxos []domain.Vtxo func (a sortVtxos) String() string { From 7d4d296dd81b44a37292ce5dae296ebc818edcb3 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 12 Feb 2026 17:08:57 -0500 Subject: [PATCH 21/54] add marker for dust --- internal/infrastructure/db/service.go | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index 583b47499..314056f8b 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -612,9 +612,22 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) VOut: uint32(outIndex), } + vtxoMarkerIDs := markerIDs isDust := script.IsSubDustScript(out.PkScript) if isDust { dustVtxoOutpoints = append(dustVtxoOutpoints, outpoint) + // Dust VTXOs get their own outpoint-based marker so they can be + // swept individually without affecting sibling non-dust VTXOs + // that share the same inherited parent markers. + dustMarkerID := outpoint.String() + if err := s.markerStore.AddMarker(ctx, domain.Marker{ + ID: dustMarkerID, + Depth: newDepth, + ParentMarkerIDs: markerIDs, + }); err != nil { + log.WithError(err).Warnf("failed to create dust marker %s", dustMarkerID) + } + vtxoMarkerIDs = append(append([]string{}, markerIDs...), dustMarkerID) } newVtxos = append(newVtxos, domain.Vtxo{ @@ -627,7 +640,7 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) Preconfirmed: true, CreatedAt: offchainTx.StartingTimestamp, Depth: newDepth, - MarkerIDs: markerIDs, + MarkerIDs: vtxoMarkerIDs, }) } From 09f331df7407a888d5d890ed5f68f3d48effd37d Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Fri, 13 Feb 2026 11:03:42 -0500 Subject: [PATCH 22/54] safe copy, db tx usage, idx_marker_parent_markers index --- internal/core/application/service_test.go | 9 +++-- .../infrastructure/db/badger/marker_repo.go | 40 ++++++++----------- .../infrastructure/db/postgres/marker_repo.go | 18 +++++++-- ...60210100000_add_depth_and_markers.down.sql | 11 ++--- ...0260210100000_add_depth_and_markers.up.sql | 3 +- .../db/postgres/sqlc/queries/query.sql.go | 2 +- .../infrastructure/db/postgres/sqlc/query.sql | 2 +- internal/infrastructure/db/service.go | 24 +++++++++-- ...60210000000_add_depth_and_markers.down.sql | 15 ++++--- .../db/sqlite/sqlc/queries/query.sql.go | 6 +-- .../infrastructure/db/sqlite/sqlc/query.sql | 7 ++-- 11 files changed, 85 insertions(+), 52 deletions(-) diff --git a/internal/core/application/service_test.go b/internal/core/application/service_test.go index 6b65bdb7e..9ded96f4c 100644 --- a/internal/core/application/service_test.go +++ b/internal/core/application/service_test.go @@ -149,15 +149,18 @@ func TestDepthCalculation(t *testing.T) { { name: "no spent vtxos (empty)", spentVtxos: []domain.Vtxo{}, - expectedDepth: 1, - description: "empty input results in depth 1 (edge case)", + expectedDepth: 0, + description: "empty input results in depth 0 (no spent vtxos means maxDepth stays 0, newDepth = 0)", }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { maxDepth := calculateMaxDepth(tc.spentVtxos) - newDepth := maxDepth + 1 + var newDepth uint32 + if len(tc.spentVtxos) > 0 { + newDepth = maxDepth + 1 + } require.Equal(t, tc.expectedDepth, newDepth, tc.description) }) } diff --git a/internal/infrastructure/db/badger/marker_repo.go b/internal/infrastructure/db/badger/marker_repo.go index 209fbf69d..62f440341 100644 --- a/internal/infrastructure/db/badger/marker_repo.go +++ b/internal/infrastructure/db/badger/marker_repo.go @@ -255,21 +255,21 @@ func (r *markerRepository) SweepMarker(ctx context.Context, markerID string, swe } } - // Update Swept field on all VTXOs that have this marker + // Update Swept field on VTXOs that contain this marker // This keeps the stored Swept field in sync for query compatibility - var allDtos []vtxoDTO - if err := r.vtxoStore.Find(&allDtos, &badgerhold.Query{}); err != nil { + var filteredDtos []vtxoDTO + if err := r.vtxoStore.Find( + &filteredDtos, + badgerhold.Where("MarkerIDs").Contains(markerID), + ); err != nil { return nil // Non-fatal, swept_marker is already updated } - for _, vtxoDto := range allDtos { - for _, id := range vtxoDto.MarkerIDs { - if id == markerID && !vtxoDto.Swept { - vtxoDto.Swept = true - vtxoDto.UpdatedAt = time.Now().UnixMilli() - _ = r.vtxoStore.Update(vtxoDto.Outpoint.String(), vtxoDto) - break - } + for _, dto := range filteredDtos { + if !dto.Swept { + dto.Swept = true + dto.UpdatedAt = time.Now().UnixMilli() + _ = r.vtxoStore.Update(dto.Outpoint.String(), dto) } } @@ -439,24 +439,18 @@ func (r *markerRepository) GetVtxosByMarker( ctx context.Context, markerID string, ) ([]domain.Vtxo, error) { - // For badger, we need to scan all VTXOs and filter by MarkerIDs slice membership var dtos []vtxoDTO - err := r.vtxoStore.Find(&dtos, &badgerhold.Query{}) + err := r.vtxoStore.Find(&dtos, badgerhold.Where("MarkerIDs").Contains(markerID)) if err != nil { return nil, err } - vtxos := make([]domain.Vtxo, 0) + vtxos := make([]domain.Vtxo, 0, len(dtos)) for _, dto := range dtos { - for _, id := range dto.MarkerIDs { - if id == markerID { - vtxo := dto.Vtxo - // Compute Swept status dynamically by checking if any marker is swept - vtxo.Swept = r.isAnyMarkerSwept(dto.MarkerIDs) - vtxos = append(vtxos, vtxo) - break - } - } + vtxo := dto.Vtxo + // Compute Swept status dynamically by checking if any marker is swept + vtxo.Swept = r.isAnyMarkerSwept(dto.MarkerIDs) + vtxos = append(vtxos, vtxo) } return vtxos, nil } diff --git a/internal/infrastructure/db/postgres/marker_repo.go b/internal/infrastructure/db/postgres/marker_repo.go index a88bd7e19..b702f0660 100644 --- a/internal/infrastructure/db/postgres/marker_repo.go +++ b/internal/infrastructure/db/postgres/marker_repo.go @@ -161,8 +161,16 @@ func (m *markerRepository) SweepMarkerWithDescendants( markerID string, sweptAt int64, ) (int64, error) { + tx, err := m.db.BeginTx(ctx, nil) + if err != nil { + return 0, fmt.Errorf("failed to begin transaction: %w", err) + } + defer tx.Rollback() //nolint:errcheck + + txQuerier := m.querier.WithTx(tx) + // Get all descendant marker IDs (including the root marker) that are not already swept - descendantIDs, err := m.querier.GetDescendantMarkerIds(ctx, markerID) + descendantIDs, err := txQuerier.GetDescendantMarkerIds(ctx, markerID) if err != nil { return 0, fmt.Errorf("failed to get descendant markers: %w", err) } @@ -170,16 +178,20 @@ func (m *markerRepository) SweepMarkerWithDescendants( // Insert each descendant into swept_marker var count int64 for _, id := range descendantIDs { - err := m.querier.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ + err := txQuerier.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ MarkerID: id, SweptAt: sweptAt, }) if err != nil { - return count, fmt.Errorf("failed to sweep marker %s: %w", id, err) + return 0, fmt.Errorf("failed to sweep marker %s: %w", id, err) } count++ } + if err := tx.Commit(); err != nil { + return 0, fmt.Errorf("failed to commit transaction: %w", err) + } + return count, nil } diff --git a/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.down.sql b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.down.sql index fee52f3f6..a13d60016 100644 --- a/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.down.sql +++ b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.down.sql @@ -1,18 +1,19 @@ --- Drop markers column from vtxo +-- Drop views first (they depend on vtxo columns via v.*) +DROP VIEW IF EXISTS intent_with_inputs_vw; +DROP VIEW IF EXISTS vtxo_vw; + +-- Drop markers index and column from vtxo DROP INDEX IF EXISTS idx_vtxo_markers; ALTER TABLE vtxo DROP COLUMN IF EXISTS markers; -- Drop depth column from vtxo ALTER TABLE vtxo DROP COLUMN IF EXISTS depth; --- Drop marker tables +-- Drop marker tables (indexes are dropped automatically with the table) DROP TABLE IF EXISTS swept_marker; DROP TABLE IF EXISTS marker; -- Recreate views without depth and markers columns -DROP VIEW IF EXISTS intent_with_inputs_vw; -DROP VIEW IF EXISTS vtxo_vw; - CREATE VIEW vtxo_vw AS SELECT v.*, string_agg(vc.commitment_txid, ',') AS commitments FROM vtxo v diff --git a/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql index f6b451ac9..4e7ca2b49 100644 --- a/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql +++ b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql @@ -1,7 +1,7 @@ -- Add depth and markers columns to vtxo ALTER TABLE vtxo ADD COLUMN IF NOT EXISTS depth INTEGER NOT NULL DEFAULT 0, - ADD COLUMN IF NOT EXISTS markers JSONB; + ADD COLUMN IF NOT EXISTS markers JSONB NOT NULL DEFAULT '[]'::jsonb; CREATE INDEX IF NOT EXISTS idx_vtxo_markers ON vtxo USING GIN (markers); -- Create marker table @@ -11,6 +11,7 @@ CREATE TABLE IF NOT EXISTS marker ( parent_markers JSONB -- JSON array of parent marker IDs ); CREATE INDEX IF NOT EXISTS idx_marker_depth ON marker(depth); +CREATE INDEX IF NOT EXISTS idx_marker_parent_markers ON marker USING GIN (parent_markers); -- Create swept_marker table (append-only) CREATE TABLE IF NOT EXISTS swept_marker ( diff --git a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go index 00667c30d..36f5d9e62 100644 --- a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go @@ -1768,7 +1768,7 @@ func (q *Queries) SelectVtxoPubKeysByCommitmentTxid(ctx context.Context, arg Sel } const selectVtxosByArkTxid = `-- name: SelectVtxosByArkTxid :many -SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, markers, commitments, swept FROM vtxo_vw WHERE txid = $1 +SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, markers, commitments, swept FROM vtxo_vw WHERE ark_txid = $1 ` // Get all VTXOs created by a specific ark tx (offchain tx) diff --git a/internal/infrastructure/db/postgres/sqlc/query.sql b/internal/infrastructure/db/postgres/sqlc/query.sql index cd0ffddbe..d458e6108 100644 --- a/internal/infrastructure/db/postgres/sqlc/query.sql +++ b/internal/infrastructure/db/postgres/sqlc/query.sql @@ -505,7 +505,7 @@ ORDER BY depth DESC; -- name: SelectVtxosByArkTxid :many -- Get all VTXOs created by a specific ark tx (offchain tx) -SELECT * FROM vtxo_vw WHERE txid = @ark_txid; +SELECT * FROM vtxo_vw WHERE ark_txid = @ark_txid; -- name: SelectVtxoChainByMarker :many -- Get VTXOs whose markers JSONB array contains any of the given marker IDs diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index 314056f8b..d709a0f75 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -212,7 +212,13 @@ func NewService(config ServiceConfig, txDecoder ports.TxDecoder) (ports.RepoMana if !ok { return nil, fmt.Errorf("failed to get badger vtxo repository") } - markerConfig := append(config.DataStoreConfig, badgerVtxoRepo.GetStore()) + markerConfig := make( + []interface{}, + len(config.DataStoreConfig), + len(config.DataStoreConfig)+1, + ) + copy(markerConfig, config.DataStoreConfig) + markerConfig = append(markerConfig, badgerVtxoRepo.GetStore()) markerStore, err = markerStoreFactory(markerConfig...) if err != nil { return nil, fmt.Errorf("failed to create marker store: %w", err) @@ -491,8 +497,16 @@ func (s *service) updateProjectionsAfterRoundEvents(events []domain.Event) { } // Create root markers for batch VTXOs (depth 0 is always at marker boundary) - if err := s.markerStore.CreateRootMarkersForVtxos(ctx, newVtxos); err != nil { - log.WithError(err).Warn("failed to create root markers for vtxos") + for { + if err := s.markerStore.CreateRootMarkersForVtxos(ctx, newVtxos); err != nil { + log.WithError(err).Warnf( + "failed to create root markers for %d vtxos, retrying soon", len(newVtxos), + ) + time.Sleep(100 * time.Millisecond) + continue + } + log.Debugf("created root markers for %d vtxos", len(newVtxos)) + break } } } @@ -552,7 +566,9 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) if len(spentOutpoints) > 0 { spentVtxos, err := s.vtxoStore.GetVtxos(ctx, spentOutpoints) if err != nil { - log.WithError(err).Warn("failed to get spent vtxos for depth calculation") + log.WithError(err). + Warn("failed to get spent vtxos for depth calculation, aborting finalization") + return } else { // Calculate depth: max(parent depths) + 1 var maxDepth uint32 diff --git a/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.down.sql b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.down.sql index 7f9fefbeb..b0cc2320a 100644 --- a/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.down.sql +++ b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.down.sql @@ -25,12 +25,17 @@ CREATE TABLE vtxo_temp ( FOREIGN KEY (intent_id) REFERENCES intent(id) ); --- Copy data +-- Copy data, computing swept from swept_marker since the column was removed in the up migration INSERT INTO vtxo_temp SELECT - txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, - spent_by, spent, unrolled, swept, preconfirmed, settled_by, ark_txid, - intent_id, updated_at -FROM vtxo; + v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, + v.spent_by, v.spent, v.unrolled, + EXISTS ( + SELECT 1 FROM swept_marker sm + WHERE v.markers LIKE '%"' || sm.marker_id || '"%' + ) AS swept, + v.preconfirmed, v.settled_by, v.ark_txid, + v.intent_id, v.updated_at +FROM vtxo v; -- Drop old table and rename DROP TABLE vtxo; diff --git a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go index 16df64b80..de93b230f 100644 --- a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go @@ -103,8 +103,8 @@ WITH RECURSIVE descendant_markers(id) AS ( UNION ALL -- Recursive case: find markers whose parent_markers JSON array contains any descendant SELECT m.id FROM marker m - INNER JOIN descendant_markers dm ON ( - m.parent_markers LIKE '%"' || dm.id || '"%' + INNER JOIN descendant_markers dm ON EXISTS ( + SELECT 1 FROM json_each(m.parent_markers) j WHERE j.value = dm.id ) ) SELECT descendant_markers.id AS marker_id FROM descendant_markers @@ -1833,7 +1833,7 @@ func (q *Queries) SelectVtxoPubKeysByCommitmentTxid(ctx context.Context, arg Sel } const selectVtxosByArkTxid = `-- name: SelectVtxosByArkTxid :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE txid = ?1 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE ark_txid = ?1 ` type SelectVtxosByArkTxidRow struct { diff --git a/internal/infrastructure/db/sqlite/sqlc/query.sql b/internal/infrastructure/db/sqlite/sqlc/query.sql index 98996023d..fb0f63097 100644 --- a/internal/infrastructure/db/sqlite/sqlc/query.sql +++ b/internal/infrastructure/db/sqlite/sqlc/query.sql @@ -470,14 +470,15 @@ SELECT EXISTS(SELECT 1 FROM swept_marker WHERE marker_id = @marker_id) AS is_swe -- name: GetDescendantMarkerIds :many -- Recursively get a marker and all its descendants (markers whose parent_markers contain it) +-- Uses json_each instead of LIKE to avoid false positives with special characters (%, _) WITH RECURSIVE descendant_markers(id) AS ( -- Base case: the marker being swept SELECT marker.id FROM marker WHERE marker.id = @root_marker_id UNION ALL -- Recursive case: find markers whose parent_markers JSON array contains any descendant SELECT m.id FROM marker m - INNER JOIN descendant_markers dm ON ( - m.parent_markers LIKE '%"' || dm.id || '"%' + INNER JOIN descendant_markers dm ON EXISTS ( + SELECT 1 FROM json_each(m.parent_markers) j WHERE j.value = dm.id ) ) SELECT descendant_markers.id AS marker_id FROM descendant_markers @@ -504,7 +505,7 @@ ORDER BY depth DESC; -- name: SelectVtxosByArkTxid :many -- Get all VTXOs created by a specific ark tx (offchain tx) -SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE txid = @ark_txid; +SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE ark_txid = @ark_txid; -- name: SelectVtxoChainByMarker :many -- Get VTXOs whose markers array contains the given marker_id From 1122bd9765ac75039dcf81beac52a0ef0cc51d45 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Fri, 13 Feb 2026 12:05:02 -0500 Subject: [PATCH 23/54] depthKnown, markersToMarshal fix --- .../db/postgres/sqlc/queries/query.sql.go | 2 +- .../infrastructure/db/postgres/sqlc/query.sql | 2 +- internal/infrastructure/db/postgres/vtxo_repo.go | 15 ++++++++------- internal/infrastructure/db/service.go | 9 ++++++--- .../db/sqlite/sqlc/queries/query.sql.go | 2 +- internal/infrastructure/db/sqlite/sqlc/query.sql | 2 +- 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go index 36f5d9e62..00667c30d 100644 --- a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go @@ -1768,7 +1768,7 @@ func (q *Queries) SelectVtxoPubKeysByCommitmentTxid(ctx context.Context, arg Sel } const selectVtxosByArkTxid = `-- name: SelectVtxosByArkTxid :many -SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, markers, commitments, swept FROM vtxo_vw WHERE ark_txid = $1 +SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, markers, commitments, swept FROM vtxo_vw WHERE txid = $1 ` // Get all VTXOs created by a specific ark tx (offchain tx) diff --git a/internal/infrastructure/db/postgres/sqlc/query.sql b/internal/infrastructure/db/postgres/sqlc/query.sql index d458e6108..cd0ffddbe 100644 --- a/internal/infrastructure/db/postgres/sqlc/query.sql +++ b/internal/infrastructure/db/postgres/sqlc/query.sql @@ -505,7 +505,7 @@ ORDER BY depth DESC; -- name: SelectVtxosByArkTxid :many -- Get all VTXOs created by a specific ark tx (offchain tx) -SELECT * FROM vtxo_vw WHERE ark_txid = @ark_txid; +SELECT * FROM vtxo_vw WHERE txid = @ark_txid; -- name: SelectVtxoChainByMarker :many -- Get VTXOs whose markers JSONB array contains any of the given marker IDs diff --git a/internal/infrastructure/db/postgres/vtxo_repo.go b/internal/infrastructure/db/postgres/vtxo_repo.go index 693752251..ab7d389a8 100644 --- a/internal/infrastructure/db/postgres/vtxo_repo.go +++ b/internal/infrastructure/db/postgres/vtxo_repo.go @@ -42,14 +42,15 @@ func (v *vtxoRepository) AddVtxos(ctx context.Context, vtxos []domain.Vtxo) erro for i := range vtxos { vtxo := vtxos[i] - var markersJSON pqtype.NullRawMessage - if len(vtxo.MarkerIDs) > 0 { - data, err := json.Marshal(vtxo.MarkerIDs) - if err != nil { - return fmt.Errorf("failed to marshal markers: %w", err) - } - markersJSON = pqtype.NullRawMessage{RawMessage: data, Valid: true} + markersToMarshal := vtxo.MarkerIDs + if markersToMarshal == nil { + markersToMarshal = []string{} + } + data, err := json.Marshal(markersToMarshal) + if err != nil { + return fmt.Errorf("failed to marshal markers: %w", err) } + markersJSON := pqtype.NullRawMessage{RawMessage: data, Valid: true} if err := querierWithTx.UpsertVtxo( ctx, queries.UpsertVtxoParams{ diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index d709a0f75..a75b174b8 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -563,12 +563,14 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) // Get spent VTXOs to calculate new depth var newDepth uint32 var parentMarkerIDs []string + depthKnown := true if len(spentOutpoints) > 0 { spentVtxos, err := s.vtxoStore.GetVtxos(ctx, spentOutpoints) if err != nil { log.WithError(err). - Warn("failed to get spent vtxos for depth calculation, aborting finalization") - return + Warn("failed to get spent vtxos for depth calculation, skipping marker creation") + // Continue with depth 0 but mark as unknown to avoid creating misleading root markers + depthKnown = false } else { // Calculate depth: max(parent depths) + 1 var maxDepth uint32 @@ -593,9 +595,10 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) } // Create marker if at boundary depth, or inherit ALL parent markers + // Skip marker creation if depth is unknown (GetVtxos failed) to avoid misleading root markers var markerIDs []string - if domain.IsAtMarkerBoundary(newDepth) { + if depthKnown && domain.IsAtMarkerBoundary(newDepth) { // Create marker ID from the first output (the ark tx id + first vtxo vout) newMarkerID := fmt.Sprintf("%s:marker:%d", txid, newDepth) marker := domain.Marker{ diff --git a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go index de93b230f..90da9662d 100644 --- a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go @@ -1833,7 +1833,7 @@ func (q *Queries) SelectVtxoPubKeysByCommitmentTxid(ctx context.Context, arg Sel } const selectVtxosByArkTxid = `-- name: SelectVtxosByArkTxid :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE ark_txid = ?1 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept FROM vtxo_vw WHERE txid = ?1 ` type SelectVtxosByArkTxidRow struct { diff --git a/internal/infrastructure/db/sqlite/sqlc/query.sql b/internal/infrastructure/db/sqlite/sqlc/query.sql index fb0f63097..40b5715dc 100644 --- a/internal/infrastructure/db/sqlite/sqlc/query.sql +++ b/internal/infrastructure/db/sqlite/sqlc/query.sql @@ -505,7 +505,7 @@ ORDER BY depth DESC; -- name: SelectVtxosByArkTxid :many -- Get all VTXOs created by a specific ark tx (offchain tx) -SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE ark_txid = @ark_txid; +SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE txid = @ark_txid; -- name: SelectVtxoChainByMarker :many -- Get VTXOs whose markers array contains the given marker_id From bd5e93cc46aae59085355c5810afc6ce06c9028e Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Fri, 13 Feb 2026 12:55:18 -0500 Subject: [PATCH 24/54] parentMarkerIds emply slice setting, test fix --- internal/core/application/service_test.go | 4 ++-- internal/infrastructure/db/postgres/marker_repo.go | 6 +++++- internal/infrastructure/db/service_test.go | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/core/application/service_test.go b/internal/core/application/service_test.go index 9ded96f4c..d1c98e43a 100644 --- a/internal/core/application/service_test.go +++ b/internal/core/application/service_test.go @@ -652,8 +652,8 @@ func TestDepth20k_MarkerBoundaryAndInheritance(t *testing.T) { require.Len(t, markerIDs, 150, "child inherits all 150 unique markers") }) - t.Run("depth calculation with max uint32 near boundary", func(t *testing.T) { - // Verify depth arithmetic doesn't overflow for large values + t.Run("depth beyond 20k target remains valid", func(t *testing.T) { + // Verify depth arithmetic works correctly beyond the 20k boundary parent := domain.Vtxo{Depth: 20000, MarkerIDs: []string{"marker-20000"}} newDepth := calculateMaxDepth([]domain.Vtxo{parent}) + 1 require.Equal(t, uint32(20001), newDepth) diff --git a/internal/infrastructure/db/postgres/marker_repo.go b/internal/infrastructure/db/postgres/marker_repo.go index b702f0660..e2122ca4d 100644 --- a/internal/infrastructure/db/postgres/marker_repo.go +++ b/internal/infrastructure/db/postgres/marker_repo.go @@ -37,7 +37,11 @@ func (m *markerRepository) Close() { } func (m *markerRepository) AddMarker(ctx context.Context, marker domain.Marker) error { - parentMarkersJSON, err := json.Marshal(marker.ParentMarkerIDs) + parentMarkerIDs := marker.ParentMarkerIDs + if parentMarkerIDs == nil { + parentMarkerIDs = []string{} + } + parentMarkersJSON, err := json.Marshal(parentMarkerIDs) if err != nil { return fmt.Errorf("failed to marshal parent markers: %w", err) } diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index 25b0c4a98..5f4de605f 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -2307,7 +2307,7 @@ func testGetVtxoChainWithMarkerOptimization(t *testing.T, svc ports.RepoManager) require.NoError(t, err) require.NotNil(t, currentMarker) require.Equal(t, uint32(0), currentMarker.Depth) - require.Nil(t, currentMarker.ParentMarkerIDs) // Root marker has no parents + require.Empty(t, currentMarker.ParentMarkerIDs) // Root marker has no parents // Test 5: Test GetMarkersByIds with the full chain markers, err := svc.Markers().GetMarkersByIds(ctx, fullMarkerChain) From 307269e5cb81bc68e4b53d54fb1c7d34240df353 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Mon, 16 Feb 2026 13:18:23 -0500 Subject: [PATCH 25/54] lint fixes --- internal/infrastructure/db/service_test.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index 4c509439f..0308b4ada 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -4,7 +4,6 @@ import ( "context" "crypto/rand" "encoding/hex" - "encoding/json" "fmt" "math" "math/big" @@ -4743,23 +4742,6 @@ func testSweepMarkerWithDescendantsDeepChain(t *testing.T, svc ports.RepoManager }) } -type sortVtxos []domain.Vtxo - -func (a sortVtxos) String() string { - buf, _ := json.Marshal(a) - return string(buf) -} - -func (a sortVtxos) Len() int { return len(a) } -func (a sortVtxos) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a sortVtxos) Less(i, j int) bool { return a[i].Txid < a[j].Txid } - -type sortReceivers []domain.Receiver - -func (a sortReceivers) Len() int { return len(a) } -func (a sortReceivers) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a sortReceivers) Less(i, j int) bool { return a[i].Amount < a[j].Amount } - func checkVtxos(t *testing.T, expectedVtxos, gotVtxos []domain.Vtxo) { sort.SliceStable(expectedVtxos, func(i, j int) bool { return expectedVtxos[i].Txid < expectedVtxos[j].Txid From a6f0d8c7ce96e9fa996da9fd83d5fe41f4ecad45 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Mon, 16 Feb 2026 22:06:33 -0500 Subject: [PATCH 26/54] test optimizations, test comments --- internal/core/application/indexer_test.go | 103 +++++----------- internal/core/application/sweeper_test.go | 137 +++++---------------- internal/core/application/utils_test.go | 9 ++ internal/infrastructure/db/service_test.go | 71 +++++++++++ internal/test/e2e/e2e_test.go | 4 - internal/test/e2e/utils_test.go | 3 - 6 files changed, 138 insertions(+), 189 deletions(-) diff --git a/internal/core/application/indexer_test.go b/internal/core/application/indexer_test.go index 3956e88c8..a633b4b4d 100644 --- a/internal/core/application/indexer_test.go +++ b/internal/core/application/indexer_test.go @@ -299,15 +299,24 @@ func (m *mockRepoManagerForIndexer) Assets() domain.AssetRepository func (m *mockRepoManagerForIndexer) Fees() domain.FeeRepository { return nil } func (m *mockRepoManagerForIndexer) Close() {} -// TestPrefetchVtxosByMarkers_BuildsCacheFromMarkerChain verifies that prefetchVtxosByMarkers -// correctly traverses the marker hierarchy (following ParentMarkerIDs) and bulk fetches -// all VTXOs associated with those markers into a cache map. -func TestPrefetchVtxosByMarkers_BuildsCacheFromMarkerChain(t *testing.T) { +// newTestIndexer creates a fresh set of mock repos and an indexerService for testing. +func newTestIndexer() ( + *mockVtxoRepoForIndexer, + *mockMarkerRepoForIndexer, + *indexerService, +) { vtxoRepo := &mockVtxoRepoForIndexer{} markerRepo := &mockMarkerRepoForIndexer{} repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} - indexer := &indexerService{repoManager: repoManager} + return vtxoRepo, markerRepo, indexer +} + +// TestPrefetchVtxosByMarkers_BuildsCacheFromMarkerChain verifies that prefetchVtxosByMarkers +// correctly traverses the marker hierarchy (following ParentMarkerIDs) and bulk fetches +// all VTXOs associated with those markers into a cache map. +func TestPrefetchVtxosByMarkers_BuildsCacheFromMarkerChain(t *testing.T) { + vtxoRepo, markerRepo, indexer := newTestIndexer() ctx := context.Background() startKey := Outpoint{Txid: "start-vtxo", VOut: 0} @@ -369,11 +378,7 @@ func TestPrefetchVtxosByMarkers_BuildsCacheFromMarkerChain(t *testing.T) { // TestPrefetchVtxosByMarkers_EmptyMarkersReturnsStartVtxoOnly verifies that when the // starting VTXO has no markers, the cache only contains the starting VTXO itself. func TestPrefetchVtxosByMarkers_EmptyMarkersReturnsStartVtxoOnly(t *testing.T) { - vtxoRepo := &mockVtxoRepoForIndexer{} - markerRepo := &mockMarkerRepoForIndexer{} - repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} - - indexer := &indexerService{repoManager: repoManager} + vtxoRepo, markerRepo, indexer := newTestIndexer() ctx := context.Background() startKey := Outpoint{Txid: "vtxo-no-markers", VOut: 0} @@ -422,11 +427,7 @@ func TestPrefetchVtxosByMarkers_NilMarkerRepoReturnsEmptyCache(t *testing.T) { // TestGetVtxosFromCacheOrDB_CacheHitAvoidsDBCall verifies that when all requested // outpoints are in the cache, no database call is made. func TestGetVtxosFromCacheOrDB_CacheHitAvoidsDBCall(t *testing.T) { - vtxoRepo := &mockVtxoRepoForIndexer{} - markerRepo := &mockMarkerRepoForIndexer{} - repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} - - indexer := &indexerService{repoManager: repoManager} + vtxoRepo, _, indexer := newTestIndexer() ctx := context.Background() @@ -459,11 +460,7 @@ func TestGetVtxosFromCacheOrDB_CacheHitAvoidsDBCall(t *testing.T) { // TestGetVtxosFromCacheOrDB_CacheMissTriggersDBCall verifies that when outpoints // are not in the cache, a database call is made for the missing ones only. func TestGetVtxosFromCacheOrDB_CacheMissTriggersDBCall(t *testing.T) { - vtxoRepo := &mockVtxoRepoForIndexer{} - markerRepo := &mockMarkerRepoForIndexer{} - repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} - - indexer := &indexerService{repoManager: repoManager} + vtxoRepo, _, indexer := newTestIndexer() ctx := context.Background() @@ -498,11 +495,7 @@ func TestGetVtxosFromCacheOrDB_CacheMissTriggersDBCall(t *testing.T) { // TestGetVtxosFromCacheOrDB_AllCacheMiss verifies behavior when cache is empty // and all outpoints must be fetched from the database. func TestGetVtxosFromCacheOrDB_AllCacheMiss(t *testing.T) { - vtxoRepo := &mockVtxoRepoForIndexer{} - markerRepo := &mockMarkerRepoForIndexer{} - repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} - - indexer := &indexerService{repoManager: repoManager} + vtxoRepo, _, indexer := newTestIndexer() ctx := context.Background() @@ -538,11 +531,7 @@ func TestGetVtxosFromCacheOrDB_AllCacheMiss(t *testing.T) { // TestGetVtxosFromCacheOrDB_DBErrorPropagated verifies that database errors // are properly propagated to the caller. func TestGetVtxosFromCacheOrDB_DBErrorPropagated(t *testing.T) { - vtxoRepo := &mockVtxoRepoForIndexer{} - markerRepo := &mockMarkerRepoForIndexer{} - repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} - - indexer := &indexerService{repoManager: repoManager} + vtxoRepo, _, indexer := newTestIndexer() ctx := context.Background() cache := make(map[string]domain.Vtxo) @@ -562,11 +551,7 @@ func TestGetVtxosFromCacheOrDB_DBErrorPropagated(t *testing.T) { // traversal correctly handles VTXOs with multiple parent markers (diamond pattern // in the marker DAG). func TestPrefetchVtxosByMarkers_HandlesMultipleParentMarkers(t *testing.T) { - vtxoRepo := &mockVtxoRepoForIndexer{} - markerRepo := &mockMarkerRepoForIndexer{} - repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} - - indexer := &indexerService{repoManager: repoManager} + vtxoRepo, markerRepo, indexer := newTestIndexer() ctx := context.Background() startKey := Outpoint{Txid: "diamond-vtxo", VOut: 0} @@ -625,11 +610,7 @@ func TestPrefetchVtxosByMarkers_HandlesMultipleParentMarkers(t *testing.T) { // to retrieve the starting VTXO, prefetchVtxosByMarkers returns an empty cache // gracefully without panicking. func TestPrefetchVtxosByMarkers_GetVtxosError(t *testing.T) { - vtxoRepo := &mockVtxoRepoForIndexer{} - markerRepo := &mockMarkerRepoForIndexer{} - repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} - - indexer := &indexerService{repoManager: repoManager} + vtxoRepo, markerRepo, indexer := newTestIndexer() ctx := context.Background() startKey := Outpoint{Txid: "vtxo-error", VOut: 0} @@ -652,11 +633,7 @@ func TestPrefetchVtxosByMarkers_GetVtxosError(t *testing.T) { // fails for a marker in the BFS traversal, the function still returns // partial results from successfully fetched markers. func TestPrefetchVtxosByMarkers_GetMarkerError(t *testing.T) { - vtxoRepo := &mockVtxoRepoForIndexer{} - markerRepo := &mockMarkerRepoForIndexer{} - repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} - - indexer := &indexerService{repoManager: repoManager} + vtxoRepo, markerRepo, indexer := newTestIndexer() ctx := context.Background() startKey := Outpoint{Txid: "vtxo-partial", VOut: 0} @@ -707,11 +684,7 @@ func TestPrefetchVtxosByMarkers_GetMarkerError(t *testing.T) { // the bulk fetch of VTXOs by markers fails, the cache still contains at // least the starting VTXO. func TestPrefetchVtxosByMarkers_GetVtxoChainByMarkersError(t *testing.T) { - vtxoRepo := &mockVtxoRepoForIndexer{} - markerRepo := &mockMarkerRepoForIndexer{} - repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} - - indexer := &indexerService{repoManager: repoManager} + vtxoRepo, markerRepo, indexer := newTestIndexer() ctx := context.Background() startKey := Outpoint{Txid: "vtxo-bulk-err", VOut: 0} @@ -748,11 +721,7 @@ func TestPrefetchVtxosByMarkers_GetVtxoChainByMarkersError(t *testing.T) { // correctly handles a deep chain with 5+ markers (depth 500), collecting all // markers without off-by-one errors or missed parents. func TestPrefetchVtxosByMarkers_DeepChainManyMarkers(t *testing.T) { - vtxoRepo := &mockVtxoRepoForIndexer{} - markerRepo := &mockMarkerRepoForIndexer{} - repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} - - indexer := &indexerService{repoManager: repoManager} + vtxoRepo, markerRepo, indexer := newTestIndexer() ctx := context.Background() startKey := Outpoint{Txid: "deep-vtxo", VOut: 0} @@ -845,11 +814,7 @@ func TestPrefetchVtxosByMarkers_DeepChainManyMarkers(t *testing.T) { // TestGetVtxosFromCacheOrDB_EmptyOutpoints verifies that an empty outpoints // list returns an empty result without making any database call. func TestGetVtxosFromCacheOrDB_EmptyOutpoints(t *testing.T) { - vtxoRepo := &mockVtxoRepoForIndexer{} - markerRepo := &mockMarkerRepoForIndexer{} - repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} - - indexer := &indexerService{repoManager: repoManager} + vtxoRepo, _, indexer := newTestIndexer() ctx := context.Background() cache := map[string]domain.Vtxo{ @@ -869,11 +834,7 @@ func TestGetVtxosFromCacheOrDB_EmptyOutpoints(t *testing.T) { // prefetchVtxosByMarkers terminates when there is a cycle in the marker DAG // (marker-A → parent marker-B → parent marker-A). func TestPrefetchVtxosByMarkers_CycleInMarkerDAG(t *testing.T) { - vtxoRepo := &mockVtxoRepoForIndexer{} - markerRepo := &mockMarkerRepoForIndexer{} - repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} - - indexer := &indexerService{repoManager: repoManager} + vtxoRepo, markerRepo, indexer := newTestIndexer() ctx := context.Background() startKey := Outpoint{Txid: "cycle-vtxo", VOut: 0} @@ -929,11 +890,7 @@ func TestPrefetchVtxosByMarkers_CycleInMarkerDAG(t *testing.T) { // TestPrefetchVtxosByMarkers_StartVtxoNotFound verifies that when the starting // VTXO is not found in the database, an empty cache is returned. func TestPrefetchVtxosByMarkers_StartVtxoNotFound(t *testing.T) { - vtxoRepo := &mockVtxoRepoForIndexer{} - markerRepo := &mockMarkerRepoForIndexer{} - repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} - - indexer := &indexerService{repoManager: repoManager} + vtxoRepo, markerRepo, indexer := newTestIndexer() ctx := context.Background() startKey := Outpoint{Txid: "nonexistent", VOut: 0} @@ -954,11 +911,7 @@ func TestPrefetchVtxosByMarkers_StartVtxoNotFound(t *testing.T) { // prefetchVtxosByMarkers correctly handles a VTXO at depth 20000 with a chain // of 200 markers (one every 100 depths). This is the target maximum depth. func TestPrefetchVtxosByMarkers_Depth20k(t *testing.T) { - vtxoRepo := &mockVtxoRepoForIndexer{} - markerRepo := &mockMarkerRepoForIndexer{} - repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: markerRepo} - - indexer := &indexerService{repoManager: repoManager} + vtxoRepo, markerRepo, indexer := newTestIndexer() ctx := context.Background() startKey := Outpoint{Txid: "deep-20k-vtxo", VOut: 0} diff --git a/internal/core/application/sweeper_test.go b/internal/core/application/sweeper_test.go index 4373afb05..7d5e79f9f 100644 --- a/internal/core/application/sweeper_test.go +++ b/internal/core/application/sweeper_test.go @@ -474,22 +474,31 @@ func (m *mockScheduler) Unit() ports.TimeUnit { return p func (m *mockScheduler) AfterNow(expiry int64) bool { return false } func (m *mockScheduler) ScheduleTaskOnce(at int64, task func()) error { return nil } -// TestCreateCheckpointSweepTask_BulkSweepsMarkers verifies that when a checkpoint -// is swept, the sweeper correctly collects all unique marker IDs from the affected -// VTXOs and calls BulkSweepMarkers with the deduplicated set. This tests the core -// optimization where multiple VTXOs sharing markers result in fewer marker sweep -// operations (3 VTXOs with overlapping markers should yield only 3 unique markers). -func TestCreateCheckpointSweepTask_BulkSweepsMarkers(t *testing.T) { - // Setup mocks +// newTestSweeper creates a fresh set of mocks and a sweeper instance for testing. +func newTestSweeper() ( + *mockWalletService, + *mockVtxoRepository, + *mockMarkerRepository, + *mockTxBuilder, + *sweeper, +) { wallet := &mockWalletService{} vtxoRepo := &mockVtxoRepository{} markerRepo := &mockMarkerRepository{} repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} builder := &mockTxBuilder{} scheduler := &mockScheduler{} - - // Create sweeper instance s := newSweeper(wallet, repoManager, builder, scheduler, "") + return wallet, vtxoRepo, markerRepo, builder, s +} + +// TestCreateCheckpointSweepTask_BulkSweepsMarkers verifies that when a checkpoint +// is swept, the sweeper correctly collects all unique marker IDs from the affected +// VTXOs and calls BulkSweepMarkers with the deduplicated set. This tests the core +// optimization where multiple VTXOs sharing markers result in fewer marker sweep +// operations (3 VTXOs with overlapping markers should yield only 3 unique markers). +func TestCreateCheckpointSweepTask_BulkSweepsMarkers(t *testing.T) { + wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() // Test data checkpointTxid := "checkpoint123" @@ -572,16 +581,7 @@ func TestCreateCheckpointSweepTask_BulkSweepsMarkers(t *testing.T) { // This is an edge case that could occur with legacy VTXOs or during error recovery, // and ensures the sweeper handles it gracefully without attempting empty bulk operations. func TestCreateCheckpointSweepTask_NoMarkersSkipsSweep(t *testing.T) { - // Setup mocks - wallet := &mockWalletService{} - vtxoRepo := &mockVtxoRepository{} - markerRepo := &mockMarkerRepository{} - repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} - builder := &mockTxBuilder{} - scheduler := &mockScheduler{} - - // Create sweeper instance - s := newSweeper(wallet, repoManager, builder, scheduler, "") + wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() // Test data checkpointTxid := "checkpoint456" @@ -639,14 +639,7 @@ func TestCreateCheckpointSweepTask_NoMarkersSkipsSweep(t *testing.T) { // marker to every existing VTXO, ensuring backward compatibility with the new marker system. func TestCreateCheckpointSweepTask_SingleMarkerPerVtxo(t *testing.T) { // Test case: each VTXO has exactly one marker (post-migration state) - wallet := &mockWalletService{} - vtxoRepo := &mockVtxoRepository{} - markerRepo := &mockMarkerRepository{} - repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} - builder := &mockTxBuilder{} - scheduler := &mockScheduler{} - - s := newSweeper(wallet, repoManager, builder, scheduler, "") + wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() checkpointTxid := "checkpoint789" vtxoOutpoint := domain.Outpoint{Txid: "vtxo789", VOut: 0} @@ -716,14 +709,7 @@ func TestCreateCheckpointSweepTask_SingleMarkerPerVtxo(t *testing.T) { // 50 VTXOs, only 2 unique markers should be swept, demonstrating the efficiency gain. func TestCreateCheckpointSweepTask_ManyVtxosWithSharedMarkers(t *testing.T) { // Test case: many VTXOs share markers (chain with depth > 100) - wallet := &mockWalletService{} - vtxoRepo := &mockVtxoRepository{} - markerRepo := &mockMarkerRepository{} - repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} - builder := &mockTxBuilder{} - scheduler := &mockScheduler{} - - s := newSweeper(wallet, repoManager, builder, scheduler, "") + wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() checkpointTxid := "checkpoint_deep" vtxoOutpoint := domain.Outpoint{Txid: "vtxo_deep", VOut: 0} @@ -792,14 +778,7 @@ func TestCreateCheckpointSweepTask_ManyVtxosWithSharedMarkers(t *testing.T) { // than being a stale or incorrect value. func TestCreateCheckpointSweepTask_SweptAtTimestamp(t *testing.T) { // Test that the sweptAt timestamp is reasonable (within a few seconds of now) - wallet := &mockWalletService{} - vtxoRepo := &mockVtxoRepository{} - markerRepo := &mockMarkerRepository{} - repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} - builder := &mockTxBuilder{} - scheduler := &mockScheduler{} - - s := newSweeper(wallet, repoManager, builder, scheduler, "") + wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() checkpointTxid := "checkpoint_timestamp" vtxoOutpoint := domain.Outpoint{Txid: "vtxo_timestamp", VOut: 0} @@ -850,14 +829,7 @@ func TestCreateCheckpointSweepTask_SweptAtTimestamp(t *testing.T) { // that marker sweep failures are not silently ignored and can be properly handled by // the calling code for retry logic or alerting. func TestCreateCheckpointSweepTask_BulkSweepMarkersError(t *testing.T) { - wallet := &mockWalletService{} - vtxoRepo := &mockVtxoRepository{} - markerRepo := &mockMarkerRepository{} - repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} - builder := &mockTxBuilder{} - scheduler := &mockScheduler{} - - s := newSweeper(wallet, repoManager, builder, scheduler, "") + wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() checkpointTxid := "checkpoint_error" vtxoOutpoint := domain.Outpoint{Txid: "vtxo_error", VOut: 0} @@ -900,14 +872,7 @@ func TestCreateCheckpointSweepTask_BulkSweepMarkersError(t *testing.T) { // retrieve the VTXOs associated with child outpoints, the error is properly propagated. // This tests the error handling path before marker collection even begins. func TestCreateCheckpointSweepTask_GetVtxosError(t *testing.T) { - wallet := &mockWalletService{} - vtxoRepo := &mockVtxoRepository{} - markerRepo := &mockMarkerRepository{} - repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} - builder := &mockTxBuilder{} - scheduler := &mockScheduler{} - - s := newSweeper(wallet, repoManager, builder, scheduler, "") + wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() checkpointTxid := "checkpoint_vtxo_err" vtxoOutpoint := domain.Outpoint{Txid: "vtxo_vtxo_err", VOut: 0} @@ -944,14 +909,7 @@ func TestCreateCheckpointSweepTask_GetVtxosError(t *testing.T) { // GetAllChildrenVtxos fails to retrieve child outpoints, the error is propagated. // This tests the earliest error handling path in the sweep task. func TestCreateCheckpointSweepTask_GetAllChildrenVtxosError(t *testing.T) { - wallet := &mockWalletService{} - vtxoRepo := &mockVtxoRepository{} - markerRepo := &mockMarkerRepository{} - repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} - builder := &mockTxBuilder{} - scheduler := &mockScheduler{} - - s := newSweeper(wallet, repoManager, builder, scheduler, "") + wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() checkpointTxid := "checkpoint_children_err" vtxoOutpoint := domain.Outpoint{Txid: "vtxo_children_err", VOut: 0} @@ -984,14 +942,7 @@ func TestCreateCheckpointSweepTask_GetAllChildrenVtxosError(t *testing.T) { // fails to create the sweep transaction, the error is propagated and no marker // operations are attempted. This tests the very first error handling path. func TestCreateCheckpointSweepTask_BuildSweepTxError(t *testing.T) { - wallet := &mockWalletService{} - vtxoRepo := &mockVtxoRepository{} - markerRepo := &mockMarkerRepository{} - repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} - builder := &mockTxBuilder{} - scheduler := &mockScheduler{} - - s := newSweeper(wallet, repoManager, builder, scheduler, "") + wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() checkpointTxid := "checkpoint_build_err" vtxoOutpoint := domain.Outpoint{Txid: "vtxo_build_err", VOut: 0} @@ -1019,14 +970,7 @@ func TestCreateCheckpointSweepTask_BuildSweepTxError(t *testing.T) { // fails, the error is propagated and marker sweep operations are not attempted. // This ensures we don't mark VTXOs as swept if the sweep transaction wasn't actually broadcast. func TestCreateCheckpointSweepTask_BroadcastError(t *testing.T) { - wallet := &mockWalletService{} - vtxoRepo := &mockVtxoRepository{} - markerRepo := &mockMarkerRepository{} - repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} - builder := &mockTxBuilder{} - scheduler := &mockScheduler{} - - s := newSweeper(wallet, repoManager, builder, scheduler, "") + wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() checkpointTxid := "checkpoint_broadcast_err" vtxoOutpoint := domain.Outpoint{Txid: "vtxo_broadcast_err", VOut: 0} @@ -1056,14 +1000,7 @@ func TestCreateCheckpointSweepTask_BroadcastError(t *testing.T) { // GetAllChildrenVtxos returns an empty slice (no children under the unrolled // vtxo), the sweeper does not attempt to fetch VTXOs or sweep markers. func TestCreateCheckpointSweepTask_NoChildrenVtxos(t *testing.T) { - wallet := &mockWalletService{} - vtxoRepo := &mockVtxoRepository{} - markerRepo := &mockMarkerRepository{} - repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} - builder := &mockTxBuilder{} - scheduler := &mockScheduler{} - - s := newSweeper(wallet, repoManager, builder, scheduler, "") + wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() checkpointTxid := "checkpoint_no_children" vtxoOutpoint := domain.Outpoint{Txid: "vtxo_no_children", VOut: 0} @@ -1099,14 +1036,7 @@ func TestCreateCheckpointSweepTask_NoChildrenVtxos(t *testing.T) { // markers are passed to BulkSweepMarkers. For example, 5 VTXOs each carrying // {"marker-X", "marker-Y"} should result in exactly 2 markers being swept. func TestCreateCheckpointSweepTask_DuplicateMarkersAcrossVtxos(t *testing.T) { - wallet := &mockWalletService{} - vtxoRepo := &mockVtxoRepository{} - markerRepo := &mockMarkerRepository{} - repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} - builder := &mockTxBuilder{} - scheduler := &mockScheduler{} - - s := newSweeper(wallet, repoManager, builder, scheduler, "") + wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() checkpointTxid := "checkpoint_dup" vtxoOutpoint := domain.Outpoint{Txid: "vtxo_dup", VOut: 0} @@ -1169,14 +1099,7 @@ func TestCreateCheckpointSweepTask_DuplicateMarkersAcrossVtxos(t *testing.T) { // or iteration, and that the deduplicated set is passed correctly to // BulkSweepMarkers. func TestCreateCheckpointSweepTask_LargeMarkerSet(t *testing.T) { - wallet := &mockWalletService{} - vtxoRepo := &mockVtxoRepository{} - markerRepo := &mockMarkerRepository{} - repoManager := &mockRepoManager{vtxos: vtxoRepo, markers: markerRepo} - builder := &mockTxBuilder{} - scheduler := &mockScheduler{} - - s := newSweeper(wallet, repoManager, builder, scheduler, "") + wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() checkpointTxid := "checkpoint_large" vtxoOutpoint := domain.Outpoint{Txid: "vtxo_large", VOut: 0} diff --git a/internal/core/application/utils_test.go b/internal/core/application/utils_test.go index 7b84c821e..3510de0b9 100644 --- a/internal/core/application/utils_test.go +++ b/internal/core/application/utils_test.go @@ -52,6 +52,10 @@ func makeP2TRLeafTx(t *testing.T, outputs []struct { return b64 } +// TestGetNewVtxosFromRound_MarkerIDsAndDepth verifies that getNewVtxosFromRound +// correctly assigns Depth=0 and MarkerIDs=[outpoint.String()] to every leaf VTXO +// produced from a round's VTXO tree. Also checks that commitment references, amounts, +// pubkeys, and sequential VOut indices are set correctly for multi-output leaf transactions. func TestGetNewVtxosFromRound_MarkerIDsAndDepth(t *testing.T) { // Generate two distinct keys for two outputs privKey1, err := btcec.NewPrivateKey() @@ -120,6 +124,8 @@ func TestGetNewVtxosFromRound_MarkerIDsAndDepth(t *testing.T) { require.Equal(t, vtxos[0].Txid, vtxos[1].Txid) } +// TestGetNewVtxosFromRound_EmptyVtxoTree verifies that a round with a nil VTXO tree +// returns nil, handling the edge case where no leaf transactions exist. func TestGetNewVtxosFromRound_EmptyVtxoTree(t *testing.T) { round := &domain.Round{ CommitmentTxid: "empty-round", @@ -130,6 +136,9 @@ func TestGetNewVtxosFromRound_EmptyVtxoTree(t *testing.T) { require.Nil(t, vtxos) } +// TestGetNewVtxosFromRound_SingleOutput verifies that a leaf transaction with a single +// P2TR output produces exactly one VTXO with Depth=0, the correct self-referencing +// MarkerID, the expected amount, and VOut=0. func TestGetNewVtxosFromRound_SingleOutput(t *testing.T) { privKey, err := btcec.NewPrivateKey() require.NoError(t, err) diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index 0308b4ada..436a45f4c 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -1381,6 +1381,8 @@ func testVtxoRepository(t *testing.T, svc ports.RepoManager) { } }) + // Verifies that the Depth field persists through AddVtxos→GetVtxos for VTXOs + // at various chain depths (0, 1, 2, 100). t.Run("test_vtxo_depth", func(t *testing.T) { ctx := context.Background() commitmentTxid := randomString(32) @@ -1455,6 +1457,10 @@ func testVtxoRepository(t *testing.T, svc ports.RepoManager) { }) } +// testMarkerBasicOperations exercises AddMarker, GetMarker, GetMarkersByDepth, and +// GetMarkersByIds. Creates a 4-marker DAG (root, two at depth 100, one at depth 200 +// with two parents), verifies field round-trips including ParentMarkerIDs, and tests +// edge cases: non-existent ID, empty ID slice, and mixed valid/invalid ID queries. func testMarkerBasicOperations(t *testing.T, svc ports.RepoManager) { t.Run("test_marker_basic_operations", func(t *testing.T) { if svc.Markers() == nil { @@ -1575,6 +1581,11 @@ func testMarkerBasicOperations(t *testing.T, svc ports.RepoManager) { }) } +// testMarkerSweep exercises the full marker sweep lifecycle: SweepMarker, IsMarkerSwept, +// GetSweptMarkers, and SweepMarkerWithDescendants. Verifies idempotency (ON CONFLICT +// DO NOTHING preserves original timestamp), multi-marker retrieval, empty-slice edge +// cases, non-existent marker handling, and recursive descendant sweeping with hierarchy +// (root→child1→grandchild1, root→child2). func testMarkerSweep(t *testing.T, svc ports.RepoManager) { t.Run("test_marker_sweep", func(t *testing.T) { if svc.Markers() == nil { @@ -1733,6 +1744,9 @@ func testMarkerSweep(t *testing.T, svc ports.RepoManager) { }) } +// testVtxoMarkerAssociation verifies UpdateVtxoMarkers correctly links VTXOs to markers +// and that the association is visible through both GetVtxosByMarker and GetVtxos. Tests +// that unassociated VTXOs remain marker-free and that non-existent markers return empty. func testVtxoMarkerAssociation(t *testing.T, svc ports.RepoManager) { t.Run("test_vtxo_marker_association", func(t *testing.T) { if svc.Markers() == nil { @@ -1828,6 +1842,10 @@ func testVtxoMarkerAssociation(t *testing.T, svc ports.RepoManager) { }) } +// testSweepVtxosByMarker creates 5 VTXOs sharing a marker, pre-sweeps 2 via individual +// markers, then calls SweepVtxosByMarker on the shared marker. Verifies only the 3 +// previously-unswept VTXOs are newly swept, tests idempotency (second call returns 0), +// and checks the non-existent marker edge case. func testSweepVtxosByMarker(t *testing.T, svc ports.RepoManager) { t.Run("test_sweep_vtxos_by_marker", func(t *testing.T) { if svc.Markers() == nil { @@ -1925,6 +1943,10 @@ func testSweepVtxosByMarker(t *testing.T, svc ports.RepoManager) { }) } +// testMarkerDepthRangeQueries verifies GetMarkersByDepthRange and GetVtxosByDepthRange +// return correct results for inclusive depth ranges. Tests partial ranges, full ranges, +// and empty ranges for both markers (at depths 0/100/200/300) and VTXOs (at depths +// 0/50/100/150). func testMarkerDepthRangeQueries(t *testing.T, svc ports.RepoManager) { t.Run("test_marker_depth_range_queries", func(t *testing.T) { if svc.Markers() == nil { @@ -2090,6 +2112,9 @@ func testMarkerDepthRangeQueries(t *testing.T, svc ports.RepoManager) { }) } +// testMarkerChainTraversal creates a two-marker chain with VTXOs linked by ark txid, +// then verifies GetVtxoChainByMarkers returns the correct VTXOs for single and +// multi-marker queries. Also tests GetVtxosByArkTxid and edge cases (empty/non-existent). func testMarkerChainTraversal(t *testing.T, svc ports.RepoManager) { t.Run("test_marker_chain_traversal", func(t *testing.T) { if svc.Markers() == nil { @@ -2650,6 +2675,9 @@ func testBulkSweepMarkersConcurrent(t *testing.T, svc ports.RepoManager) { }) } +// testCreateRootMarkersForVtxos verifies that CreateRootMarkersForVtxos creates a +// depth-0 root marker for each batch VTXO using the outpoint string as the marker ID. +// Also tests idempotency — calling again with the same VTXOs does not error. func testCreateRootMarkersForVtxos(t *testing.T, svc ports.RepoManager) { t.Run("test_create_root_markers_for_vtxos", func(t *testing.T) { if svc.Markers() == nil { @@ -2719,6 +2747,10 @@ func testCreateRootMarkersForVtxos(t *testing.T, svc ports.RepoManager) { }) } +// testMarkerCreationAtBoundaryDepth simulates the service logic when a child VTXO +// lands at a marker boundary (depth 100). Verifies that a new marker is created with +// the parent's marker IDs as its ParentMarkerIDs, and that the child VTXO carries +// only the new marker ID. func testMarkerCreationAtBoundaryDepth(t *testing.T, svc ports.RepoManager) { t.Run("test_marker_creation_at_boundary_depth", func(t *testing.T) { if svc.Markers() == nil { @@ -2795,6 +2827,10 @@ func testMarkerCreationAtBoundaryDepth(t *testing.T, svc ports.RepoManager) { }) } +// testMarkerInheritanceAtNonBoundary verifies that a child VTXO at a non-boundary +// depth (e.g. 51) inherits all parent marker IDs rather than creating a new marker. +// Confirms the inherited markers persist through a DB round trip and no spurious +// marker is created. func testMarkerInheritanceAtNonBoundary(t *testing.T, svc ports.RepoManager) { t.Run("test_marker_inheritance_at_non_boundary", func(t *testing.T) { if svc.Markers() == nil { @@ -2870,6 +2906,9 @@ func testMarkerInheritanceAtNonBoundary(t *testing.T, svc ports.RepoManager) { }) } +// testDustVtxoMarkersSweptImmediately simulates the immediate sweep of dust VTXO +// markers that occurs in updateProjectionsAfterOffchainTxEvents. Verifies that +// BulkSweepMarkers marks dust markers as swept with the correct timestamp. func testDustVtxoMarkersSweptImmediately(t *testing.T, svc ports.RepoManager) { t.Run("test_dust_vtxo_markers_swept_immediately", func(t *testing.T) { if svc.Markers() == nil { @@ -2929,6 +2968,9 @@ func testDustVtxoMarkersSweptImmediately(t *testing.T, svc ports.RepoManager) { }) } +// testSweepVtxosWithMarkersEmptyInput verifies that BulkSweepMarkers handles an +// empty marker ID slice without errors, covering the early-return path when there +// are no VTXOs to sweep. func testSweepVtxosWithMarkersEmptyInput(t *testing.T, svc ports.RepoManager) { t.Run("test_sweep_vtxos_with_markers_empty_input", func(t *testing.T) { if svc.Markers() == nil { @@ -2949,6 +2991,9 @@ func testSweepVtxosWithMarkersEmptyInput(t *testing.T, svc ports.RepoManager) { }) } +// testSweepVtxosWithMarkersNoMarkersOnVtxos verifies that VTXOs with empty or nil +// MarkerIDs produce an empty marker set when collected, ensuring the sweep logic +// gracefully skips marker operations for legacy or marker-less VTXOs. func testSweepVtxosWithMarkersNoMarkersOnVtxos(t *testing.T, svc ports.RepoManager) { t.Run("test_sweep_vtxos_with_markers_no_markers_on_vtxos", func(t *testing.T) { if svc.Markers() == nil { @@ -2999,6 +3044,9 @@ func testSweepVtxosWithMarkersNoMarkersOnVtxos(t *testing.T, svc ports.RepoManag }) } +// testVtxoMarkerIDsRoundTrip verifies that MarkerIDs and Depth survive a write→read +// round trip through the database for various configurations: single marker, multiple +// markers, empty markers, nil markers, and deep VTXOs with two markers. func testVtxoMarkerIDsRoundTrip(t *testing.T, svc ports.RepoManager) { t.Run("test_vtxo_marker_ids_round_trip", func(t *testing.T) { if svc.Markers() == nil { @@ -3098,6 +3146,9 @@ func testVtxoMarkerIDsRoundTrip(t *testing.T, svc ports.RepoManager) { }) } +// testGetVtxosByArkTxidMultipleOutputs verifies that GetVtxosByArkTxid returns all +// VTXOs (multiple vouts) produced by a single ark transaction, each with the correct +// depth, markers, and amounts. Also checks that a non-existent ark txid returns empty. func testGetVtxosByArkTxidMultipleOutputs(t *testing.T, svc ports.RepoManager) { t.Run("test_get_vtxos_by_ark_txid_multiple_outputs", func(t *testing.T) { if svc.Markers() == nil { @@ -3174,6 +3225,8 @@ func testGetVtxosByArkTxidMultipleOutputs(t *testing.T, svc ports.RepoManager) { }) } +// testCreateRootMarkersForEmptyVtxos verifies that CreateRootMarkersForVtxos handles +// empty and nil VTXO slices gracefully without errors or side effects. func testCreateRootMarkersForEmptyVtxos(t *testing.T, svc ports.RepoManager) { t.Run("test_create_root_markers_for_empty_vtxos", func(t *testing.T) { if svc.Markers() == nil { @@ -4115,6 +4168,9 @@ func testSweepVtxosWithMarkersIntegration(t *testing.T, svc ports.RepoManager) { }) } +// testPartialMarkerSweep creates a 3-marker chain (depth 0→100→200) with 2 VTXOs +// per marker, sweeps only the deeper two markers, and verifies that VTXOs under the +// unswept root marker remain unswept while VTXOs under swept markers are marked as swept. func testPartialMarkerSweep(t *testing.T, svc ports.RepoManager) { t.Run("test_partial_marker_sweep", func(t *testing.T) { if svc.Markers() == nil { @@ -4253,6 +4309,10 @@ func testPartialMarkerSweep(t *testing.T, svc ports.RepoManager) { }) } +// testListVtxosMarkerSweptFiltering verifies that GetAllNonUnrolledVtxos correctly +// classifies VTXOs as spent/unspent based on marker sweep status. Creates 4 VTXOs +// across two markers, sweeps one marker, and confirms the swept VTXOs appear in the +// spent list while the unswept ones remain in the unspent list. func testListVtxosMarkerSweptFiltering(t *testing.T, svc ports.RepoManager) { t.Run("test_list_vtxos_marker_swept_filtering", func(t *testing.T) { if svc.Markers() == nil { @@ -4387,6 +4447,9 @@ func testListVtxosMarkerSweptFiltering(t *testing.T, svc ports.RepoManager) { }) } +// testSweepableUnrolledExcludesMarkerSwept verifies that GetAllSweepableUnrolledVtxos +// excludes VTXOs whose markers have been swept. Creates 3 spent+unrolled VTXOs across +// two markers, sweeps one marker, and confirms only the unswept VTXOs appear as sweepable. func testSweepableUnrolledExcludesMarkerSwept(t *testing.T, svc ports.RepoManager) { t.Run("test_sweepable_unrolled_excludes_marker_swept", func(t *testing.T) { if svc.Markers() == nil { @@ -4516,6 +4579,10 @@ func testSweepableUnrolledExcludesMarkerSwept(t *testing.T, svc ports.RepoManage }) } +// testConvergentMultiParentMarkerDAG builds a diamond-shaped marker DAG where two +// independent root→mid branches converge into a single merge marker, then extend +// to a leaf. Verifies GetVtxoChainByMarkers returns correct VTXOs per marker set, +// and that sweeping individual markers only affects VTXOs associated with those markers. func testConvergentMultiParentMarkerDAG(t *testing.T, svc ports.RepoManager) { t.Run("test_convergent_multi_parent_marker_dag", func(t *testing.T) { if svc.Markers() == nil { @@ -4685,6 +4752,10 @@ func testConvergentMultiParentMarkerDAG(t *testing.T, svc ports.RepoManager) { }) } +// testSweepMarkerWithDescendantsDeepChain builds a 201-marker linear chain (depth 0 +// to 20000) and calls SweepMarkerWithDescendants from the root. Verifies all 201 +// markers are swept in a single recursive operation and that a second call is +// idempotent (returns 0). func testSweepMarkerWithDescendantsDeepChain(t *testing.T, svc ports.RepoManager) { t.Run("test_sweep_marker_with_descendants_deep_chain", func(t *testing.T) { if svc.Markers() == nil { diff --git a/internal/test/e2e/e2e_test.go b/internal/test/e2e/e2e_test.go index 980ed5ecf..3ff653117 100644 --- a/internal/test/e2e/e2e_test.go +++ b/internal/test/e2e/e2e_test.go @@ -4134,10 +4134,6 @@ func TestFee(t *testing.T) { require.Empty(t, bobBalance.OnchainBalance.LockedAmount) } -// TODO: TestVtxoDepth is commented out until the SDK proto package includes the Depth field. -// Once github.com/arkade-os/go-sdk/api-spec/protobuf/gen/ark/v1 has GetDepth() on IndexerVtxo, -// this test can be re-enabled to verify that VTXO depth increments correctly during offchain transactions. - func TestAsset(t *testing.T) { // This test ensures that an asset vtxo can be issued, transfered and then refreshed t.Run("transfer and renew", func(t *testing.T) { diff --git a/internal/test/e2e/utils_test.go b/internal/test/e2e/utils_test.go index 7c811b2ed..3b67d329e 100644 --- a/internal/test/e2e/utils_test.go +++ b/internal/test/e2e/utils_test.go @@ -740,9 +740,6 @@ func refill(httpClient *http.Client) error { return nil } -// TODO: setupRawIndexerClient and getVtxoDepthByOutpoint are commented out until -// the SDK proto package includes the Depth field on IndexerVtxo. - func listVtxosWithAsset(t *testing.T, client arksdk.ArkClient, assetID string) []types.Vtxo { t.Helper() vtxos, err := client.ListSpendableVtxos(t.Context()) From fe8a75595c1084a6a468284a827ee7cb1863e442 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Mon, 16 Feb 2026 22:51:43 -0500 Subject: [PATCH 27/54] sqlite json_each, badger touchups --- internal/core/application/indexer.go | 4 +- .../infrastructure/db/badger/marker_repo.go | 124 ++++++------------ .../infrastructure/db/badger/vtxo_repo.go | 5 - ...60210000000_add_depth_and_markers.down.sql | 2 +- ...0260210000000_add_depth_and_markers.up.sql | 2 +- .../db/sqlite/sqlc/queries/query.sql.go | 18 ++- .../infrastructure/db/sqlite/sqlc/query.sql | 18 ++- 7 files changed, 64 insertions(+), 109 deletions(-) diff --git a/internal/core/application/indexer.go b/internal/core/application/indexer.go index d60bd14e9..ecad6a820 100644 --- a/internal/core/application/indexer.go +++ b/internal/core/application/indexer.go @@ -417,7 +417,9 @@ func (i *indexerService) prefetchVtxosByMarkers( markerIDs := make([]string, 0, len(startVtxo.MarkerIDs)) markerIDs = append(markerIDs, startVtxo.MarkerIDs...) - // Bob: we have to follow all parent markers because a vtxo can be associated with multiple markers if it was created at a depth that is a multiple of the marker interval. For example, if the marker interval is 100, a vtxo created at depth 200 would be associated with the markers at depth 100 and 200. To ensure we prefetch all relevant VTXOs, we need to follow all parent markers up the chain until we reach the root marker (depth 0). + // We have to follow all parent markers because a vtxo can be associated with multiple markers if it was created at a depth that is + // a multiple of the marker interval. For example, if the marker interval is 100, a vtxo created at depth 200 would be associated + // with the markers at depth 100 and 200. To ensure we prefetch all relevant VTXOs, we need to follow all parent markers up the chain until we reach the root marker (depth 0). // BFS to follow all parent markers visited := make(map[string]bool) for _, id := range startVtxo.MarkerIDs { diff --git a/internal/infrastructure/db/badger/marker_repo.go b/internal/infrastructure/db/badger/marker_repo.go index 62f440341..c2b67192e 100644 --- a/internal/infrastructure/db/badger/marker_repo.go +++ b/internal/infrastructure/db/badger/marker_repo.go @@ -255,21 +255,31 @@ func (r *markerRepository) SweepMarker(ctx context.Context, markerID string, swe } } - // Update Swept field on VTXOs that contain this marker - // This keeps the stored Swept field in sync for query compatibility + // Update Swept field on VTXOs that contain this marker. + // This keeps the stored Swept field in sync for query compatibility. + // Errors here are non-fatal since swept_marker is already recorded. var filteredDtos []vtxoDTO if err := r.vtxoStore.Find( &filteredDtos, badgerhold.Where("MarkerIDs").Contains(markerID), ); err != nil { - return nil // Non-fatal, swept_marker is already updated + return nil } for _, dto := range filteredDtos { if !dto.Swept { dto.Swept = true dto.UpdatedAt = time.Now().UnixMilli() - _ = r.vtxoStore.Update(dto.Outpoint.String(), dto) + if err := r.vtxoStore.Update(dto.Outpoint.String(), dto); err != nil { + if errors.Is(err, badger.ErrConflict) { + for attempts := 1; attempts <= maxRetries; attempts++ { + time.Sleep(100 * time.Millisecond) + if err = r.vtxoStore.Update(dto.Outpoint.String(), dto); err == nil { + break + } + } + } + } } } @@ -447,81 +457,26 @@ func (r *markerRepository) GetVtxosByMarker( vtxos := make([]domain.Vtxo, 0, len(dtos)) for _, dto := range dtos { - vtxo := dto.Vtxo - // Compute Swept status dynamically by checking if any marker is swept - vtxo.Swept = r.isAnyMarkerSwept(dto.MarkerIDs) - vtxos = append(vtxos, vtxo) + vtxos = append(vtxos, dto.Vtxo) } return vtxos, nil } -// isAnyMarkerSwept checks if any of the given markers are in the swept_marker store -func (r *markerRepository) isAnyMarkerSwept(markerIDs []string) bool { - for _, markerID := range markerIDs { - var dto sweptMarkerDTO - err := r.sweptMarkerStore.Get(markerID, &dto) - if err == nil { - return true - } - } - return false -} - func (r *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID string) (int64, error) { - // For badger, we need to: - // 1. Mark the marker as swept - // 2. Update vtxo.Swept field for all VTXOs with this marker (for query compatibility) - - // Find all VTXOs whose MarkerIDs contains markerID and are not swept - var allDtos []vtxoDTO - err := r.vtxoStore.Find(&allDtos, badgerhold.Where("Swept").Eq(false)) - if err != nil { + // Mark the marker as swept (this also updates vtxo Swept fields) + if err := r.SweepMarker(ctx, markerID, time.Now().Unix()); err != nil { return 0, err } - var count int64 - for _, dto := range allDtos { - // Check if this VTXO has the markerID - hasMarker := false - for _, id := range dto.MarkerIDs { - if id == markerID { - hasMarker = true - break - } - } - if !hasMarker { - continue - } - - // Update the vtxo's Swept field - dto.Swept = true - dto.UpdatedAt = time.Now().UnixMilli() - - err := r.vtxoStore.Update(dto.Outpoint.String(), dto) - if err != nil { - if errors.Is(err, badger.ErrConflict) { - for attempts := 1; attempts <= maxRetries; attempts++ { - time.Sleep(100 * time.Millisecond) - err = r.vtxoStore.Update(dto.Outpoint.String(), dto) - if err == nil { - break - } - } - } - if err != nil { - return count, err - } - } - count++ - } - - // Also insert the marker into swept_marker for consistency - if err := r.SweepMarker(ctx, markerID, time.Now().Unix()); err != nil { - // Non-fatal - the vtxos are already marked as swept - _ = err + // Count VTXOs affected + var dtos []vtxoDTO + err := r.vtxoStore.Find(&dtos, + badgerhold.Where("MarkerIDs").Contains(markerID)) + if err != nil { + return 0, err } - return count, nil + return int64(len(dtos)), nil } func (r *markerRepository) CreateRootMarkersForVtxos( @@ -592,26 +547,21 @@ func (r *markerRepository) GetVtxoChainByMarkers( return nil, nil } - // Build a set of marker IDs for efficient lookup - markerIDSet := make(map[string]bool) - for _, id := range markerIDs { - markerIDSet[id] = true - } - - // Find all VTXOs that have any marker_id in our set - var dtos []vtxoDTO - err := r.vtxoStore.Find(&dtos, &badgerhold.Query{}) - if err != nil { - return nil, err - } - + seen := make(map[string]bool) vtxos := make([]domain.Vtxo, 0) - for _, dto := range dtos { - // Check if any of the VTXO's markers are in our set - for _, markerID := range dto.MarkerIDs { - if markerIDSet[markerID] { + + for _, markerID := range markerIDs { + var dtos []vtxoDTO + err := r.vtxoStore.Find(&dtos, + badgerhold.Where("MarkerIDs").Contains(markerID)) + if err != nil { + return nil, err + } + for _, dto := range dtos { + key := dto.Outpoint.String() + if !seen[key] { + seen[key] = true vtxos = append(vtxos, dto.Vtxo) - break } } } diff --git a/internal/infrastructure/db/badger/vtxo_repo.go b/internal/infrastructure/db/badger/vtxo_repo.go index 6ca13ad80..6a44420a1 100644 --- a/internal/infrastructure/db/badger/vtxo_repo.go +++ b/internal/infrastructure/db/badger/vtxo_repo.go @@ -418,11 +418,6 @@ func (r *VtxoRepository) Close() { r.store.Close() } -// Store returns the underlying badgerhold store for sharing with other repositories. -func (r *VtxoRepository) Store() *badgerhold.Store { - return r.store -} - func (r *VtxoRepository) addVtxos( ctx context.Context, vtxos []domain.Vtxo, ) error { diff --git a/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.down.sql b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.down.sql index b0cc2320a..7e0e5880f 100644 --- a/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.down.sql +++ b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.down.sql @@ -31,7 +31,7 @@ INSERT INTO vtxo_temp SELECT v.spent_by, v.spent, v.unrolled, EXISTS ( SELECT 1 FROM swept_marker sm - WHERE v.markers LIKE '%"' || sm.marker_id || '"%' + JOIN json_each(v.markers) j ON j.value = sm.marker_id ) AS swept, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at diff --git a/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql index 08955fa5f..1376100a3 100644 --- a/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql +++ b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql @@ -120,7 +120,7 @@ SELECT v.*, ), '') AS commitments, EXISTS ( SELECT 1 FROM swept_marker sm - WHERE v.markers LIKE '%"' || sm.marker_id || '"%' + JOIN json_each(v.markers) j ON j.value = sm.marker_id ) AS swept, COALESCE(ap.asset_id, '') AS asset_id, COALESCE(ap.amount, 0) AS asset_amount diff --git a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go index c1bf47f5d..6e8500eef 100644 --- a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go @@ -88,7 +88,8 @@ const countUnsweptVtxosByMarkerId = `-- name: CountUnsweptVtxosByMarkerId :one SELECT COUNT(*) FROM vtxo_vw WHERE markers LIKE '%"' || ?1 || '"%' AND swept = false ` -// Count VTXOs whose markers JSON array contains the given marker_id and are not swept +// Count VTXOs whose markers JSON array contains the given marker_id and are not swept. +// Uses LIKE because sqlc cannot parse json_each with view columns. func (q *Queries) CountUnsweptVtxosByMarkerId(ctx context.Context, markerID sql.NullString) (int64, error) { row := q.db.QueryRowContext(ctx, countUnsweptVtxosByMarkerId, markerID) var count int64 @@ -553,7 +554,7 @@ SELECT COALESCE(SUM(v.amount), 0) AS amount FROM vtxo v WHERE NOT EXISTS ( SELECT 1 FROM swept_marker sm - WHERE v.markers LIKE '%"' || sm.marker_id || '"%' + JOIN json_each(v.markers) j ON j.value = sm.marker_id ) AND v.spent = false AND v.unrolled = false @@ -1032,7 +1033,7 @@ SELECT COALESCE(SUM(v.amount), 0) AS amount FROM vtxo v WHERE EXISTS ( SELECT 1 FROM swept_marker sm - WHERE v.markers LIKE '%"' || sm.marker_id || '"%' + JOIN json_each(v.markers) j ON j.value = sm.marker_id ) AND v.spent = false ` @@ -1940,7 +1941,7 @@ func (q *Queries) SelectVtxo(ctx context.Context, arg SelectVtxoParams) ([]Selec } const selectVtxoChainByMarker = `-- name: SelectVtxoChainByMarker :many -Select vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept, vtxo_vw.asset_id, vtxo_vw.asset_amount FROM vtxo_vw +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept, vtxo_vw.asset_id, vtxo_vw.asset_amount FROM vtxo_vw WHERE markers LIKE '%"' || ?1 || '"%' ORDER BY vtxo_vw.depth DESC ` @@ -1949,8 +1950,9 @@ type SelectVtxoChainByMarkerRow struct { VtxoVw VtxoVw } -// Get VTXOs whose markers array contains the given marker_id -// For multiple markers, call this multiple times and deduplicate in Go +// Get VTXOs whose markers array contains the given marker_id. +// For multiple markers, call this multiple times and deduplicate in Go. +// Uses LIKE because sqlc cannot parse json_each with view columns. func (q *Queries) SelectVtxoChainByMarker(ctx context.Context, markerID sql.NullString) ([]SelectVtxoChainByMarkerRow, error) { rows, err := q.db.QueryContext(ctx, selectVtxoChainByMarker, markerID) if err != nil { @@ -2157,7 +2159,9 @@ type SelectVtxosByMarkerIdRow struct { VtxoVw VtxoVw } -// Find VTXOs whose markers JSON array contains the given marker_id +// Find VTXOs whose markers JSON array contains the given marker_id. +// Uses LIKE because sqlc cannot parse json_each with view columns. +// Safe for txid:vout format marker IDs (no special characters). func (q *Queries) SelectVtxosByMarkerId(ctx context.Context, markerID sql.NullString) ([]SelectVtxosByMarkerIdRow, error) { rows, err := q.db.QueryContext(ctx, selectVtxosByMarkerId, markerID) if err != nil { diff --git a/internal/infrastructure/db/sqlite/sqlc/query.sql b/internal/infrastructure/db/sqlite/sqlc/query.sql index 236c9be1e..8a2fb04e6 100644 --- a/internal/infrastructure/db/sqlite/sqlc/query.sql +++ b/internal/infrastructure/db/sqlite/sqlc/query.sql @@ -262,7 +262,7 @@ SELECT COALESCE(SUM(v.amount), 0) AS amount FROM vtxo v WHERE NOT EXISTS ( SELECT 1 FROM swept_marker sm - WHERE v.markers LIKE '%"' || sm.marker_id || '"%' + JOIN json_each(v.markers) j ON j.value = sm.marker_id ) AND v.spent = false AND v.unrolled = false @@ -274,7 +274,7 @@ SELECT COALESCE(SUM(v.amount), 0) AS amount FROM vtxo v WHERE EXISTS ( SELECT 1 FROM swept_marker sm - WHERE v.markers LIKE '%"' || sm.marker_id || '"%' + JOIN json_each(v.markers) j ON j.value = sm.marker_id ) AND v.spent = false; @@ -488,11 +488,14 @@ WHERE descendant_markers.id NOT IN (SELECT sm.marker_id FROM swept_marker sm); UPDATE vtxo SET markers = @markers WHERE txid = @txid AND vout = @vout; -- name: SelectVtxosByMarkerId :many --- Find VTXOs whose markers JSON array contains the given marker_id +-- Find VTXOs whose markers JSON array contains the given marker_id. +-- Uses LIKE because sqlc cannot parse json_each with view columns. +-- Safe for txid:vout format marker IDs (no special characters). SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE markers LIKE '%"' || @marker_id || '"%'; -- name: CountUnsweptVtxosByMarkerId :one --- Count VTXOs whose markers JSON array contains the given marker_id and are not swept +-- Count VTXOs whose markers JSON array contains the given marker_id and are not swept. +-- Uses LIKE because sqlc cannot parse json_each with view columns. SELECT COUNT(*) FROM vtxo_vw WHERE markers LIKE '%"' || @marker_id || '"%' AND swept = false; -- Chain traversal queries for GetVtxoChain optimization @@ -508,9 +511,10 @@ ORDER BY depth DESC; SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE txid = @ark_txid; -- name: SelectVtxoChainByMarker :many --- Get VTXOs whose markers array contains the given marker_id --- For multiple markers, call this multiple times and deduplicate in Go -Select sqlc.embed(vtxo_vw) FROM vtxo_vw +-- Get VTXOs whose markers array contains the given marker_id. +-- For multiple markers, call this multiple times and deduplicate in Go. +-- Uses LIKE because sqlc cannot parse json_each with view columns. +SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE markers LIKE '%"' || @marker_id || '"%' ORDER BY vtxo_vw.depth DESC; From 55618c4839441c19fb7e2c6a5c38128627b36e60 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Tue, 17 Feb 2026 00:49:52 -0500 Subject: [PATCH 28/54] cursor-based pagination for GetVtxoChain with lazy marker-window loading --- .../swagger/ark/v1/indexer.openapi.json | 13 + api-spec/protobuf/ark/v1/indexer.proto | 2 + api-spec/protobuf/gen/ark/v1/indexer.pb.go | 25 +- .../protobuf/gen/ark/v1/indexer.pb.rgw.go | 4 +- internal/core/application/indexer.go | 221 +++--- internal/core/application/indexer_test.go | 718 +++--------------- internal/core/application/types.go | 9 +- internal/interface/grpc/handlers/indexer.go | 9 +- 8 files changed, 298 insertions(+), 703 deletions(-) diff --git a/api-spec/openapi/swagger/ark/v1/indexer.openapi.json b/api-spec/openapi/swagger/ark/v1/indexer.openapi.json index b48534d48..a6c7032d3 100644 --- a/api-spec/openapi/swagger/ark/v1/indexer.openapi.json +++ b/api-spec/openapi/swagger/ark/v1/indexer.openapi.json @@ -593,6 +593,13 @@ "format": "uint32" } }, + { + "name": "pageToken", + "in": "query", + "schema": { + "type": "string" + } + }, { "name": "page.size", "in": "query", @@ -995,6 +1002,9 @@ }, "page": { "$ref": "#/components/schemas/IndexerPageRequest" + }, + "pageToken": { + "type": "string" } } }, @@ -1008,6 +1018,9 @@ "$ref": "#/components/schemas/IndexerChain" } }, + "nextPageToken": { + "type": "string" + }, "page": { "$ref": "#/components/schemas/IndexerPageResponse" } diff --git a/api-spec/protobuf/ark/v1/indexer.proto b/api-spec/protobuf/ark/v1/indexer.proto index 2261d44bd..fbae54dd0 100644 --- a/api-spec/protobuf/ark/v1/indexer.proto +++ b/api-spec/protobuf/ark/v1/indexer.proto @@ -210,10 +210,12 @@ message GetVtxosResponse { message GetVtxoChainRequest { IndexerOutpoint outpoint = 1; IndexerPageRequest page = 2; + string page_token = 3; } message GetVtxoChainResponse { repeated IndexerChain chain = 1; IndexerPageResponse page = 2; + string next_page_token = 3; } message GetVirtualTxsRequest { diff --git a/api-spec/protobuf/gen/ark/v1/indexer.pb.go b/api-spec/protobuf/gen/ark/v1/indexer.pb.go index afd6ad520..0577a80a3 100644 --- a/api-spec/protobuf/gen/ark/v1/indexer.pb.go +++ b/api-spec/protobuf/gen/ark/v1/indexer.pb.go @@ -853,6 +853,7 @@ type GetVtxoChainRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Outpoint *IndexerOutpoint `protobuf:"bytes,1,opt,name=outpoint,proto3" json:"outpoint,omitempty"` Page *IndexerPageRequest `protobuf:"bytes,2,opt,name=page,proto3" json:"page,omitempty"` + PageToken string `protobuf:"bytes,3,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -901,10 +902,18 @@ func (x *GetVtxoChainRequest) GetPage() *IndexerPageRequest { return nil } +func (x *GetVtxoChainRequest) GetPageToken() string { + if x != nil { + return x.PageToken + } + return "" +} + type GetVtxoChainResponse struct { state protoimpl.MessageState `protogen:"open.v1"` Chain []*IndexerChain `protobuf:"bytes,1,rep,name=chain,proto3" json:"chain,omitempty"` Page *IndexerPageResponse `protobuf:"bytes,2,opt,name=page,proto3" json:"page,omitempty"` + NextPageToken string `protobuf:"bytes,3,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -953,6 +962,13 @@ func (x *GetVtxoChainResponse) GetPage() *IndexerPageResponse { return nil } +func (x *GetVtxoChainResponse) GetNextPageToken() string { + if x != nil { + return x.NextPageToken + } + return "" +} + type GetVirtualTxsRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Txids []string `protobuf:"bytes,1,rep,name=txids,proto3" json:"txids,omitempty"` @@ -2542,13 +2558,16 @@ const file_ark_v1_indexer_proto_rawDesc = "" + "\x06before\x18\t \x01(\x03R\x06before\"n\n" + "\x10GetVtxosResponse\x12)\n" + "\x05vtxos\x18\x01 \x03(\v2\x13.ark.v1.IndexerVtxoR\x05vtxos\x12/\n" + - "\x04page\x18\x02 \x01(\v2\x1b.ark.v1.IndexerPageResponseR\x04page\"z\n" + + "\x04page\x18\x02 \x01(\v2\x1b.ark.v1.IndexerPageResponseR\x04page\"\x99\x01\n" + "\x13GetVtxoChainRequest\x123\n" + "\boutpoint\x18\x01 \x01(\v2\x17.ark.v1.IndexerOutpointR\boutpoint\x12.\n" + - "\x04page\x18\x02 \x01(\v2\x1a.ark.v1.IndexerPageRequestR\x04page\"s\n" + + "\x04page\x18\x02 \x01(\v2\x1a.ark.v1.IndexerPageRequestR\x04page\x12\x1d\n" + + "\n" + + "page_token\x18\x03 \x01(\tR\tpageToken\"\x9b\x01\n" + "\x14GetVtxoChainResponse\x12*\n" + "\x05chain\x18\x01 \x03(\v2\x14.ark.v1.IndexerChainR\x05chain\x12/\n" + - "\x04page\x18\x02 \x01(\v2\x1b.ark.v1.IndexerPageResponseR\x04page\"\\\n" + + "\x04page\x18\x02 \x01(\v2\x1b.ark.v1.IndexerPageResponseR\x04page\x12&\n" + + "\x0fnext_page_token\x18\x03 \x01(\tR\rnextPageToken\"\\\n" + "\x14GetVirtualTxsRequest\x12\x14\n" + "\x05txids\x18\x01 \x03(\tR\x05txids\x12.\n" + "\x04page\x18\x02 \x01(\v2\x1a.ark.v1.IndexerPageRequestR\x04page\"Z\n" + diff --git a/api-spec/protobuf/gen/ark/v1/indexer.pb.rgw.go b/api-spec/protobuf/gen/ark/v1/indexer.pb.rgw.go index 64965a26f..397f10840 100644 --- a/api-spec/protobuf/gen/ark/v1/indexer.pb.rgw.go +++ b/api-spec/protobuf/gen/ark/v1/indexer.pb.rgw.go @@ -125,7 +125,7 @@ func request_IndexerService_GetConnectors_0(ctx context.Context, marshaler gatew var ( query_params_IndexerService_GetVtxoTree_0 = gateway.QueryParameterParseOptions{ - Filter: trie.New("batch_outpoint.txid", "batch_outpoint.vout", "txid", "vout"), + Filter: trie.New("vout", "batch_outpoint.txid", "batch_outpoint.vout", "txid"), } ) @@ -173,7 +173,7 @@ func request_IndexerService_GetVtxoTree_0(ctx context.Context, marshaler gateway var ( query_params_IndexerService_GetVtxoTreeLeaves_0 = gateway.QueryParameterParseOptions{ - Filter: trie.New("vout", "batch_outpoint.txid", "batch_outpoint.vout", "txid"), + Filter: trie.New("batch_outpoint.txid", "batch_outpoint.vout", "txid", "vout"), } ) diff --git a/internal/core/application/indexer.go b/internal/core/application/indexer.go index ecad6a820..172d75445 100644 --- a/internal/core/application/indexer.go +++ b/internal/core/application/indexer.go @@ -2,6 +2,8 @@ package application import ( "context" + "encoding/base64" + "encoding/json" "fmt" "math" "strings" @@ -35,7 +37,12 @@ type IndexerService interface { GetVtxosByOutpoint( ctx context.Context, outpoints []Outpoint, page *Page, ) (*GetVtxosResp, error) - GetVtxoChain(ctx context.Context, vtxoKey Outpoint, page *Page) (*VtxoChainResp, error) + GetVtxoChain( + ctx context.Context, + vtxoKey Outpoint, + page *Page, + pageToken string, + ) (*VtxoChainResp, error) GetVirtualTxs(ctx context.Context, txids []string, page *Page) (*VirtualTxsResp, error) GetBatchSweepTxs(ctx context.Context, batchOutpoint Outpoint) ([]string, error) GetAsset(ctx context.Context, assetID string) ([]Asset, error) @@ -247,20 +254,51 @@ func (i *indexerService) GetVtxosByOutpoint( } func (i *indexerService) GetVtxoChain( - ctx context.Context, vtxoKey Outpoint, page *Page, + ctx context.Context, vtxoKey Outpoint, page *Page, pageToken string, ) (*VtxoChainResp, error) { + // Determine page size. + // Backward compat: nil page + empty token → return full chain (no pagination). + pageSize := math.MaxInt32 + if page != nil { + pageSize = int(page.PageSize) + if pageSize <= 0 { + pageSize = maxPageSizeVtxoChain + } + } else if pageToken != "" { + pageSize = maxPageSizeVtxoChain + } + + // Determine frontier: decode pageToken, or use [vtxoKey] for first page. + var frontier []domain.Outpoint + if pageToken != "" { + decoded, err := decodeChainCursor(pageToken) + if err != nil { + return nil, fmt.Errorf("invalid page_token: %w", err) + } + frontier = decoded + } else { + frontier = []domain.Outpoint{vtxoKey} + } + chain := make([]ChainTx, 0) - nextVtxos := []domain.Outpoint{vtxoKey} + nextVtxos := frontier visited := make(map[string]bool) - // Pre-fetch VTXOs using markers for optimization (reduces DB calls for deep chains) - vtxoCache := i.prefetchVtxosByMarkers(ctx, vtxoKey) + // Lazy cache for VTXOs loaded during this page. + vtxoCache := make(map[string]domain.Vtxo) + loadedMarkers := make(map[string]bool) for len(nextVtxos) > 0 { - vtxos, err := i.getVtxosFromCacheOrDB(ctx, nextVtxos, vtxoCache) - if err != nil { + if err := i.ensureVtxosCached(ctx, nextVtxos, vtxoCache, loadedMarkers); err != nil { return nil, err } + + vtxos := make([]domain.Vtxo, 0, len(nextVtxos)) + for _, op := range nextVtxos { + if v, ok := vtxoCache[op.String()]; ok { + vtxos = append(vtxos, v) + } + } if len(vtxos) == 0 { return nil, fmt.Errorf("vtxo not found for outpoint: %v", nextVtxos) } @@ -273,6 +311,22 @@ func (i *indexerService) GetVtxoChain( } visited[key] = true + // Early termination: save unprocessed VTXOs to frontier for next page. + if len(chain) >= pageSize { + remaining := make([]domain.Outpoint, 0) + for _, v := range vtxos { + if !visited[v.Outpoint.String()] { + remaining = append(remaining, v.Outpoint) + } + } + remaining = append(remaining, newNextVtxos...) + token := encodeChainCursor(remaining) + return &VtxoChainResp{ + Chain: chain, + NextPageToken: token, + }, nil + } + // if the vtxo is preconfirmed, it means it has been created by an offchain tx // we need to add the virtual tx + the associated checkpoints txs // also, we have to populate the newNextVtxos with the checkpoints inputs @@ -379,118 +433,95 @@ func (i *indexerService) GetVtxoChain( nextVtxos = newNextVtxos } - txChain, pageResp := paginate(chain, page, maxPageSizeVtxoChain) + // Chain exhausted — no more pages. return &VtxoChainResp{ - Chain: txChain, - Page: pageResp, + Chain: chain, }, nil } -// prefetchVtxosByMarkers pre-fetches VTXOs using markers for optimization. -// This reduces the number of DB calls for deep chains by bulk fetching VTXOs -// associated with the marker chain instead of fetching one at a time. -func (i *indexerService) prefetchVtxosByMarkers( - ctx context.Context, startKey Outpoint, -) map[string]domain.Vtxo { - // outpoint string -> VTXO cache - cache := make(map[string]domain.Vtxo) - - if i.repoManager.Markers() == nil { - return cache - } - - // Get starting VTXO to find its marker - startVtxos, err := i.repoManager.Vtxos().GetVtxos(ctx, []domain.Outpoint{startKey}) - if err != nil || len(startVtxos) == 0 { - return cache +// encodeChainCursor encodes a frontier of outpoints into an opaque page token. +func encodeChainCursor(frontier []domain.Outpoint) string { + if len(frontier) == 0 { + return "" } - - startVtxo := startVtxos[0] - // Add starting VTXO to cache - cache[startVtxo.Outpoint.String()] = startVtxo - - if len(startVtxo.MarkerIDs) == 0 { - return cache - } - - // Collect marker chain by following ParentMarkerIDs from all markers - markerIDs := make([]string, 0, len(startVtxo.MarkerIDs)) - markerIDs = append(markerIDs, startVtxo.MarkerIDs...) - - // We have to follow all parent markers because a vtxo can be associated with multiple markers if it was created at a depth that is - // a multiple of the marker interval. For example, if the marker interval is 100, a vtxo created at depth 200 would be associated - // with the markers at depth 100 and 200. To ensure we prefetch all relevant VTXOs, we need to follow all parent markers up the chain until we reach the root marker (depth 0). - // BFS to follow all parent markers - visited := make(map[string]bool) - for _, id := range startVtxo.MarkerIDs { - visited[id] = true - } - - queue := make([]string, 0, len(startVtxo.MarkerIDs)) - queue = append(queue, startVtxo.MarkerIDs...) - - for len(queue) > 0 { - currentID := queue[0] - queue = queue[1:] - - marker, err := i.repoManager.Markers().GetMarker(ctx, currentID) - if err != nil || marker == nil { - continue - } - - for _, parentID := range marker.ParentMarkerIDs { - if !visited[parentID] { - visited[parentID] = true - markerIDs = append(markerIDs, parentID) - queue = append(queue, parentID) - } - } + cur := vtxoChainCursor{Frontier: make([]Outpoint, len(frontier))} + for i, op := range frontier { + cur.Frontier[i] = Outpoint(op) } + data, _ := json.Marshal(cur) + return base64.RawURLEncoding.EncodeToString(data) +} - // Bulk fetch VTXOs for all markers in the chain - vtxos, err := i.repoManager.Markers().GetVtxoChainByMarkers(ctx, markerIDs) +// decodeChainCursor decodes a page token back into a frontier of outpoints. +func decodeChainCursor(token string) ([]domain.Outpoint, error) { + data, err := base64.RawURLEncoding.DecodeString(token) if err != nil { - return cache + return nil, fmt.Errorf("invalid base64: %w", err) } - - for _, v := range vtxos { - cache[v.Outpoint.String()] = v + var cur vtxoChainCursor + if err := json.Unmarshal(data, &cur); err != nil { + return nil, fmt.Errorf("invalid JSON: %w", err) } - - return cache + outpoints := make([]domain.Outpoint, len(cur.Frontier)) + for i, op := range cur.Frontier { + outpoints[i] = domain.Outpoint(op) + } + return outpoints, nil } -// getVtxosFromCacheOrDB retrieves VTXOs from cache first, falling back to DB for cache misses. -// This is used in conjunction with prefetchVtxosByMarkers to reduce DB calls. -func (i *indexerService) getVtxosFromCacheOrDB( +// ensureVtxosCached loads the given outpoints into the cache if not already present. +// For each fetched VTXO, it also loads its marker window into the cache to prefetch +// nearby VTXOs that will likely be needed in subsequent iterations. +func (i *indexerService) ensureVtxosCached( ctx context.Context, outpoints []domain.Outpoint, cache map[string]domain.Vtxo, -) ([]domain.Vtxo, error) { - result := make([]domain.Vtxo, 0, len(outpoints)) + loadedMarkers map[string]bool, +) error { + // Collect cache misses. missingOutpoints := make([]domain.Outpoint, 0) - for _, op := range outpoints { - if v, ok := cache[op.String()]; ok { - result = append(result, v) - } else { + if _, ok := cache[op.String()]; !ok { missingOutpoints = append(missingOutpoints, op) } } + if len(missingOutpoints) == 0 { + return nil + } - if len(missingOutpoints) > 0 { - dbVtxos, err := i.repoManager.Vtxos().GetVtxos(ctx, missingOutpoints) - if err != nil { - return nil, err - } - result = append(result, dbVtxos...) - // Add to cache for future lookups in this chain traversal - for _, v := range dbVtxos { - cache[v.Outpoint.String()] = v + // Fetch misses from DB. + dbVtxos, err := i.repoManager.Vtxos().GetVtxos(ctx, missingOutpoints) + if err != nil { + return err + } + for _, v := range dbVtxos { + cache[v.Outpoint.String()] = v + } + + // For each fetched VTXO, load its marker window(s) into cache. + if i.repoManager.Markers() == nil { + return nil + } + for _, v := range dbVtxos { + for _, markerID := range v.MarkerIDs { + if loadedMarkers[markerID] { + continue + } + loadedMarkers[markerID] = true + + windowVtxos, err := i.repoManager.Markers().GetVtxosByMarker(ctx, markerID) + if err != nil { + continue + } + for _, wv := range windowVtxos { + if _, ok := cache[wv.Outpoint.String()]; !ok { + cache[wv.Outpoint.String()] = wv + } + } } } - return result, nil + return nil } func (i *indexerService) GetVirtualTxs( diff --git a/internal/core/application/indexer_test.go b/internal/core/application/indexer_test.go index a633b4b4d..778e4dc22 100644 --- a/internal/core/application/indexer_test.go +++ b/internal/core/application/indexer_test.go @@ -244,7 +244,11 @@ func (m *mockMarkerRepoForIndexer) GetVtxosByMarker( ctx context.Context, markerID string, ) ([]domain.Vtxo, error) { - return nil, nil + args := m.Called(ctx, markerID) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).([]domain.Vtxo), args.Error(1) } func (m *mockMarkerRepoForIndexer) SweepVtxosByMarker( @@ -312,670 +316,188 @@ func newTestIndexer() ( return vtxoRepo, markerRepo, indexer } -// TestPrefetchVtxosByMarkers_BuildsCacheFromMarkerChain verifies that prefetchVtxosByMarkers -// correctly traverses the marker hierarchy (following ParentMarkerIDs) and bulk fetches -// all VTXOs associated with those markers into a cache map. -func TestPrefetchVtxosByMarkers_BuildsCacheFromMarkerChain(t *testing.T) { - vtxoRepo, markerRepo, indexer := newTestIndexer() - - ctx := context.Background() - startKey := Outpoint{Txid: "start-vtxo", VOut: 0} - - // Starting VTXO with markers at depth 200 - startVtxo := domain.Vtxo{ - Outpoint: domain.Outpoint{Txid: "start-vtxo", VOut: 0}, - MarkerIDs: []string{"marker-200"}, - Depth: 200, - } - - // Marker chain: marker-200 -> marker-100 -> marker-0 (root) - marker200 := &domain.Marker{ - ID: "marker-200", - Depth: 200, - ParentMarkerIDs: []string{"marker-100"}, - } - marker100 := &domain.Marker{ID: "marker-100", Depth: 100, ParentMarkerIDs: []string{"marker-0"}} - marker0 := &domain.Marker{ID: "marker-0", Depth: 0, ParentMarkerIDs: []string{}} - - // VTXOs associated with all markers in the chain - chainVtxos := []domain.Vtxo{ - {Outpoint: domain.Outpoint{Txid: "vtxo-a", VOut: 0}, Depth: 50}, - {Outpoint: domain.Outpoint{Txid: "vtxo-b", VOut: 0}, Depth: 100}, - {Outpoint: domain.Outpoint{Txid: "vtxo-c", VOut: 0}, Depth: 150}, - {Outpoint: domain.Outpoint{Txid: "vtxo-d", VOut: 0}, Depth: 200}, - } - - // Setup expectations - vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{startKey}). - Return([]domain.Vtxo{startVtxo}, nil) - - markerRepo.On("GetMarker", ctx, "marker-200").Return(marker200, nil) - markerRepo.On("GetMarker", ctx, "marker-100").Return(marker100, nil) - markerRepo.On("GetMarker", ctx, "marker-0").Return(marker0, nil) - - // Expect bulk fetch with all markers in the chain - markerRepo.On("GetVtxoChainByMarkers", ctx, mock.MatchedBy(func(ids []string) bool { - // Should contain marker-200, marker-100, marker-0 - idSet := make(map[string]bool) - for _, id := range ids { - idSet[id] = true - } - return len(ids) == 3 && idSet["marker-200"] && idSet["marker-100"] && idSet["marker-0"] - })).Return(chainVtxos, nil) - - // Execute - cache := indexer.prefetchVtxosByMarkers(ctx, startKey) - - // Verify cache contains all VTXOs plus the start VTXO - require.Len(t, cache, 5) // 4 chain VTXOs + 1 start VTXO - require.Contains(t, cache, "start-vtxo:0") - require.Contains(t, cache, "vtxo-a:0") - require.Contains(t, cache, "vtxo-b:0") - require.Contains(t, cache, "vtxo-c:0") - require.Contains(t, cache, "vtxo-d:0") -} - -// TestPrefetchVtxosByMarkers_EmptyMarkersReturnsStartVtxoOnly verifies that when the -// starting VTXO has no markers, the cache only contains the starting VTXO itself. -func TestPrefetchVtxosByMarkers_EmptyMarkersReturnsStartVtxoOnly(t *testing.T) { - vtxoRepo, markerRepo, indexer := newTestIndexer() - - ctx := context.Background() - startKey := Outpoint{Txid: "vtxo-no-markers", VOut: 0} - - // VTXO with no markers - startVtxo := domain.Vtxo{ - Outpoint: domain.Outpoint{Txid: "vtxo-no-markers", VOut: 0}, - MarkerIDs: []string{}, // Empty markers - Depth: 0, +// TestEncodeDecodeChainCursor_RoundTrip verifies that encoding then decoding +// a frontier of outpoints returns the same outpoints. +func TestEncodeDecodeChainCursor_RoundTrip(t *testing.T) { + frontier := []domain.Outpoint{ + {Txid: "abc123", VOut: 0}, + {Txid: "def456", VOut: 2}, + {Txid: "ghi789", VOut: 1}, } - vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{startKey}). - Return([]domain.Vtxo{startVtxo}, nil) + token := encodeChainCursor(frontier) + require.NotEmpty(t, token) - cache := indexer.prefetchVtxosByMarkers(ctx, startKey) - - // Cache should only contain the start VTXO - require.Len(t, cache, 1) - require.Contains(t, cache, "vtxo-no-markers:0") - - // Marker repo should not be called for chain traversal - markerRepo.AssertNotCalled(t, "GetMarker", mock.Anything, mock.Anything) - markerRepo.AssertNotCalled(t, "GetVtxoChainByMarkers", mock.Anything, mock.Anything) + decoded, err := decodeChainCursor(token) + require.NoError(t, err) + require.Equal(t, frontier, decoded) } -// TestPrefetchVtxosByMarkers_NilMarkerRepoReturnsEmptyCache verifies that when the -// marker repository is nil (not configured), an empty cache is returned gracefully. -func TestPrefetchVtxosByMarkers_NilMarkerRepoReturnsEmptyCache(t *testing.T) { - vtxoRepo := &mockVtxoRepoForIndexer{} - repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: nil} - - indexer := &indexerService{repoManager: repoManager} - - ctx := context.Background() - startKey := Outpoint{Txid: "vtxo-any", VOut: 0} +// TestEncodeDecodeChainCursor_EmptyFrontier verifies that an empty frontier +// encodes to an empty string. +func TestEncodeDecodeChainCursor_EmptyFrontier(t *testing.T) { + token := encodeChainCursor(nil) + require.Empty(t, token) - cache := indexer.prefetchVtxosByMarkers(ctx, startKey) + token = encodeChainCursor([]domain.Outpoint{}) + require.Empty(t, token) +} - // Cache should be empty when marker repo is nil - require.Empty(t, cache) +// TestDecodeChainCursor_InvalidBase64 verifies that invalid base64 returns an error. +func TestDecodeChainCursor_InvalidBase64(t *testing.T) { + _, err := decodeChainCursor("not-valid-base64!!!") + require.Error(t, err) + require.Contains(t, err.Error(), "invalid base64") +} - // VTXO repo should not be called - vtxoRepo.AssertNotCalled(t, "GetVtxos", mock.Anything, mock.Anything) +// TestDecodeChainCursor_InvalidJSON verifies that valid base64 but invalid JSON +// returns an error. +func TestDecodeChainCursor_InvalidJSON(t *testing.T) { + // Encode something that is not valid JSON + token := "bm90LWpzb24" // base64url of "not-json" + _, err := decodeChainCursor(token) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid JSON") } -// TestGetVtxosFromCacheOrDB_CacheHitAvoidsDBCall verifies that when all requested -// outpoints are in the cache, no database call is made. -func TestGetVtxosFromCacheOrDB_CacheHitAvoidsDBCall(t *testing.T) { +// TestEnsureVtxosCached_AllCacheHits verifies that when all outpoints are already +// in the cache, no DB call is made. +func TestEnsureVtxosCached_AllCacheHits(t *testing.T) { vtxoRepo, _, indexer := newTestIndexer() ctx := context.Background() - - // Pre-populated cache cache := map[string]domain.Vtxo{ - "cached-vtxo-1:0": { - Outpoint: domain.Outpoint{Txid: "cached-vtxo-1", VOut: 0}, - Amount: 1000, - }, - "cached-vtxo-2:0": { - Outpoint: domain.Outpoint{Txid: "cached-vtxo-2", VOut: 0}, - Amount: 2000, - }, + "vtxo-1:0": {Outpoint: domain.Outpoint{Txid: "vtxo-1", VOut: 0}, Amount: 100}, + "vtxo-2:0": {Outpoint: domain.Outpoint{Txid: "vtxo-2", VOut: 0}, Amount: 200}, } + loadedMarkers := make(map[string]bool) outpoints := []domain.Outpoint{ - {Txid: "cached-vtxo-1", VOut: 0}, - {Txid: "cached-vtxo-2", VOut: 0}, + {Txid: "vtxo-1", VOut: 0}, + {Txid: "vtxo-2", VOut: 0}, } - result, err := indexer.getVtxosFromCacheOrDB(ctx, outpoints, cache) - + err := indexer.ensureVtxosCached(ctx, outpoints, cache, loadedMarkers) require.NoError(t, err) - require.Len(t, result, 2) - // Verify GetVtxos was never called (all cache hits) + // No DB call should be made vtxoRepo.AssertNotCalled(t, "GetVtxos", mock.Anything, mock.Anything) } -// TestGetVtxosFromCacheOrDB_CacheMissTriggersDBCall verifies that when outpoints -// are not in the cache, a database call is made for the missing ones only. -func TestGetVtxosFromCacheOrDB_CacheMissTriggersDBCall(t *testing.T) { - vtxoRepo, _, indexer := newTestIndexer() +// TestEnsureVtxosCached_CacheMissLoadsFromDBAndMarkerWindow verifies that cache +// misses trigger a DB lookup and marker window prefetch. +func TestEnsureVtxosCached_CacheMissLoadsFromDBAndMarkerWindow(t *testing.T) { + vtxoRepo, markerRepo, indexer := newTestIndexer() ctx := context.Background() + cache := make(map[string]domain.Vtxo) + loadedMarkers := make(map[string]bool) - // Cache with one VTXO - cache := map[string]domain.Vtxo{ - "cached-vtxo:0": {Outpoint: domain.Outpoint{Txid: "cached-vtxo", VOut: 0}, Amount: 1000}, - } + outpoints := []domain.Outpoint{{Txid: "vtxo-miss", VOut: 0}} - // Request both cached and uncached - outpoints := []domain.Outpoint{ - {Txid: "cached-vtxo", VOut: 0}, - {Txid: "uncached-vtxo", VOut: 0}, + // DB returns VTXO with a marker + dbVtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "vtxo-miss", VOut: 0}, + Amount: 500, + MarkerIDs: []string{"marker-100"}, } + vtxoRepo.On("GetVtxos", ctx, outpoints).Return([]domain.Vtxo{dbVtxo}, nil) - // DB should be called only for uncached outpoint - dbVtxo := domain.Vtxo{Outpoint: domain.Outpoint{Txid: "uncached-vtxo", VOut: 0}, Amount: 3000} - vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: "uncached-vtxo", VOut: 0}}). - Return([]domain.Vtxo{dbVtxo}, nil) - - result, err := indexer.getVtxosFromCacheOrDB(ctx, outpoints, cache) + // Marker window returns additional VTXOs + windowVtxos := []domain.Vtxo{ + {Outpoint: domain.Outpoint{Txid: "window-vtxo-1", VOut: 0}, Amount: 300}, + {Outpoint: domain.Outpoint{Txid: "window-vtxo-2", VOut: 0}, Amount: 400}, + } + markerRepo.On("GetVtxosByMarker", ctx, "marker-100").Return(windowVtxos, nil) + err := indexer.ensureVtxosCached(ctx, outpoints, cache, loadedMarkers) require.NoError(t, err) - require.Len(t, result, 2) - // Verify the uncached VTXO was added to cache - require.Contains(t, cache, "uncached-vtxo:0") - require.Equal(t, uint64(3000), cache["uncached-vtxo:0"].Amount) + // Cache should contain the original VTXO plus window VTXOs + require.Contains(t, cache, "vtxo-miss:0") + require.Contains(t, cache, "window-vtxo-1:0") + require.Contains(t, cache, "window-vtxo-2:0") - vtxoRepo.AssertExpectations(t) + // Marker should be marked as loaded + require.True(t, loadedMarkers["marker-100"]) } -// TestGetVtxosFromCacheOrDB_AllCacheMiss verifies behavior when cache is empty -// and all outpoints must be fetched from the database. -func TestGetVtxosFromCacheOrDB_AllCacheMiss(t *testing.T) { - vtxoRepo, _, indexer := newTestIndexer() +// TestEnsureVtxosCached_NilMarkerRepo verifies that when the marker repository +// is nil, ensureVtxosCached falls back to direct DB lookup without window prefetch. +func TestEnsureVtxosCached_NilMarkerRepo(t *testing.T) { + vtxoRepo := &mockVtxoRepoForIndexer{} + repoManager := &mockRepoManagerForIndexer{vtxos: vtxoRepo, markers: nil} + indexer := &indexerService{repoManager: repoManager} ctx := context.Background() - - // Empty cache cache := make(map[string]domain.Vtxo) + loadedMarkers := make(map[string]bool) - outpoints := []domain.Outpoint{ - {Txid: "vtxo-1", VOut: 0}, - {Txid: "vtxo-2", VOut: 0}, - {Txid: "vtxo-3", VOut: 0}, - } - - dbVtxos := []domain.Vtxo{ - {Outpoint: domain.Outpoint{Txid: "vtxo-1", VOut: 0}, Amount: 100}, - {Outpoint: domain.Outpoint{Txid: "vtxo-2", VOut: 0}, Amount: 200}, - {Outpoint: domain.Outpoint{Txid: "vtxo-3", VOut: 0}, Amount: 300}, + outpoints := []domain.Outpoint{{Txid: "vtxo-no-markers", VOut: 0}} + dbVtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "vtxo-no-markers", VOut: 0}, + Amount: 100, + MarkerIDs: []string{"marker-X"}, } + vtxoRepo.On("GetVtxos", ctx, outpoints).Return([]domain.Vtxo{dbVtxo}, nil) - vtxoRepo.On("GetVtxos", ctx, outpoints).Return(dbVtxos, nil) - - result, err := indexer.getVtxosFromCacheOrDB(ctx, outpoints, cache) - + err := indexer.ensureVtxosCached(ctx, outpoints, cache, loadedMarkers) require.NoError(t, err) - require.Len(t, result, 3) - // All VTXOs should now be in cache - require.Len(t, cache, 3) - require.Contains(t, cache, "vtxo-1:0") - require.Contains(t, cache, "vtxo-2:0") - require.Contains(t, cache, "vtxo-3:0") + // VTXO should be cached even without marker window loading + require.Contains(t, cache, "vtxo-no-markers:0") } -// TestGetVtxosFromCacheOrDB_DBErrorPropagated verifies that database errors -// are properly propagated to the caller. -func TestGetVtxosFromCacheOrDB_DBErrorPropagated(t *testing.T) { +// TestEnsureVtxosCached_DBErrorPropagated verifies that database errors +// are properly propagated. +func TestEnsureVtxosCached_DBErrorPropagated(t *testing.T) { vtxoRepo, _, indexer := newTestIndexer() ctx := context.Background() cache := make(map[string]domain.Vtxo) + loadedMarkers := make(map[string]bool) outpoints := []domain.Outpoint{{Txid: "vtxo-err", VOut: 0}} - vtxoRepo.On("GetVtxos", ctx, outpoints). - Return(nil, fmt.Errorf("vtxo not found")) - - result, err := indexer.getVtxosFromCacheOrDB(ctx, outpoints, cache) + Return(nil, fmt.Errorf("database error")) + err := indexer.ensureVtxosCached(ctx, outpoints, cache, loadedMarkers) require.Error(t, err) - require.Nil(t, result) -} - -// TestPrefetchVtxosByMarkers_HandlesMultipleParentMarkers verifies that the BFS -// traversal correctly handles VTXOs with multiple parent markers (diamond pattern -// in the marker DAG). -func TestPrefetchVtxosByMarkers_HandlesMultipleParentMarkers(t *testing.T) { - vtxoRepo, markerRepo, indexer := newTestIndexer() - - ctx := context.Background() - startKey := Outpoint{Txid: "diamond-vtxo", VOut: 0} - - // VTXO with multiple markers (diamond pattern) - // marker-C has two parents: marker-A and marker-B, both pointing to marker-root - startVtxo := domain.Vtxo{ - Outpoint: domain.Outpoint{Txid: "diamond-vtxo", VOut: 0}, - MarkerIDs: []string{"marker-C"}, - Depth: 200, - } - - markerC := &domain.Marker{ - ID: "marker-C", - Depth: 200, - ParentMarkerIDs: []string{"marker-A", "marker-B"}, - } - markerA := &domain.Marker{ID: "marker-A", Depth: 100, ParentMarkerIDs: []string{"marker-root"}} - markerB := &domain.Marker{ID: "marker-B", Depth: 100, ParentMarkerIDs: []string{"marker-root"}} - markerRoot := &domain.Marker{ID: "marker-root", Depth: 0, ParentMarkerIDs: []string{}} - - vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{startKey}). - Return([]domain.Vtxo{startVtxo}, nil) - - markerRepo.On("GetMarker", ctx, "marker-C").Return(markerC, nil) - markerRepo.On("GetMarker", ctx, "marker-A").Return(markerA, nil) - markerRepo.On("GetMarker", ctx, "marker-B").Return(markerB, nil) - markerRepo.On("GetMarker", ctx, "marker-root").Return(markerRoot, nil) - - chainVtxos := []domain.Vtxo{ - {Outpoint: domain.Outpoint{Txid: "vtxo-from-chain", VOut: 0}}, - } - - // Should collect all 4 markers despite diamond pattern - markerRepo.On("GetVtxoChainByMarkers", ctx, mock.MatchedBy(func(ids []string) bool { - idSet := make(map[string]bool) - for _, id := range ids { - idSet[id] = true - } - // Must have all 4 markers, no duplicates - return len(ids) == 4 && - idSet["marker-C"] && - idSet["marker-A"] && - idSet["marker-B"] && - idSet["marker-root"] - })).Return(chainVtxos, nil) - - cache := indexer.prefetchVtxosByMarkers(ctx, startKey) - - // Verify we got the starting VTXO plus chain VTXOs - require.Contains(t, cache, "diamond-vtxo:0") - require.Contains(t, cache, "vtxo-from-chain:0") -} - -// TestPrefetchVtxosByMarkers_GetVtxosError verifies that when GetVtxos fails -// to retrieve the starting VTXO, prefetchVtxosByMarkers returns an empty cache -// gracefully without panicking. -func TestPrefetchVtxosByMarkers_GetVtxosError(t *testing.T) { - vtxoRepo, markerRepo, indexer := newTestIndexer() - - ctx := context.Background() - startKey := Outpoint{Txid: "vtxo-error", VOut: 0} - - // Simulate DB error when fetching start VTXO - vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{startKey}). - Return(nil, fmt.Errorf("database connection lost")) - - cache := indexer.prefetchVtxosByMarkers(ctx, startKey) - - // Cache should be empty on error - require.Empty(t, cache) - - // Marker repo should not be called - markerRepo.AssertNotCalled(t, "GetMarker", mock.Anything, mock.Anything) - markerRepo.AssertNotCalled(t, "GetVtxoChainByMarkers", mock.Anything, mock.Anything) -} - -// TestPrefetchVtxosByMarkers_GetMarkerError verifies that when GetMarker -// fails for a marker in the BFS traversal, the function still returns -// partial results from successfully fetched markers. -func TestPrefetchVtxosByMarkers_GetMarkerError(t *testing.T) { - vtxoRepo, markerRepo, indexer := newTestIndexer() - - ctx := context.Background() - startKey := Outpoint{Txid: "vtxo-partial", VOut: 0} - - // Starting VTXO with a marker at depth 200 - startVtxo := domain.Vtxo{ - Outpoint: domain.Outpoint{Txid: "vtxo-partial", VOut: 0}, - MarkerIDs: []string{"marker-200"}, - Depth: 200, - } - - // marker-200 has parent marker-100, but marker-100 lookup will fail - marker200 := &domain.Marker{ - ID: "marker-200", - Depth: 200, - ParentMarkerIDs: []string{"marker-100"}, - } - - vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{startKey}). - Return([]domain.Vtxo{startVtxo}, nil) - - markerRepo.On("GetMarker", ctx, "marker-200").Return(marker200, nil) - // marker-100 lookup fails - markerRepo.On("GetMarker", ctx, "marker-100"). - Return(nil, fmt.Errorf("marker not found")) - - // GetVtxoChainByMarkers should still be called with the markers we did collect - chainVtxos := []domain.Vtxo{ - {Outpoint: domain.Outpoint{Txid: "vtxo-from-200", VOut: 0}, Depth: 180}, - } - markerRepo.On("GetVtxoChainByMarkers", ctx, mock.MatchedBy(func(ids []string) bool { - // Should have marker-200 and marker-100 (both were added to markerIDs) - idSet := make(map[string]bool) - for _, id := range ids { - idSet[id] = true - } - return len(ids) == 2 && idSet["marker-200"] && idSet["marker-100"] - })).Return(chainVtxos, nil) - - cache := indexer.prefetchVtxosByMarkers(ctx, startKey) - - // Should have the start VTXO plus chain VTXOs from marker-200 - require.Contains(t, cache, "vtxo-partial:0") - require.Contains(t, cache, "vtxo-from-200:0") -} - -// TestPrefetchVtxosByMarkers_GetVtxoChainByMarkersError verifies that when -// the bulk fetch of VTXOs by markers fails, the cache still contains at -// least the starting VTXO. -func TestPrefetchVtxosByMarkers_GetVtxoChainByMarkersError(t *testing.T) { - vtxoRepo, markerRepo, indexer := newTestIndexer() - - ctx := context.Background() - startKey := Outpoint{Txid: "vtxo-bulk-err", VOut: 0} - - startVtxo := domain.Vtxo{ - Outpoint: domain.Outpoint{Txid: "vtxo-bulk-err", VOut: 0}, - MarkerIDs: []string{"marker-100"}, - Depth: 100, - } - - marker100 := &domain.Marker{ - ID: "marker-100", - Depth: 100, - ParentMarkerIDs: []string{}, - } - - vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{startKey}). - Return([]domain.Vtxo{startVtxo}, nil) - - markerRepo.On("GetMarker", ctx, "marker-100").Return(marker100, nil) - - // Bulk fetch fails - markerRepo.On("GetVtxoChainByMarkers", ctx, mock.Anything). - Return(nil, fmt.Errorf("bulk fetch failed")) - - cache := indexer.prefetchVtxosByMarkers(ctx, startKey) - - // Cache should still contain the start VTXO - require.Len(t, cache, 1) - require.Contains(t, cache, "vtxo-bulk-err:0") + require.Contains(t, err.Error(), "database error") } -// TestPrefetchVtxosByMarkers_DeepChainManyMarkers verifies that BFS traversal -// correctly handles a deep chain with 5+ markers (depth 500), collecting all -// markers without off-by-one errors or missed parents. -func TestPrefetchVtxosByMarkers_DeepChainManyMarkers(t *testing.T) { - vtxoRepo, markerRepo, indexer := newTestIndexer() +// TestGetVtxoChain_InvalidPageToken verifies that an invalid page_token +// returns an error. +func TestGetVtxoChain_InvalidPageToken(t *testing.T) { + _, _, indexer := newTestIndexer() ctx := context.Background() - startKey := Outpoint{Txid: "deep-vtxo", VOut: 0} - - // VTXO at depth 500 with marker at depth 500 - startVtxo := domain.Vtxo{ - Outpoint: domain.Outpoint{Txid: "deep-vtxo", VOut: 0}, - MarkerIDs: []string{"marker-500"}, - Depth: 500, - } + vtxoKey := Outpoint{Txid: "abc123", VOut: 0} - // Linear marker chain: 500 -> 400 -> 300 -> 200 -> 100 -> 0 - marker500 := &domain.Marker{ - ID: "marker-500", - Depth: 500, - ParentMarkerIDs: []string{"marker-400"}, - } - marker400 := &domain.Marker{ - ID: "marker-400", - Depth: 400, - ParentMarkerIDs: []string{"marker-300"}, - } - marker300 := &domain.Marker{ - ID: "marker-300", - Depth: 300, - ParentMarkerIDs: []string{"marker-200"}, - } - marker200 := &domain.Marker{ - ID: "marker-200", - Depth: 200, - ParentMarkerIDs: []string{"marker-100"}, - } - marker100 := &domain.Marker{ID: "marker-100", Depth: 100, ParentMarkerIDs: []string{"marker-0"}} - marker0 := &domain.Marker{ID: "marker-0", Depth: 0, ParentMarkerIDs: []string{}} - - vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{startKey}). - Return([]domain.Vtxo{startVtxo}, nil) - - markerRepo.On("GetMarker", ctx, "marker-500").Return(marker500, nil) - markerRepo.On("GetMarker", ctx, "marker-400").Return(marker400, nil) - markerRepo.On("GetMarker", ctx, "marker-300").Return(marker300, nil) - markerRepo.On("GetMarker", ctx, "marker-200").Return(marker200, nil) - markerRepo.On("GetMarker", ctx, "marker-100").Return(marker100, nil) - markerRepo.On("GetMarker", ctx, "marker-0").Return(marker0, nil) - - // Chain VTXOs from the bulk fetch - chainVtxos := []domain.Vtxo{ - {Outpoint: domain.Outpoint{Txid: "v-450", VOut: 0}, Depth: 450}, - {Outpoint: domain.Outpoint{Txid: "v-350", VOut: 0}, Depth: 350}, - {Outpoint: domain.Outpoint{Txid: "v-250", VOut: 0}, Depth: 250}, - {Outpoint: domain.Outpoint{Txid: "v-150", VOut: 0}, Depth: 150}, - {Outpoint: domain.Outpoint{Txid: "v-050", VOut: 0}, Depth: 50}, - {Outpoint: domain.Outpoint{Txid: "v-000", VOut: 0}, Depth: 0}, - } - - // All 6 markers should be collected - markerRepo.On("GetVtxoChainByMarkers", ctx, mock.MatchedBy(func(ids []string) bool { - if len(ids) != 6 { - return false - } - idSet := make(map[string]bool) - for _, id := range ids { - idSet[id] = true - } - return idSet["marker-500"] && idSet["marker-400"] && idSet["marker-300"] && - idSet["marker-200"] && idSet["marker-100"] && idSet["marker-0"] - })).Return(chainVtxos, nil) - - cache := indexer.prefetchVtxosByMarkers(ctx, startKey) - - // 6 chain VTXOs + 1 start VTXO = 7 - require.Len(t, cache, 7) - require.Contains(t, cache, "deep-vtxo:0") - require.Contains(t, cache, "v-450:0") - require.Contains(t, cache, "v-350:0") - require.Contains(t, cache, "v-250:0") - require.Contains(t, cache, "v-150:0") - require.Contains(t, cache, "v-050:0") - require.Contains(t, cache, "v-000:0") - - // Verify every marker in the chain was visited - markerRepo.AssertCalled(t, "GetMarker", ctx, "marker-500") - markerRepo.AssertCalled(t, "GetMarker", ctx, "marker-400") - markerRepo.AssertCalled(t, "GetMarker", ctx, "marker-300") - markerRepo.AssertCalled(t, "GetMarker", ctx, "marker-200") - markerRepo.AssertCalled(t, "GetMarker", ctx, "marker-100") - markerRepo.AssertCalled(t, "GetMarker", ctx, "marker-0") -} - -// TestGetVtxosFromCacheOrDB_EmptyOutpoints verifies that an empty outpoints -// list returns an empty result without making any database call. -func TestGetVtxosFromCacheOrDB_EmptyOutpoints(t *testing.T) { - vtxoRepo, _, indexer := newTestIndexer() - - ctx := context.Background() - cache := map[string]domain.Vtxo{ - "existing:0": {Outpoint: domain.Outpoint{Txid: "existing", VOut: 0}}, - } - - result, err := indexer.getVtxosFromCacheOrDB(ctx, []domain.Outpoint{}, cache) - - require.NoError(t, err) - require.Empty(t, result) - - // DB should never be called for empty input - vtxoRepo.AssertNotCalled(t, "GetVtxos", mock.Anything, mock.Anything) + _, err := indexer.GetVtxoChain(ctx, vtxoKey, nil, "invalid-token!!!") + require.Error(t, err) + require.Contains(t, err.Error(), "invalid page_token") } -// TestPrefetchVtxosByMarkers_CycleInMarkerDAG verifies that the BFS in -// prefetchVtxosByMarkers terminates when there is a cycle in the marker DAG -// (marker-A → parent marker-B → parent marker-A). -func TestPrefetchVtxosByMarkers_CycleInMarkerDAG(t *testing.T) { +// TestGetVtxoChain_BackwardCompat_NilPageEmptyToken verifies that when +// page is nil and pageToken is empty, the VTXO not found error comes from +// the DB lookup (not from pagination parsing), confirming backward compat. +func TestGetVtxoChain_BackwardCompat_NilPageEmptyToken(t *testing.T) { vtxoRepo, markerRepo, indexer := newTestIndexer() ctx := context.Background() - startKey := Outpoint{Txid: "cycle-vtxo", VOut: 0} - - // Starting VTXO references marker-A - startVtxo := domain.Vtxo{ - Outpoint: domain.Outpoint{Txid: "cycle-vtxo", VOut: 0}, - MarkerIDs: []string{"marker-A"}, - Depth: 200, - } - vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: "cycle-vtxo", VOut: 0}}). - Return([]domain.Vtxo{startVtxo}, nil) - - // marker-A points to marker-B as parent - markerRepo.On("GetMarker", ctx, "marker-A").Return(&domain.Marker{ - ID: "marker-A", - Depth: 200, - ParentMarkerIDs: []string{"marker-B"}, - }, nil) - - // marker-B points BACK to marker-A (cycle!) - markerRepo.On("GetMarker", ctx, "marker-B").Return(&domain.Marker{ - ID: "marker-B", - Depth: 100, - ParentMarkerIDs: []string{"marker-A"}, - }, nil) - - // Both markers should be collected despite the cycle - markerRepo.On("GetVtxoChainByMarkers", ctx, mock.MatchedBy(func(ids []string) bool { - if len(ids) != 2 { - return false - } - idSet := make(map[string]bool) - for _, id := range ids { - idSet[id] = true - } - return idSet["marker-A"] && idSet["marker-B"] - })).Return([]domain.Vtxo{ - {Outpoint: domain.Outpoint{Txid: "chain-vtxo-1", VOut: 0}, Depth: 150}, - }, nil) - - cache := indexer.prefetchVtxosByMarkers(ctx, startKey) - - // Should terminate and contain the start VTXO + chain VTXO - require.Len(t, cache, 2) - require.Contains(t, cache, "cycle-vtxo:0") - require.Contains(t, cache, "chain-vtxo-1:0") - - // Each marker should be visited exactly once - markerRepo.AssertNumberOfCalls(t, "GetMarker", 2) -} - -// TestPrefetchVtxosByMarkers_StartVtxoNotFound verifies that when the starting -// VTXO is not found in the database, an empty cache is returned. -func TestPrefetchVtxosByMarkers_StartVtxoNotFound(t *testing.T) { - vtxoRepo, markerRepo, indexer := newTestIndexer() + vtxoKey := Outpoint{Txid: "root-vtxo", VOut: 0} - ctx := context.Background() - startKey := Outpoint{Txid: "nonexistent", VOut: 0} - - // GetVtxos returns empty slice (VTXO not found) - vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: "nonexistent", VOut: 0}}). + // Return no VTXOs so the chain walk fails with "vtxo not found" + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{vtxoKey}). Return([]domain.Vtxo{}, nil) + markerRepo.On("GetVtxosByMarker", ctx, mock.Anything). + Return([]domain.Vtxo{}, nil).Maybe() - cache := indexer.prefetchVtxosByMarkers(ctx, startKey) - - require.Empty(t, cache) - // Marker repo should never be touched - markerRepo.AssertNotCalled(t, "GetMarker", mock.Anything, mock.Anything) - markerRepo.AssertNotCalled(t, "GetVtxoChainByMarkers", mock.Anything, mock.Anything) -} - -// TestPrefetchVtxosByMarkers_Depth20k verifies that the BFS traversal in -// prefetchVtxosByMarkers correctly handles a VTXO at depth 20000 with a chain -// of 200 markers (one every 100 depths). This is the target maximum depth. -func TestPrefetchVtxosByMarkers_Depth20k(t *testing.T) { - vtxoRepo, markerRepo, indexer := newTestIndexer() - - ctx := context.Background() - startKey := Outpoint{Txid: "deep-20k-vtxo", VOut: 0} - - const maxDepth = 20000 - const markerInterval = 100 - const numMarkers = maxDepth / markerInterval // 200 markers + _, err := indexer.GetVtxoChain(ctx, vtxoKey, nil, "") - // Starting VTXO at depth 20000 with marker at depth 20000 - startVtxo := domain.Vtxo{ - Outpoint: domain.Outpoint{Txid: "deep-20k-vtxo", VOut: 0}, - MarkerIDs: []string{fmt.Sprintf("marker-%d", maxDepth)}, - Depth: maxDepth, - } - vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: "deep-20k-vtxo", VOut: 0}}). - Return([]domain.Vtxo{startVtxo}, nil) - - // Build the 200-marker chain: marker-20000 -> marker-19900 -> ... -> marker-100 -> marker-0 - for depth := uint32(maxDepth); depth > 0; depth -= markerInterval { - parentDepth := depth - markerInterval - markerID := fmt.Sprintf("marker-%d", depth) - parentMarkerID := fmt.Sprintf("marker-%d", parentDepth) - markerRepo.On("GetMarker", ctx, markerID).Return(&domain.Marker{ - ID: markerID, - Depth: depth, - ParentMarkerIDs: []string{parentMarkerID}, - }, nil) - } - // Root marker at depth 0 has no parents - markerRepo.On("GetMarker", ctx, "marker-0").Return(&domain.Marker{ - ID: "marker-0", - Depth: 0, - ParentMarkerIDs: []string{}, - }, nil) - - // Generate VTXOs that would be returned by GetVtxoChainByMarkers - // One VTXO per marker interval midpoint to simulate a populated chain - chainVtxos := make([]domain.Vtxo, 0, numMarkers) - for i := 0; i < numMarkers; i++ { - chainVtxos = append(chainVtxos, domain.Vtxo{ - Outpoint: domain.Outpoint{ - Txid: fmt.Sprintf("chain-vtxo-%d", i), - VOut: 0, - }, - Depth: uint32(i*markerInterval + 50), // midpoint of each interval - }) - } - - // All 201 markers (0, 100, 200, ..., 20000) should be collected - markerRepo.On("GetVtxoChainByMarkers", ctx, mock.MatchedBy(func(ids []string) bool { - return len(ids) == numMarkers+1 // 201 markers total - })).Return(chainVtxos, nil) - - cache := indexer.prefetchVtxosByMarkers(ctx, startKey) - - // 200 chain VTXOs + 1 start VTXO = 201 - require.Len(t, cache, numMarkers+1) - require.Contains(t, cache, "deep-20k-vtxo:0") - - // Verify a sample of chain VTXOs are in cache - require.Contains(t, cache, "chain-vtxo-0:0") - require.Contains(t, cache, "chain-vtxo-99:0") - require.Contains(t, cache, "chain-vtxo-199:0") - - // All 201 markers should have been visited via GetMarker (200 non-root + 1 root) - markerRepo.AssertNumberOfCalls(t, "GetMarker", numMarkers+1) + // Error should be from the chain walk, not from pagination setup + require.Error(t, err) + require.Contains(t, err.Error(), "vtxo not found") + require.NotContains(t, err.Error(), "invalid page_token") } diff --git a/internal/core/application/types.go b/internal/core/application/types.go index c94b7cf11..e05d681ca 100644 --- a/internal/core/application/types.go +++ b/internal/core/application/types.go @@ -133,8 +133,13 @@ type TeleportAsset struct { } type VtxoChainResp struct { - Chain []ChainTx - Page PageResp + Chain []ChainTx + Page PageResp + NextPageToken string +} + +type vtxoChainCursor struct { + Frontier []Outpoint `json:"frontier"` } type VOut int diff --git a/internal/interface/grpc/handlers/indexer.go b/internal/interface/grpc/handlers/indexer.go index c89350f7c..ff0cc3ddf 100644 --- a/internal/interface/grpc/handlers/indexer.go +++ b/internal/interface/grpc/handlers/indexer.go @@ -327,7 +327,9 @@ func (e *indexerService) GetVtxoChain( return nil, status.Error(codes.InvalidArgument, err.Error()) } - resp, err := e.indexerSvc.GetVtxoChain(ctx, *outpoint, page) + pageToken := request.GetPageToken() + + resp, err := e.indexerSvc.GetVtxoChain(ctx, *outpoint, page, pageToken) if err != nil { return nil, status.Errorf(codes.Internal, "%s", err.Error()) } @@ -355,8 +357,9 @@ func (e *indexerService) GetVtxoChain( } return &arkv1.GetVtxoChainResponse{ - Chain: chain, - Page: protoPage(resp.Page), + Chain: chain, + Page: protoPage(resp.Page), + NextPageToken: resp.NextPageToken, }, nil } From 4dddd829d395bcd10861746cd6f135a471ba0880 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Tue, 17 Feb 2026 01:05:43 -0500 Subject: [PATCH 29/54] add end-to-end pagination tests and fix early termination bug --- internal/core/application/indexer.go | 4 +- internal/core/application/indexer_test.go | 371 +++++++++++++++++++++- 2 files changed, 365 insertions(+), 10 deletions(-) diff --git a/internal/core/application/indexer.go b/internal/core/application/indexer.go index 172d75445..50eef871b 100644 --- a/internal/core/application/indexer.go +++ b/internal/core/application/indexer.go @@ -309,9 +309,9 @@ func (i *indexerService) GetVtxoChain( if visited[key] { continue } - visited[key] = true // Early termination: save unprocessed VTXOs to frontier for next page. + // Check before marking visited so the current VTXO is included in the frontier. if len(chain) >= pageSize { remaining := make([]domain.Outpoint, 0) for _, v := range vtxos { @@ -327,6 +327,8 @@ func (i *indexerService) GetVtxoChain( }, nil } + visited[key] = true + // if the vtxo is preconfirmed, it means it has been created by an offchain tx // we need to add the virtual tx + the associated checkpoints txs // also, we have to populate the newNextVtxos with the checkpoints inputs diff --git a/internal/core/application/indexer_test.go b/internal/core/application/indexer_test.go index 778e4dc22..7dd539522 100644 --- a/internal/core/application/indexer_test.go +++ b/internal/core/application/indexer_test.go @@ -3,9 +3,13 @@ package application import ( "context" "fmt" + "strings" "testing" "github.com/arkade-os/arkd/internal/core/domain" + "github.com/btcsuite/btcd/btcutil/psbt" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/wire" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -280,28 +284,54 @@ func (m *mockMarkerRepoForIndexer) GetVtxosByArkTxid( } func (m *mockMarkerRepoForIndexer) Close() {} +type mockOffchainTxRepoForIndexer struct { + mock.Mock +} + +func (m *mockOffchainTxRepoForIndexer) GetOffchainTx( + ctx context.Context, txid string, +) (*domain.OffchainTx, error) { + args := m.Called(ctx, txid) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).(*domain.OffchainTx), args.Error(1) +} + +func (m *mockOffchainTxRepoForIndexer) AddOrUpdateOffchainTx( + ctx context.Context, offchainTx *domain.OffchainTx, +) error { + return nil +} + +func (m *mockOffchainTxRepoForIndexer) Close() {} + type mockRepoManagerForIndexer struct { - vtxos *mockVtxoRepoForIndexer - markers *mockMarkerRepoForIndexer + vtxos *mockVtxoRepoForIndexer + markers *mockMarkerRepoForIndexer + offchainTxs *mockOffchainTxRepoForIndexer } func (m *mockRepoManagerForIndexer) Events() domain.EventRepository { return nil } func (m *mockRepoManagerForIndexer) Rounds() domain.RoundRepository { return nil } func (m *mockRepoManagerForIndexer) Vtxos() domain.VtxoRepository { return m.vtxos } func (m *mockRepoManagerForIndexer) Markers() domain.MarkerRepository { - // Must explicitly return nil to avoid Go's nil interface issue - // where a nil concrete type wrapped in an interface != nil if m.markers == nil { return nil } return m.markers } func (m *mockRepoManagerForIndexer) ScheduledSession() domain.ScheduledSessionRepo { return nil } -func (m *mockRepoManagerForIndexer) OffchainTxs() domain.OffchainTxRepository { return nil } -func (m *mockRepoManagerForIndexer) Convictions() domain.ConvictionRepository { return nil } -func (m *mockRepoManagerForIndexer) Assets() domain.AssetRepository { return nil } -func (m *mockRepoManagerForIndexer) Fees() domain.FeeRepository { return nil } -func (m *mockRepoManagerForIndexer) Close() {} +func (m *mockRepoManagerForIndexer) OffchainTxs() domain.OffchainTxRepository { + if m.offchainTxs == nil { + return nil + } + return m.offchainTxs +} +func (m *mockRepoManagerForIndexer) Convictions() domain.ConvictionRepository { return nil } +func (m *mockRepoManagerForIndexer) Assets() domain.AssetRepository { return nil } +func (m *mockRepoManagerForIndexer) Fees() domain.FeeRepository { return nil } +func (m *mockRepoManagerForIndexer) Close() {} // newTestIndexer creates a fresh set of mock repos and an indexerService for testing. func newTestIndexer() ( @@ -316,6 +346,46 @@ func newTestIndexer() ( return vtxoRepo, markerRepo, indexer } +// newTestIndexerWithOffchain creates mock repos including offchain tx repo. +func newTestIndexerWithOffchain() ( + *mockVtxoRepoForIndexer, + *mockMarkerRepoForIndexer, + *mockOffchainTxRepoForIndexer, + *indexerService, +) { + vtxoRepo := &mockVtxoRepoForIndexer{} + markerRepo := &mockMarkerRepoForIndexer{} + offchainTxRepo := &mockOffchainTxRepoForIndexer{} + repoManager := &mockRepoManagerForIndexer{ + vtxos: vtxoRepo, markers: markerRepo, offchainTxs: offchainTxRepo, + } + indexer := &indexerService{repoManager: repoManager} + return vtxoRepo, markerRepo, offchainTxRepo, indexer +} + +// makeCheckpointPSBT creates a base64-encoded PSBT with a single input from +// the given previous outpoint. Used to build test checkpoint transactions. +func makeCheckpointPSBT(t *testing.T, inputTxid string, inputVout uint32) string { + t.Helper() + prevHash, err := chainhash.NewHashFromStr(inputTxid) + require.NoError(t, err) + + outPoint := wire.NewOutPoint(prevHash, inputVout) + output := wire.NewTxOut(1000, []byte{0x51}) // OP_TRUE + + p, err := psbt.New( + []*wire.OutPoint{outPoint}, + []*wire.TxOut{output}, + 2, 0, + []uint32{wire.MaxTxInSequenceNum}, + ) + require.NoError(t, err) + + b64, err := p.B64Encode() + require.NoError(t, err) + return b64 +} + // TestEncodeDecodeChainCursor_RoundTrip verifies that encoding then decoding // a frontier of outpoints returns the same outpoints. func TestEncodeDecodeChainCursor_RoundTrip(t *testing.T) { @@ -466,6 +536,118 @@ func TestEnsureVtxosCached_DBErrorPropagated(t *testing.T) { require.Contains(t, err.Error(), "database error") } +// TestEnsureVtxosCached_MarkerDedupAvoidsDuplicateLoad verifies that +// loadedMarkers prevents redundant GetVtxosByMarker calls when the same +// marker is encountered across multiple ensureVtxosCached invocations. +func TestEnsureVtxosCached_MarkerDedupAvoidsDuplicateLoad(t *testing.T) { + vtxoRepo, markerRepo, indexer := newTestIndexer() + + ctx := context.Background() + cache := make(map[string]domain.Vtxo) + loadedMarkers := make(map[string]bool) + + // First call: vtxo-1 has marker-A + vtxo1 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "vtxo-1", VOut: 0}, + Amount: 100, + MarkerIDs: []string{"marker-A"}, + } + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: "vtxo-1", VOut: 0}}). + Return([]domain.Vtxo{vtxo1}, nil) + markerRepo.On("GetVtxosByMarker", ctx, "marker-A"). + Return([]domain.Vtxo{ + {Outpoint: domain.Outpoint{Txid: "window-1", VOut: 0}, Amount: 200}, + }, nil).Once() // Expect exactly one call + + err := indexer.ensureVtxosCached( + ctx, + []domain.Outpoint{{Txid: "vtxo-1", VOut: 0}}, + cache, + loadedMarkers, + ) + require.NoError(t, err) + require.True(t, loadedMarkers["marker-A"]) + + // Second call: vtxo-2 also has marker-A + vtxo2 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "vtxo-2", VOut: 0}, + Amount: 300, + MarkerIDs: []string{"marker-A"}, + } + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: "vtxo-2", VOut: 0}}). + Return([]domain.Vtxo{vtxo2}, nil) + + err = indexer.ensureVtxosCached( + ctx, + []domain.Outpoint{{Txid: "vtxo-2", VOut: 0}}, + cache, + loadedMarkers, + ) + require.NoError(t, err) + + // GetVtxosByMarker for marker-A should have been called only once + markerRepo.AssertNumberOfCalls(t, "GetVtxosByMarker", 1) +} + +// TestEnsureVtxosCached_GetVtxosByMarkerErrorSwallowed verifies that an error +// from GetVtxosByMarker is gracefully swallowed — the VTXO itself is still +// cached and the function returns no error. +func TestEnsureVtxosCached_GetVtxosByMarkerErrorSwallowed(t *testing.T) { + vtxoRepo, markerRepo, indexer := newTestIndexer() + + ctx := context.Background() + cache := make(map[string]domain.Vtxo) + loadedMarkers := make(map[string]bool) + + vtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "vtxo-ok", VOut: 0}, + Amount: 500, + MarkerIDs: []string{"marker-bad"}, + } + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: "vtxo-ok", VOut: 0}}). + Return([]domain.Vtxo{vtxo}, nil) + markerRepo.On("GetVtxosByMarker", ctx, "marker-bad"). + Return(nil, fmt.Errorf("marker window load failed")) + + err := indexer.ensureVtxosCached( + ctx, + []domain.Outpoint{{Txid: "vtxo-ok", VOut: 0}}, + cache, + loadedMarkers, + ) + + // No error propagated + require.NoError(t, err) + // The VTXO itself is still in cache + require.Contains(t, cache, "vtxo-ok:0") + // Marker is marked as loaded (won't retry) + require.True(t, loadedMarkers["marker-bad"]) +} + +// TestGetVtxoChain_DefaultPageSizeWithTokenOnly verifies that when page is nil +// but a pageToken is provided, the default page size (maxPageSizeVtxoChain=100) +// is used instead of returning the full chain. +func TestGetVtxoChain_DefaultPageSizeWithTokenOnly(t *testing.T) { + vtxoRepo, markerRepo, offchainTxRepo, indexer := newTestIndexerWithOffchain() + ctx := context.Background() + + vtxoKey := setupPreconfirmedChain(t, ctx, vtxoRepo, markerRepo, offchainTxRepo) + + // Get the first page with an explicit page size to obtain a token + page := &Page{PageSize: 2} + resp1, err := indexer.GetVtxoChain(ctx, vtxoKey, page, "") + require.NoError(t, err) + require.NotEmpty(t, resp1.NextPageToken) + + // Resume with token but nil page — should use default page size (100), + // which is large enough to return the remaining chain in one shot. + resp2, err := indexer.GetVtxoChain(ctx, vtxoKey, nil, resp1.NextPageToken) + require.NoError(t, err) + // Remaining chain: B(ark+cp) + C(ark) = 3 items, all fit in default page + require.Equal(t, 3, len(resp2.Chain)) + require.Empty(t, resp2.NextPageToken) +} + // TestGetVtxoChain_InvalidPageToken verifies that an invalid page_token // returns an error. func TestGetVtxoChain_InvalidPageToken(t *testing.T) { @@ -501,3 +683,174 @@ func TestGetVtxoChain_BackwardCompat_NilPageEmptyToken(t *testing.T) { require.Contains(t, err.Error(), "vtxo not found") require.NotContains(t, err.Error(), "invalid page_token") } + +// setupPreconfirmedChain sets up a chain of preconfirmed VTXOs for pagination tests. +// Returns the VTXOs, the starting outpoint, and configures all mock expectations. +// Chain: vtxo-A -> checkpoint(input=vtxo-B) -> vtxo-B -> checkpoint(input=vtxo-C) -> vtxo-C (terminal) +func setupPreconfirmedChain( + t *testing.T, + ctx context.Context, + vtxoRepo *mockVtxoRepoForIndexer, + markerRepo *mockMarkerRepoForIndexer, + offchainTxRepo *mockOffchainTxRepoForIndexer, +) Outpoint { + t.Helper() + + txidA := strings.Repeat("a", 64) + txidB := strings.Repeat("b", 64) + txidC := strings.Repeat("c", 64) + + vtxoA := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: txidA, VOut: 0}, + Preconfirmed: true, + ExpiresAt: 1000, + } + vtxoB := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: txidB, VOut: 0}, + Preconfirmed: true, + ExpiresAt: 2000, + } + vtxoC := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: txidC, VOut: 0}, + Preconfirmed: true, + ExpiresAt: 3000, + } + + // VTXOs returned from DB + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: txidA, VOut: 0}}). + Return([]domain.Vtxo{vtxoA}, nil) + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: txidB, VOut: 0}}). + Return([]domain.Vtxo{vtxoB}, nil) + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: txidC, VOut: 0}}). + Return([]domain.Vtxo{vtxoC}, nil) + + // Marker repo won't be used (no markers on these VTXOs) + markerRepo.On("GetVtxosByMarker", ctx, mock.Anything). + Return([]domain.Vtxo{}, nil).Maybe() + + // Checkpoint PSBTs: A's checkpoint points to B, B's checkpoint points to C + cpA := makeCheckpointPSBT(t, txidB, 0) + cpB := makeCheckpointPSBT(t, txidC, 0) + + offchainTxRepo.On("GetOffchainTx", ctx, txidA). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{"cp-a": cpA}}, nil) + offchainTxRepo.On("GetOffchainTx", ctx, txidB). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{"cp-b": cpB}}, nil) + offchainTxRepo.On("GetOffchainTx", ctx, txidC). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{}}, nil) + + return Outpoint{Txid: txidA, VOut: 0} +} + +// TestGetVtxoChain_PaginationFirstPage verifies that the first page returns +// the expected number of items and a non-empty next_page_token when the chain +// exceeds the page size. +func TestGetVtxoChain_PaginationFirstPage(t *testing.T) { + vtxoRepo, markerRepo, offchainTxRepo, indexer := newTestIndexerWithOffchain() + ctx := context.Background() + + vtxoKey := setupPreconfirmedChain(t, ctx, vtxoRepo, markerRepo, offchainTxRepo) + + // Page size 2: vtxo-A produces 2 chain items (ark + checkpoint), + // then vtxo-B triggers early termination. + page := &Page{PageSize: 2} + resp, err := indexer.GetVtxoChain(ctx, vtxoKey, page, "") + + require.NoError(t, err) + require.Len(t, resp.Chain, 2) + require.Equal(t, IndexerChainedTxTypeArk, resp.Chain[0].Type) + require.Equal(t, IndexerChainedTxTypeCheckpoint, resp.Chain[1].Type) + require.NotEmpty(t, resp.NextPageToken, "should have next page token") +} + +// TestGetVtxoChain_PaginationResumeWithToken verifies that resuming with a +// page token continues the chain from where the previous page left off, +// eventually exhausting the chain with an empty token. +func TestGetVtxoChain_PaginationResumeWithToken(t *testing.T) { + vtxoRepo, markerRepo, offchainTxRepo, indexer := newTestIndexerWithOffchain() + ctx := context.Background() + + vtxoKey := setupPreconfirmedChain(t, ctx, vtxoRepo, markerRepo, offchainTxRepo) + + // Chain: A(ark+cp) -> B(ark+cp) -> C(ark) = 5 items total + // Page size 2: page1=2, page2=2, page3=1 + page := &Page{PageSize: 2} + + // Page 1 + resp1, err := indexer.GetVtxoChain(ctx, vtxoKey, page, "") + require.NoError(t, err) + require.Len(t, resp1.Chain, 2) + require.NotEmpty(t, resp1.NextPageToken) + + // Page 2: resume with token from page 1 + resp2, err := indexer.GetVtxoChain(ctx, vtxoKey, page, resp1.NextPageToken) + require.NoError(t, err) + require.Len(t, resp2.Chain, 2) + require.NotEmpty(t, resp2.NextPageToken) + + // Page 3: resume with token from page 2 + resp3, err := indexer.GetVtxoChain(ctx, vtxoKey, page, resp2.NextPageToken) + require.NoError(t, err) + require.Len(t, resp3.Chain, 1) + require.Empty(t, resp3.NextPageToken, "last page should have empty token") + + // Verify total items across all pages + totalItems := len(resp1.Chain) + len(resp2.Chain) + len(resp3.Chain) + require.Equal(t, 5, totalItems) + + // Verify chain types: each vtxo with checkpoints produces ark+checkpoint, + // terminal vtxo (C) produces only ark. + require.Equal(t, IndexerChainedTxTypeArk, resp3.Chain[0].Type) +} + +// TestGetVtxoChain_ShortChainNoToken verifies that when the chain is shorter +// than the page size, all items are returned with an empty next_page_token. +func TestGetVtxoChain_ShortChainNoToken(t *testing.T) { + vtxoRepo, markerRepo, offchainTxRepo, indexer := newTestIndexerWithOffchain() + ctx := context.Background() + + txidA := strings.Repeat("a", 64) + + // Single terminal preconfirmed VTXO (no checkpoints) + vtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: txidA, VOut: 0}, + Preconfirmed: true, + ExpiresAt: 1000, + } + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: txidA, VOut: 0}}). + Return([]domain.Vtxo{vtxo}, nil) + markerRepo.On("GetVtxosByMarker", ctx, mock.Anything). + Return([]domain.Vtxo{}, nil).Maybe() + offchainTxRepo.On("GetOffchainTx", ctx, txidA). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{}}, nil) + + // Page size larger than chain + page := &Page{PageSize: 100} + resp, err := indexer.GetVtxoChain(ctx, Outpoint{Txid: txidA, VOut: 0}, page, "") + + require.NoError(t, err) + require.Len(t, resp.Chain, 1) // Just the ark tx + require.Empty(t, resp.NextPageToken, "short chain should have empty token") + require.Equal(t, IndexerChainedTxTypeArk, resp.Chain[0].Type) +} + +// TestGetVtxoChain_PageSizeRespected verifies that each page never exceeds the +// page size (with allowance for grouped items from a single VTXO). +func TestGetVtxoChain_PageSizeRespected(t *testing.T) { + vtxoRepo, markerRepo, offchainTxRepo, indexer := newTestIndexerWithOffchain() + ctx := context.Background() + + vtxoKey := setupPreconfirmedChain(t, ctx, vtxoRepo, markerRepo, offchainTxRepo) + + // Use page size 1 — each VTXO produces 2 items (ark+checkpoint) for A and B, + // so pages will slightly overflow since items for one VTXO are emitted together. + page := &Page{PageSize: 1} + + resp, err := indexer.GetVtxoChain(ctx, vtxoKey, page, "") + require.NoError(t, err) + + // vtxo-A emits 2 items (ark + checkpoint) even though pageSize=1, + // because all items for a VTXO are emitted together. + require.Equal(t, 2, len(resp.Chain)) + require.NotEmpty(t, resp.NextPageToken) +} From dc702314ad1ef8218adcbeb54b0faf2155ea583d Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Wed, 18 Feb 2026 17:34:08 -0500 Subject: [PATCH 30/54] make proto --- api-spec/protobuf/gen/ark/v1/indexer.pb.rgw.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api-spec/protobuf/gen/ark/v1/indexer.pb.rgw.go b/api-spec/protobuf/gen/ark/v1/indexer.pb.rgw.go index 724f6eb80..6a67b4ec6 100644 --- a/api-spec/protobuf/gen/ark/v1/indexer.pb.rgw.go +++ b/api-spec/protobuf/gen/ark/v1/indexer.pb.rgw.go @@ -125,7 +125,7 @@ func request_IndexerService_GetConnectors_0(ctx context.Context, marshaler gatew var ( query_params_IndexerService_GetVtxoTree_0 = gateway.QueryParameterParseOptions{ - Filter: trie.New("txid", "vout", "batch_outpoint.vout", "batch_outpoint.txid"), + Filter: trie.New("batch_outpoint.txid", "batch_outpoint.vout", "txid", "vout"), } ) From ae19758d8af3c8010ddc4a9d64778b2148ad0a54 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Mon, 23 Feb 2026 18:33:11 -0500 Subject: [PATCH 31/54] Fix SweptAt timestamps, GetVtxosByArkTxid queries, and dust marker sweep --- internal/core/application/sweeper.go | 2 +- internal/core/application/sweeper_test.go | 4 ++-- .../infrastructure/db/badger/marker_repo.go | 2 +- .../infrastructure/db/postgres/marker_repo.go | 2 +- .../db/postgres/sqlc/queries/query.sql.go | 2 +- .../infrastructure/db/postgres/sqlc/query.sql | 2 +- internal/infrastructure/db/service.go | 24 ++++++++++--------- internal/infrastructure/db/service_test.go | 4 ++-- .../infrastructure/db/sqlite/marker_repo.go | 2 +- .../db/sqlite/sqlc/queries/query.sql.go | 2 +- .../infrastructure/db/sqlite/sqlc/query.sql | 2 +- 11 files changed, 25 insertions(+), 23 deletions(-) diff --git a/internal/core/application/sweeper.go b/internal/core/application/sweeper.go index 7d2aa9e5a..e84bd5227 100644 --- a/internal/core/application/sweeper.go +++ b/internal/core/application/sweeper.go @@ -785,7 +785,7 @@ func (s *sweeper) createCheckpointSweepTask( markerIDs = append(markerIDs, markerID) } - sweptAt := time.Now().Unix() + sweptAt := time.Now().UnixMilli() markerStore := s.repoManager.Markers() if err := markerStore.BulkSweepMarkers(ctx, markerIDs, sweptAt); err != nil { log.WithError(err).Warn("failed to bulk sweep markers") diff --git a/internal/core/application/sweeper_test.go b/internal/core/application/sweeper_test.go index 7d5e79f9f..8843d3ee7 100644 --- a/internal/core/application/sweeper_test.go +++ b/internal/core/application/sweeper_test.go @@ -805,7 +805,7 @@ func TestCreateCheckpointSweepTask_SweptAtTimestamp(t *testing.T) { Return(vtxos, nil) // Capture the sweptAt timestamp - beforeExec := time.Now().Unix() + beforeExec := time.Now().UnixMilli() var capturedSweptAt int64 markerRepo.On("BulkSweepMarkers", mock.Anything, mock.Anything, mock.MatchedBy(func(sweptAt int64) bool { @@ -816,7 +816,7 @@ func TestCreateCheckpointSweepTask_SweptAtTimestamp(t *testing.T) { task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) err := task() - afterExec := time.Now().Unix() + afterExec := time.Now().UnixMilli() require.NoError(t, err) // Verify timestamp is within the execution window diff --git a/internal/infrastructure/db/badger/marker_repo.go b/internal/infrastructure/db/badger/marker_repo.go index c2b67192e..700c15f08 100644 --- a/internal/infrastructure/db/badger/marker_repo.go +++ b/internal/infrastructure/db/badger/marker_repo.go @@ -527,7 +527,7 @@ func (r *markerRepository) GetVtxosByArkTxid( arkTxid string, ) ([]domain.Vtxo, error) { var dtos []vtxoDTO - err := r.vtxoStore.Find(&dtos, badgerhold.Where("Txid").Eq(arkTxid)) + err := r.vtxoStore.Find(&dtos, badgerhold.Where("ArkTxid").Eq(arkTxid)) if err != nil { return nil, err } diff --git a/internal/infrastructure/db/postgres/marker_repo.go b/internal/infrastructure/db/postgres/marker_repo.go index 244132ddd..46cc620ab 100644 --- a/internal/infrastructure/db/postgres/marker_repo.go +++ b/internal/infrastructure/db/postgres/marker_repo.go @@ -281,7 +281,7 @@ func (m *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID stri // Insert the marker into swept_marker (sweep state is computed via view) if err := m.querier.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ MarkerID: markerID, - SweptAt: time.Now().Unix(), + SweptAt: time.Now().UnixMilli(), }); err != nil { return 0, fmt.Errorf("failed to insert swept marker: %w", err) } diff --git a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go index 0a931f699..e19f6eef2 100644 --- a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go @@ -1943,7 +1943,7 @@ func (q *Queries) SelectVtxoPubKeysByCommitmentTxid(ctx context.Context, arg Sel } const selectVtxosByArkTxid = `-- name: SelectVtxosByArkTxid :many -SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, markers, commitments, swept, asset_id, asset_amount FROM vtxo_vw WHERE txid = $1 +SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spent_by, spent, unrolled, preconfirmed, settled_by, ark_txid, intent_id, updated_at, depth, markers, commitments, swept, asset_id, asset_amount FROM vtxo_vw WHERE ark_txid = $1 ` // Get all VTXOs created by a specific ark tx (offchain tx) diff --git a/internal/infrastructure/db/postgres/sqlc/query.sql b/internal/infrastructure/db/postgres/sqlc/query.sql index aa7ad459c..47c335993 100644 --- a/internal/infrastructure/db/postgres/sqlc/query.sql +++ b/internal/infrastructure/db/postgres/sqlc/query.sql @@ -505,7 +505,7 @@ ORDER BY depth DESC; -- name: SelectVtxosByArkTxid :many -- Get all VTXOs created by a specific ark tx (offchain tx) -SELECT * FROM vtxo_vw WHERE txid = @ark_txid; +SELECT * FROM vtxo_vw WHERE ark_txid = @ark_txid; -- name: SelectVtxoChainByMarker :many -- Get VTXOs whose markers JSONB array contains any of the given marker IDs diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index f4ee3e0ff..d6d7ddcf7 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -659,7 +659,7 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) // once the offchain tx is finalized, the user signed the checkpoint txs // thus, we can create the new vtxos in the db. newVtxos := make([]domain.Vtxo, 0, len(outs)) - dustVtxoOutpoints := make([]domain.Outpoint, 0) + createdDustMarkerIDs := make([]string, 0) for outIndex, out := range outs { // ignore anchors if bytes.Equal(out.PkScript, txutils.ANCHOR_PKSCRIPT) || @@ -675,7 +675,6 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) vtxoMarkerIDs := markerIDs isDust := script.IsSubDustScript(out.PkScript) if isDust { - dustVtxoOutpoints = append(dustVtxoOutpoints, outpoint) // Dust VTXOs get their own outpoint-based marker so they can be // swept individually without affecting sibling non-dust VTXOs // that share the same inherited parent markers. @@ -686,6 +685,8 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) ParentMarkerIDs: markerIDs, }); err != nil { log.WithError(err).Warnf("failed to create dust marker %s", dustMarkerID) + } else { + createdDustMarkerIDs = append(createdDustMarkerIDs, dustMarkerID) } vtxoMarkerIDs = append(append([]string{}, markerIDs...), dustMarkerID) } @@ -730,14 +731,15 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) // Mark dust VTXOs as swept via their markers // Dust vtxos are below dust limit and can't be spent again in future offchain tx // Because sub-dust vtxos are using OP_RETURN output script, they can't be unilaterally exited - if len(dustVtxoOutpoints) > 0 { - dustMarkerIDs := make([]string, 0, len(dustVtxoOutpoints)) - for _, outpoint := range dustVtxoOutpoints { - dustMarkerIDs = append(dustMarkerIDs, outpoint.String()) - } - sweptAt := time.Now().Unix() - if err := s.markerStore.BulkSweepMarkers(ctx, dustMarkerIDs, sweptAt); err != nil { - log.WithError(err).Warnf("failed to sweep %d dust vtxo markers", len(dustMarkerIDs)) + if len(createdDustMarkerIDs) > 0 { + sweptAt := time.Now().UnixMilli() + if err := s.markerStore.BulkSweepMarkers( + ctx, + createdDustMarkerIDs, + sweptAt, + ); err != nil { + log.WithError(err). + Warnf("failed to sweep %d dust vtxo markers", len(createdDustMarkerIDs)) } } } @@ -862,7 +864,7 @@ func (s *service) sweepVtxosWithMarkers( markerIDs = append(markerIDs, markerID) } - sweptAt := time.Now().Unix() + sweptAt := time.Now().UnixMilli() if err := s.markerStore.BulkSweepMarkers(ctx, markerIDs, sweptAt); err != nil { log.WithError(err).Warn("failed to bulk sweep markers") return 0 diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index 436a45f4c..87181332c 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -2211,8 +2211,8 @@ func testMarkerChainTraversal(t *testing.T, svc ports.RepoManager) { // Test GetVtxosByArkTxid - returns VTXOs created by specific ark tx vtxosByArkTxid, err := svc.Markers().GetVtxosByArkTxid(ctx, arkTxid) require.NoError(t, err) - require.Len(t, vtxosByArkTxid, 1) // Only vtxo2 has Txid == arkTxid - require.Equal(t, vtxo2.Txid, vtxosByArkTxid[0].Txid) + require.Len(t, vtxosByArkTxid, 1) // Only vtxo1 has ArkTxid == arkTxid + require.Equal(t, vtxo1.Txid, vtxosByArkTxid[0].Txid) // Test GetVtxosByArkTxid with non-existent ark txid vtxosByArkTxid, err = svc.Markers().GetVtxosByArkTxid(ctx, "nonexistent") diff --git a/internal/infrastructure/db/sqlite/marker_repo.go b/internal/infrastructure/db/sqlite/marker_repo.go index aacbcab4b..25d4098ba 100644 --- a/internal/infrastructure/db/sqlite/marker_repo.go +++ b/internal/infrastructure/db/sqlite/marker_repo.go @@ -276,7 +276,7 @@ func (m *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID stri // Insert the marker into swept_marker (sweep state is computed via view) if err := m.querier.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ MarkerID: markerID, - SweptAt: time.Now().Unix(), + SweptAt: time.Now().UnixMilli(), }); err != nil { return 0, fmt.Errorf("failed to insert swept marker: %w", err) } diff --git a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go index 6e8500eef..4f7b34981 100644 --- a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go @@ -2035,7 +2035,7 @@ func (q *Queries) SelectVtxoPubKeysByCommitmentTxid(ctx context.Context, arg Sel } const selectVtxosByArkTxid = `-- name: SelectVtxosByArkTxid :many -SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept, vtxo_vw.asset_id, vtxo_vw.asset_amount FROM vtxo_vw WHERE txid = ?1 +SELECT vtxo_vw.txid, vtxo_vw.vout, vtxo_vw.pubkey, vtxo_vw.amount, vtxo_vw.expires_at, vtxo_vw.created_at, vtxo_vw.commitment_txid, vtxo_vw.spent_by, vtxo_vw.spent, vtxo_vw.unrolled, vtxo_vw.preconfirmed, vtxo_vw.settled_by, vtxo_vw.ark_txid, vtxo_vw.intent_id, vtxo_vw.updated_at, vtxo_vw.depth, vtxo_vw.markers, vtxo_vw.commitments, vtxo_vw.swept, vtxo_vw.asset_id, vtxo_vw.asset_amount FROM vtxo_vw WHERE ark_txid = ?1 ` type SelectVtxosByArkTxidRow struct { diff --git a/internal/infrastructure/db/sqlite/sqlc/query.sql b/internal/infrastructure/db/sqlite/sqlc/query.sql index 8a2fb04e6..b26c28f19 100644 --- a/internal/infrastructure/db/sqlite/sqlc/query.sql +++ b/internal/infrastructure/db/sqlite/sqlc/query.sql @@ -508,7 +508,7 @@ ORDER BY depth DESC; -- name: SelectVtxosByArkTxid :many -- Get all VTXOs created by a specific ark tx (offchain tx) -SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE txid = @ark_txid; +SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE ark_txid = @ark_txid; -- name: SelectVtxoChainByMarker :many -- Get VTXOs whose markers array contains the given marker_id. From 9463acb1921a76e1b1844c1752a2d805c31b000c Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Mon, 2 Mar 2026 12:02:05 -0500 Subject: [PATCH 32/54] make lint --- internal/infrastructure/db/sqlite/marker_repo.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/infrastructure/db/sqlite/marker_repo.go b/internal/infrastructure/db/sqlite/marker_repo.go index 43d212885..37c1d1943 100644 --- a/internal/infrastructure/db/sqlite/marker_repo.go +++ b/internal/infrastructure/db/sqlite/marker_repo.go @@ -335,7 +335,10 @@ func (m *markerRepository) GetVtxosByArkTxid( ctx context.Context, arkTxid string, ) ([]domain.Vtxo, error) { - rows, err := m.querier.SelectVtxosByArkTxid(ctx, sql.NullString{String: arkTxid, Valid: arkTxid != ""}) + rows, err := m.querier.SelectVtxosByArkTxid( + ctx, + sql.NullString{String: arkTxid, Valid: arkTxid != ""}, + ) if err != nil { return nil, err } From eabad6f5f0cce365608d2e293a82cb2d76b7458b Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Mon, 16 Mar 2026 19:05:18 -0400 Subject: [PATCH 33/54] refactor marker logic into domain and carry depth via events --- internal/core/application/service.go | 23 + internal/core/application/service_test.go | 599 --------------------- internal/core/domain/marker.go | 26 +- internal/core/domain/marker_test.go | 597 +++++++++++++++++++- internal/core/domain/offchain_tx.go | 7 + internal/core/domain/offchain_tx_event.go | 2 + internal/core/domain/offchain_tx_test.go | 6 + internal/infrastructure/db/service.go | 73 +-- internal/infrastructure/db/service_test.go | 4 +- 9 files changed, 666 insertions(+), 671 deletions(-) diff --git a/internal/core/application/service.go b/internal/core/application/service.go index b1096a8c3..aa2e4a0d2 100644 --- a/internal/core/application/service.go +++ b/internal/core/application/service.go @@ -1156,9 +1156,32 @@ func (s *service) SubmitOffchainTx( signedCheckpointTxsMap[rebuiltCheckpointTx.UnsignedTx.TxID()] = signedCheckpointTx } + // Compute depth and parent markers from spent VTXOs for the accepted event. + var maxDepth uint32 + parentMarkerSet := make(map[string]struct{}) + for _, v := range spentVtxos { + if v.Depth > maxDepth { + maxDepth = v.Depth + } + for _, markerID := range v.MarkerIDs { + if markerID != "" { + parentMarkerSet[markerID] = struct{}{} + } + } + } + var newDepth uint32 + if len(spentVtxos) > 0 { + newDepth = maxDepth + 1 + } + parentMarkerIDs := make([]string, 0, len(parentMarkerSet)) + for id := range parentMarkerSet { + parentMarkerIDs = append(parentMarkerIDs, id) + } + change, err := offchainTx.Accept( fullySignedArkTx, signedCheckpointTxsMap, commitmentTxsByCheckpointTxid, rootCommitmentTxid, expiration, + newDepth, parentMarkerIDs, ) if err != nil { return nil, errors.INTERNAL_ERROR.New("failed to accept offchain tx: %w", err). diff --git a/internal/core/application/service_test.go b/internal/core/application/service_test.go index d1c98e43a..cda4a5e1b 100644 --- a/internal/core/application/service_test.go +++ b/internal/core/application/service_test.go @@ -1,12 +1,9 @@ package application import ( - "fmt" - "sort" "testing" "time" - "github.com/arkade-os/arkd/internal/core/domain" "github.com/stretchr/testify/require" ) @@ -69,599 +66,3 @@ func parseTime(t *testing.T, value string) time.Time { require.NoError(t, err) return tm } - -// calculateMaxDepth mimics the depth calculation logic in the service event handler. -// This function exists to make the depth calculation testable independently. -func calculateMaxDepth(spentVtxos []domain.Vtxo) uint32 { - var maxDepth uint32 - for _, v := range spentVtxos { - if v.Depth > maxDepth { - maxDepth = v.Depth - } - } - return maxDepth -} - -func TestDepthCalculation(t *testing.T) { - testCases := []struct { - name string - spentVtxos []domain.Vtxo - expectedDepth uint32 - description string - }{ - { - name: "single batch vtxo at depth 0", - spentVtxos: []domain.Vtxo{{Depth: 0}}, - expectedDepth: 1, - description: "spending a batch vtxo creates vtxo at depth 1", - }, - { - name: "single vtxo at depth 50", - spentVtxos: []domain.Vtxo{{Depth: 50}}, - expectedDepth: 51, - description: "spending a chained vtxo increments depth", - }, - { - name: "multiple vtxos with same depth", - spentVtxos: []domain.Vtxo{ - {Depth: 10}, - {Depth: 10}, - {Depth: 10}, - }, - expectedDepth: 11, - description: "combining vtxos at same depth increments once", - }, - { - name: "multiple vtxos with different depths", - spentVtxos: []domain.Vtxo{ - {Depth: 5}, - {Depth: 25}, - {Depth: 15}, - }, - expectedDepth: 26, - description: "uses max depth from inputs", - }, - { - name: "vtxos spanning marker boundary", - spentVtxos: []domain.Vtxo{ - {Depth: 95}, - {Depth: 105}, - }, - expectedDepth: 106, - description: "handles depths across marker boundaries", - }, - { - name: "deep chain near marker boundary", - spentVtxos: []domain.Vtxo{ - {Depth: 99}, - }, - expectedDepth: 100, - description: "result at marker boundary (100)", - }, - { - name: "very deep chain", - spentVtxos: []domain.Vtxo{ - {Depth: 500}, - }, - expectedDepth: 501, - description: "handles deep chains beyond multiple marker intervals", - }, - { - name: "no spent vtxos (empty)", - spentVtxos: []domain.Vtxo{}, - expectedDepth: 0, - description: "empty input results in depth 0 (no spent vtxos means maxDepth stays 0, newDepth = 0)", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - maxDepth := calculateMaxDepth(tc.spentVtxos) - var newDepth uint32 - if len(tc.spentVtxos) > 0 { - newDepth = maxDepth + 1 - } - require.Equal(t, tc.expectedDepth, newDepth, tc.description) - }) - } -} - -func TestDepthAtMarkerBoundary(t *testing.T) { - // Test integration of depth and marker boundary detection - testCases := []struct { - depth uint32 - isAtBoundary bool - description string - }{ - {0, true, "depth 0 is at marker boundary"}, - {1, false, "depth 1 is not at boundary"}, - {50, false, "depth 50 is not at boundary"}, - {99, false, "depth 99 is not at boundary"}, - {100, true, "depth 100 is at marker boundary"}, - {101, false, "depth 101 is not at boundary"}, - {200, true, "depth 200 is at marker boundary"}, - {500, true, "depth 500 is at marker boundary"}, - {1000, true, "depth 1000 is at marker boundary"}, - } - - for _, tc := range testCases { - t.Run(tc.description, func(t *testing.T) { - isAtBoundary := domain.IsAtMarkerBoundary(tc.depth) - require.Equal(t, tc.isAtBoundary, isAtBoundary) - }) - } -} - -func TestDepthIncrementCreatesMarkerAtBoundary(t *testing.T) { - // Test scenario: when depth increments to a marker boundary, - // a marker should be created for that VTXO - testCases := []struct { - parentDepth uint32 - newDepth uint32 - shouldCreateMarker bool - }{ - {99, 100, true}, // crossing into boundary - {100, 101, false}, // leaving boundary - {199, 200, true}, // crossing into next boundary - {0, 1, false}, // moving away from initial boundary - {98, 99, false}, // approaching but not at boundary - } - - for _, tc := range testCases { - t.Run("", func(t *testing.T) { - // Simulate the depth increment - spentVtxos := []domain.Vtxo{{Depth: tc.parentDepth}} - maxDepth := calculateMaxDepth(spentVtxos) - newDepth := maxDepth + 1 - - require.Equal(t, tc.newDepth, newDepth) - isAtBoundary := domain.IsAtMarkerBoundary(newDepth) - require.Equal(t, tc.shouldCreateMarker, isAtBoundary) - }) - } -} - -// collectParentMarkers mimics the parent marker collection logic in the -// service's updateProjectionsAfterOffchainTxEvents handler. -// It collects ALL unique, non-empty marker IDs from the spent VTXOs. -func collectParentMarkers(spentVtxos []domain.Vtxo) []string { - parentMarkerSet := make(map[string]struct{}) - for _, v := range spentVtxos { - for _, markerID := range v.MarkerIDs { - if markerID != "" { - parentMarkerSet[markerID] = struct{}{} - } - } - } - result := make([]string, 0, len(parentMarkerSet)) - for id := range parentMarkerSet { - result = append(result, id) - } - return result -} - -// deriveMarkerIDs mimics the marker creation/inheritance decision in the -// service's updateProjectionsAfterOffchainTxEvents handler. -// At boundary depths a new marker is created; otherwise parent markers are inherited. -func deriveMarkerIDs( - newDepth uint32, - parentMarkerIDs []string, - txid string, -) (markerIDs []string, createdMarker *domain.Marker) { - if domain.IsAtMarkerBoundary(newDepth) { - newMarkerID := txid + ":marker:" + fmt.Sprintf("%d", newDepth) - marker := domain.Marker{ - ID: newMarkerID, - Depth: newDepth, - ParentMarkerIDs: parentMarkerIDs, - } - return []string{newMarkerID}, &marker - } - if len(parentMarkerIDs) > 0 { - return parentMarkerIDs, nil - } - return nil, nil -} - -func TestParentMarkerCollectionFromMultipleParents(t *testing.T) { - // When spending multiple VTXOs with different marker sets, - // the parent marker set should be the deduplicated union of all inputs' markers. - testCases := []struct { - name string - spentVtxos []domain.Vtxo - expectedMarkers []string - }{ - { - name: "single parent with one marker", - spentVtxos: []domain.Vtxo{ - {MarkerIDs: []string{"marker-A"}}, - }, - expectedMarkers: []string{"marker-A"}, - }, - { - name: "two parents with distinct markers", - spentVtxos: []domain.Vtxo{ - {MarkerIDs: []string{"marker-A"}}, - {MarkerIDs: []string{"marker-B"}}, - }, - expectedMarkers: []string{"marker-A", "marker-B"}, - }, - { - name: "three parents with overlapping markers", - spentVtxos: []domain.Vtxo{ - {MarkerIDs: []string{"marker-A", "marker-B"}}, - {MarkerIDs: []string{"marker-B", "marker-C"}}, - {MarkerIDs: []string{"marker-A", "marker-C"}}, - }, - expectedMarkers: []string{"marker-A", "marker-B", "marker-C"}, - }, - { - name: "all parents share the same marker", - spentVtxos: []domain.Vtxo{ - {MarkerIDs: []string{"root-marker"}}, - {MarkerIDs: []string{"root-marker"}}, - {MarkerIDs: []string{"root-marker"}}, - }, - expectedMarkers: []string{"root-marker"}, - }, - { - name: "no parents", - spentVtxos: []domain.Vtxo{}, - expectedMarkers: []string{}, - }, - { - name: "parent with no markers", - spentVtxos: []domain.Vtxo{ - {MarkerIDs: []string{}}, - }, - expectedMarkers: []string{}, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - result := collectParentMarkers(tc.spentVtxos) - sort.Strings(result) - sort.Strings(tc.expectedMarkers) - require.Equal(t, tc.expectedMarkers, result) - }) - } -} - -func TestParentMarkerCollectionSkipsEmptyMarkerIDs(t *testing.T) { - // Empty string marker IDs should be filtered out. - spentVtxos := []domain.Vtxo{ - {MarkerIDs: []string{"marker-A", "", "marker-B"}}, - {MarkerIDs: []string{"", ""}}, - {MarkerIDs: []string{"marker-C", ""}}, - } - - result := collectParentMarkers(spentVtxos) - sort.Strings(result) - require.Equal(t, []string{"marker-A", "marker-B", "marker-C"}, result) -} - -func TestMarkerInheritanceAtNonBoundary(t *testing.T) { - // When the new depth is NOT at a marker boundary, the child VTXO - // should inherit ALL parent marker IDs (no new marker created). - testCases := []struct { - name string - parentDepths []uint32 - parentMarkerSets [][]string - expectedDepth uint32 - expectedMarkers []string - description string - }{ - { - name: "single parent at depth 0, child at depth 1", - parentDepths: []uint32{0}, - parentMarkerSets: [][]string{{"root-marker-1"}}, - expectedDepth: 1, - expectedMarkers: []string{"root-marker-1"}, - description: "child inherits single parent marker", - }, - { - name: "single parent at depth 50, child at depth 51", - parentDepths: []uint32{50}, - parentMarkerSets: [][]string{{"marker-A", "marker-B"}}, - expectedDepth: 51, - expectedMarkers: []string{"marker-A", "marker-B"}, - description: "child inherits multiple parent markers", - }, - { - name: "two parents at different depths, child not at boundary", - parentDepths: []uint32{30, 40}, - parentMarkerSets: [][]string{{"marker-X"}, {"marker-Y"}}, - expectedDepth: 41, - expectedMarkers: []string{"marker-X", "marker-Y"}, - description: "child inherits union of all parent markers", - }, - { - name: "three parents with overlapping markers", - parentDepths: []uint32{10, 20, 15}, - parentMarkerSets: [][]string{{"m1", "m2"}, {"m2", "m3"}, {"m1"}}, - expectedDepth: 21, - expectedMarkers: []string{"m1", "m2", "m3"}, - description: "child inherits deduplicated union", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - spentVtxos := make([]domain.Vtxo, len(tc.parentDepths)) - for i, depth := range tc.parentDepths { - spentVtxos[i] = domain.Vtxo{ - Depth: depth, - MarkerIDs: tc.parentMarkerSets[i], - } - } - - maxDepth := calculateMaxDepth(spentVtxos) - newDepth := maxDepth + 1 - require.Equal(t, tc.expectedDepth, newDepth) - - // Should NOT be at a marker boundary - require.False(t, domain.IsAtMarkerBoundary(newDepth), - "depth %d should not be at marker boundary for this test", newDepth) - - parentMarkers := collectParentMarkers(spentVtxos) - markerIDs, createdMarker := deriveMarkerIDs(newDepth, parentMarkers, "some-txid") - - // No new marker should be created - require.Nil(t, createdMarker, tc.description) - // Should inherit all parent markers - sort.Strings(markerIDs) - sort.Strings(tc.expectedMarkers) - require.Equal(t, tc.expectedMarkers, markerIDs, tc.description) - }) - } -} - -func TestMarkerCreationAtBoundary(t *testing.T) { - // When the new depth IS at a marker boundary, a new marker should be - // created with the collected parent markers as its ParentMarkerIDs. - testCases := []struct { - name string - parentDepths []uint32 - parentMarkerSets [][]string - expectedDepth uint32 - description string - }{ - { - name: "parent at depth 99, child at depth 100", - parentDepths: []uint32{99}, - parentMarkerSets: [][]string{{"root-marker"}}, - expectedDepth: 100, - description: "first non-root boundary", - }, - { - name: "parent at depth 199, child at depth 200", - parentDepths: []uint32{199}, - parentMarkerSets: [][]string{{"marker-100", "root-marker"}}, - expectedDepth: 200, - description: "second boundary with two parent markers", - }, - { - name: "multiple parents converging at boundary", - parentDepths: []uint32{95, 99}, - parentMarkerSets: [][]string{{"marker-A"}, {"marker-B"}}, - expectedDepth: 100, - description: "boundary with multiple parent VTXOs", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - spentVtxos := make([]domain.Vtxo, len(tc.parentDepths)) - for i, depth := range tc.parentDepths { - spentVtxos[i] = domain.Vtxo{ - Depth: depth, - MarkerIDs: tc.parentMarkerSets[i], - } - } - - maxDepth := calculateMaxDepth(spentVtxos) - newDepth := maxDepth + 1 - require.Equal(t, tc.expectedDepth, newDepth) - - // Should be at a marker boundary - require.True(t, domain.IsAtMarkerBoundary(newDepth), - "depth %d should be at marker boundary for this test", newDepth) - - parentMarkers := collectParentMarkers(spentVtxos) - markerIDs, createdMarker := deriveMarkerIDs(newDepth, parentMarkers, "test-txid") - - // A new marker should be created - require.NotNil(t, createdMarker, tc.description) - require.Equal(t, newDepth, createdMarker.Depth) - // The new marker's ParentMarkerIDs should match collected parent markers - sort.Strings(createdMarker.ParentMarkerIDs) - sort.Strings(parentMarkers) - require.Equal(t, parentMarkers, createdMarker.ParentMarkerIDs) - // The child VTXO should get ONLY the new marker ID, not parent markers - require.Len(t, markerIDs, 1) - require.Equal(t, createdMarker.ID, markerIDs[0]) - }) - } -} - -// TestAllNewVtxosGetSameDepth verifies that when a single offchain tx produces -// multiple output VTXOs, all of them receive the same depth (max parent depth + 1) -// and the same marker IDs. This mirrors the logic in updateProjectionsAfterOffchainTxEvents -// where newDepth is computed once and applied to all new VTXOs from the same tx. -func TestAllNewVtxosGetSameDepth(t *testing.T) { - testCases := []struct { - name string - parentDepths []uint32 - parentMarkerSets [][]string - numOutputVtxos int - expectedDepth uint32 - expectedMarkerLen int - description string - }{ - { - name: "3 outputs from single parent at depth 0", - parentDepths: []uint32{0}, - parentMarkerSets: [][]string{{"root-marker-1"}}, - numOutputVtxos: 3, - expectedDepth: 1, - expectedMarkerLen: 1, - description: "all 3 outputs get depth 1 and inherit root marker", - }, - { - name: "5 outputs from two parents at different depths", - parentDepths: []uint32{30, 50}, - parentMarkerSets: [][]string{{"marker-A"}, {"marker-B", "marker-C"}}, - numOutputVtxos: 5, - expectedDepth: 51, - expectedMarkerLen: 3, - description: "all 5 outputs get depth 51 (max+1) and inherit union of markers", - }, - { - name: "2 outputs at marker boundary", - parentDepths: []uint32{99}, - parentMarkerSets: [][]string{{"root-marker"}}, - numOutputVtxos: 2, - expectedDepth: 100, - expectedMarkerLen: 1, - description: "both outputs get depth 100 and the same new marker", - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - spentVtxos := make([]domain.Vtxo, len(tc.parentDepths)) - for i, depth := range tc.parentDepths { - spentVtxos[i] = domain.Vtxo{ - Depth: depth, - MarkerIDs: tc.parentMarkerSets[i], - } - } - - maxDepth := calculateMaxDepth(spentVtxos) - newDepth := maxDepth + 1 - require.Equal(t, tc.expectedDepth, newDepth) - - parentMarkers := collectParentMarkers(spentVtxos) - markerIDs, _ := deriveMarkerIDs(newDepth, parentMarkers, "tx-with-multiple-outputs") - - // Simulate creating multiple output VTXOs — each gets the same depth and markers - outputs := make([]domain.Vtxo, tc.numOutputVtxos) - for i := 0; i < tc.numOutputVtxos; i++ { - outputs[i] = domain.Vtxo{ - Outpoint: domain.Outpoint{Txid: "tx-with-multiple-outputs", VOut: uint32(i)}, - Depth: newDepth, - MarkerIDs: markerIDs, - } - } - - // All outputs must have the same depth - for i, v := range outputs { - require.Equal(t, tc.expectedDepth, v.Depth, - "output %d has wrong depth", i) - } - - // All outputs must have the same marker IDs - for i := 1; i < len(outputs); i++ { - sort.Strings(outputs[0].MarkerIDs) - sort.Strings(outputs[i].MarkerIDs) - require.Equal(t, outputs[0].MarkerIDs, outputs[i].MarkerIDs, - "output %d has different markers than output 0", i) - } - - require.Len(t, outputs[0].MarkerIDs, tc.expectedMarkerLen, tc.description) - }) - } -} - -// TestDepth20k_MarkerBoundaryAndInheritance verifies marker behavior at the -// target maximum depth of 20000. Tests boundary transitions, inheritance with -// large marker sets, and depth calculation with deeply chained VTXOs. -func TestDepth20k_MarkerBoundaryAndInheritance(t *testing.T) { - t.Run("depth 19999 inherits markers, depth 20000 creates new marker", func(t *testing.T) { - // Parent at depth 19999 => child at 20000 (boundary) - parent := domain.Vtxo{Depth: 19999, MarkerIDs: []string{"marker-19900"}} - parentMarkers := collectParentMarkers([]domain.Vtxo{parent}) - - newDepth := calculateMaxDepth([]domain.Vtxo{parent}) + 1 - require.Equal(t, uint32(20000), newDepth) - require.True(t, domain.IsAtMarkerBoundary(newDepth)) - - markerIDs, createdMarker := deriveMarkerIDs(newDepth, parentMarkers, "tx-at-20k") - require.NotNil(t, createdMarker, "marker should be created at depth 20000") - require.Equal(t, uint32(20000), createdMarker.Depth) - require.Equal(t, []string{"marker-19900"}, createdMarker.ParentMarkerIDs) - require.Len(t, markerIDs, 1) - require.Equal(t, createdMarker.ID, markerIDs[0]) - }) - - t.Run("depth 20001 inherits markers from boundary parent", func(t *testing.T) { - parent := domain.Vtxo{Depth: 20000, MarkerIDs: []string{"marker-20000"}} - parentMarkers := collectParentMarkers([]domain.Vtxo{parent}) - - newDepth := calculateMaxDepth([]domain.Vtxo{parent}) + 1 - require.Equal(t, uint32(20001), newDepth) - require.False(t, domain.IsAtMarkerBoundary(newDepth)) - - markerIDs, createdMarker := deriveMarkerIDs(newDepth, parentMarkers, "tx-at-20001") - require.Nil(t, createdMarker, "no marker at non-boundary depth") - require.Equal(t, []string{"marker-20000"}, markerIDs) - }) - - t.Run("VTXO with 200 inherited markers from deep chain", func(t *testing.T) { - // Simulate a VTXO at depth 19950 that has accumulated 200 marker IDs - // from a chain where markers were created at every boundary - markers := make([]string, 200) - for i := range markers { - markers[i] = fmt.Sprintf("marker-%d", i*100) - } - - parent := domain.Vtxo{Depth: 19950, MarkerIDs: markers} - collected := collectParentMarkers([]domain.Vtxo{parent}) - sort.Strings(collected) - sort.Strings(markers) - require.Equal(t, markers, collected, "all 200 markers should be collected") - }) - - t.Run("multiple deep parents merge 200+ markers correctly", func(t *testing.T) { - // Two parents deep in the chain with overlapping markers - markersA := make([]string, 100) - markersB := make([]string, 150) - for i := range markersA { - markersA[i] = fmt.Sprintf("marker-%d", i*100) // 0, 100, ..., 9900 - } - for i := range markersB { - markersB[i] = fmt.Sprintf("marker-%d", i*100) // 0, 100, ..., 14900 - } - - parents := []domain.Vtxo{ - {Depth: 10000, MarkerIDs: markersA}, - {Depth: 15000, MarkerIDs: markersB}, - } - collected := collectParentMarkers(parents) - - // Union should be 150 unique markers (0..14900) - require.Len(t, collected, 150) - - newDepth := calculateMaxDepth(parents) + 1 - require.Equal(t, uint32(15001), newDepth) - require.False(t, domain.IsAtMarkerBoundary(newDepth)) - - markerIDs, createdMarker := deriveMarkerIDs(newDepth, collected, "merge-tx") - require.Nil(t, createdMarker) - require.Len(t, markerIDs, 150, "child inherits all 150 unique markers") - }) - - t.Run("depth beyond 20k target remains valid", func(t *testing.T) { - // Verify depth arithmetic works correctly beyond the 20k boundary - parent := domain.Vtxo{Depth: 20000, MarkerIDs: []string{"marker-20000"}} - newDepth := calculateMaxDepth([]domain.Vtxo{parent}) + 1 - require.Equal(t, uint32(20001), newDepth) - - // Depth 20100 should also be a boundary - require.True(t, domain.IsAtMarkerBoundary(20100)) - require.True(t, domain.IsAtMarkerBoundary(20200)) - require.False(t, domain.IsAtMarkerBoundary(20001)) - require.False(t, domain.IsAtMarkerBoundary(20099)) - }) -} diff --git a/internal/core/domain/marker.go b/internal/core/domain/marker.go index 9a540523a..5d70be1ab 100644 --- a/internal/core/domain/marker.go +++ b/internal/core/domain/marker.go @@ -1,5 +1,7 @@ package domain +import "fmt" + // MarkerInterval is the depth interval at which markers are created. // VTXOs at depth 0, 100, 200, etc. create new markers. const MarkerInterval = 100 @@ -16,11 +18,31 @@ type Marker struct { ParentMarkerIDs []string } -// IsAtMarkerBoundary returns true if the given depth is at a marker boundary. -func IsAtMarkerBoundary(depth uint32) bool { +// isAtMarkerBoundary returns true if the given depth is at a marker boundary. +func isAtMarkerBoundary(depth uint32) bool { return depth%MarkerInterval == 0 } +// NewMarker computes marker information for a new offchain transaction. +// If the depth is at a marker boundary, it returns a new Marker and the marker IDs +// to assign to the child VTXOs (just the new marker ID). +// Otherwise, it returns nil and the inherited parent marker IDs. +func NewMarker(txid string, depth uint32, parentMarkerIDs []string) (*Marker, []string) { + if isAtMarkerBoundary(depth) { + id := fmt.Sprintf("%s:marker:%d", txid, depth) + marker := &Marker{ + ID: id, + Depth: depth, + ParentMarkerIDs: parentMarkerIDs, + } + return marker, []string{id} + } + if len(parentMarkerIDs) > 0 { + return nil, parentMarkerIDs + } + return nil, nil +} + // SweptMarker records when a marker (and all VTXOs it covers) was swept. // This is an append-only table that enables efficient bulk sweep operations. type SweptMarker struct { diff --git a/internal/core/domain/marker_test.go b/internal/core/domain/marker_test.go index 2834e4938..57873b0ab 100644 --- a/internal/core/domain/marker_test.go +++ b/internal/core/domain/marker_test.go @@ -1,6 +1,8 @@ package domain import ( + "fmt" + "sort" "testing" "github.com/stretchr/testify/require" @@ -28,19 +30,17 @@ func TestIsAtMarkerBoundary(t *testing.T) { } for _, tt := range tests { - result := IsAtMarkerBoundary(tt.depth) + result := isAtMarkerBoundary(tt.depth) require.Equal(t, tt.expected, result, - "IsAtMarkerBoundary(%d) should be %v", tt.depth, tt.expected) + "isAtMarkerBoundary(%d) should be %v", tt.depth, tt.expected) } } func TestMarkerInterval(t *testing.T) { - // Verify the constant is set correctly require.Equal(t, uint32(100), uint32(MarkerInterval)) } func TestMarkerStruct(t *testing.T) { - // Test Marker struct creation marker := Marker{ ID: "test-marker-id", Depth: 100, @@ -55,7 +55,6 @@ func TestMarkerStruct(t *testing.T) { } func TestSweptMarkerStruct(t *testing.T) { - // Test SweptMarker struct creation sweptMarker := SweptMarker{ MarkerID: "swept-marker-id", SweptAt: 1234567890, @@ -66,13 +65,597 @@ func TestSweptMarkerStruct(t *testing.T) { } func TestRootMarkerHasNoParents(t *testing.T) { - // Root markers (depth 0) should have no parent markers rootMarker := Marker{ ID: "root-marker", Depth: 0, ParentMarkerIDs: nil, } - require.True(t, IsAtMarkerBoundary(rootMarker.Depth)) + require.True(t, isAtMarkerBoundary(rootMarker.Depth)) require.Nil(t, rootMarker.ParentMarkerIDs) } + +func TestNewMarker(t *testing.T) { + t.Run("at boundary creates marker", func(t *testing.T) { + parentIDs := []string{"parent-A", "parent-B"} + marker, markerIDs := NewMarker("txid123", 100, parentIDs) + + require.NotNil(t, marker) + require.Equal(t, "txid123:marker:100", marker.ID) + require.Equal(t, uint32(100), marker.Depth) + require.Equal(t, parentIDs, marker.ParentMarkerIDs) + require.Equal(t, []string{"txid123:marker:100"}, markerIDs) + }) + + t.Run("at depth 0 creates root marker", func(t *testing.T) { + marker, markerIDs := NewMarker("txid-root", 0, nil) + + require.NotNil(t, marker) + require.Equal(t, "txid-root:marker:0", marker.ID) + require.Equal(t, uint32(0), marker.Depth) + require.Nil(t, marker.ParentMarkerIDs) + require.Equal(t, []string{"txid-root:marker:0"}, markerIDs) + }) + + t.Run("non-boundary inherits parent markers", func(t *testing.T) { + parentIDs := []string{"marker-A", "marker-B"} + marker, markerIDs := NewMarker("txid456", 51, parentIDs) + + require.Nil(t, marker) + require.Equal(t, parentIDs, markerIDs) + }) + + t.Run("non-boundary no parents returns nil", func(t *testing.T) { + marker, markerIDs := NewMarker("txid789", 5, nil) + + require.Nil(t, marker) + require.Nil(t, markerIDs) + }) + + t.Run("at depth 200 with parents", func(t *testing.T) { + parentIDs := []string{"marker-100"} + marker, markerIDs := NewMarker("deep-tx", 200, parentIDs) + + require.NotNil(t, marker) + require.Equal(t, "deep-tx:marker:200", marker.ID) + require.Equal(t, uint32(200), marker.Depth) + require.Equal(t, parentIDs, marker.ParentMarkerIDs) + require.Len(t, markerIDs, 1) + require.Equal(t, marker.ID, markerIDs[0]) + }) +} + +// calculateMaxDepth returns the maximum depth from a set of spent VTXOs. +func calculateMaxDepth(spentVtxos []Vtxo) uint32 { + var maxDepth uint32 + for _, v := range spentVtxos { + if v.Depth > maxDepth { + maxDepth = v.Depth + } + } + return maxDepth +} + +// collectParentMarkers collects all unique, non-empty marker IDs from spent VTXOs. +func collectParentMarkers(spentVtxos []Vtxo) []string { + parentMarkerSet := make(map[string]struct{}) + for _, v := range spentVtxos { + for _, markerID := range v.MarkerIDs { + if markerID != "" { + parentMarkerSet[markerID] = struct{}{} + } + } + } + result := make([]string, 0, len(parentMarkerSet)) + for id := range parentMarkerSet { + result = append(result, id) + } + return result +} + +func TestDepthCalculation(t *testing.T) { + testCases := []struct { + name string + spentVtxos []Vtxo + expectedDepth uint32 + description string + }{ + { + name: "single batch vtxo at depth 0", + spentVtxos: []Vtxo{{Depth: 0}}, + expectedDepth: 1, + description: "spending a batch vtxo creates vtxo at depth 1", + }, + { + name: "single vtxo at depth 50", + spentVtxos: []Vtxo{{Depth: 50}}, + expectedDepth: 51, + description: "spending a chained vtxo increments depth", + }, + { + name: "multiple vtxos with same depth", + spentVtxos: []Vtxo{ + {Depth: 10}, + {Depth: 10}, + {Depth: 10}, + }, + expectedDepth: 11, + description: "combining vtxos at same depth increments once", + }, + { + name: "multiple vtxos with different depths", + spentVtxos: []Vtxo{ + {Depth: 5}, + {Depth: 25}, + {Depth: 15}, + }, + expectedDepth: 26, + description: "uses max depth from inputs", + }, + { + name: "vtxos spanning marker boundary", + spentVtxos: []Vtxo{ + {Depth: 95}, + {Depth: 105}, + }, + expectedDepth: 106, + description: "handles depths across marker boundaries", + }, + { + name: "deep chain near marker boundary", + spentVtxos: []Vtxo{ + {Depth: 99}, + }, + expectedDepth: 100, + description: "result at marker boundary (100)", + }, + { + name: "very deep chain", + spentVtxos: []Vtxo{ + {Depth: 500}, + }, + expectedDepth: 501, + description: "handles deep chains beyond multiple marker intervals", + }, + { + name: "no spent vtxos (empty)", + spentVtxos: []Vtxo{}, + expectedDepth: 0, + description: "empty input results in depth 0 (no spent vtxos means maxDepth stays 0, newDepth = 0)", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + maxDepth := calculateMaxDepth(tc.spentVtxos) + var newDepth uint32 + if len(tc.spentVtxos) > 0 { + newDepth = maxDepth + 1 + } + require.Equal(t, tc.expectedDepth, newDepth, tc.description) + }) + } +} + +func TestDepthAtMarkerBoundary(t *testing.T) { + testCases := []struct { + depth uint32 + isAtBoundary bool + description string + }{ + {0, true, "depth 0 is at marker boundary"}, + {1, false, "depth 1 is not at boundary"}, + {50, false, "depth 50 is not at boundary"}, + {99, false, "depth 99 is not at boundary"}, + {100, true, "depth 100 is at marker boundary"}, + {101, false, "depth 101 is not at boundary"}, + {200, true, "depth 200 is at marker boundary"}, + {500, true, "depth 500 is at marker boundary"}, + {1000, true, "depth 1000 is at marker boundary"}, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + require.Equal(t, tc.isAtBoundary, isAtMarkerBoundary(tc.depth)) + }) + } +} + +func TestDepthIncrementCreatesMarkerAtBoundary(t *testing.T) { + testCases := []struct { + parentDepth uint32 + newDepth uint32 + shouldCreateMarker bool + }{ + {99, 100, true}, // crossing into boundary + {100, 101, false}, // leaving boundary + {199, 200, true}, // crossing into next boundary + {0, 1, false}, // moving away from initial boundary + {98, 99, false}, // approaching but not at boundary + } + + for _, tc := range testCases { + t.Run("", func(t *testing.T) { + spentVtxos := []Vtxo{{Depth: tc.parentDepth}} + maxDepth := calculateMaxDepth(spentVtxos) + newDepth := maxDepth + 1 + + require.Equal(t, tc.newDepth, newDepth) + marker, _ := NewMarker("test-txid", newDepth, nil) + require.Equal(t, tc.shouldCreateMarker, marker != nil) + }) + } +} + +func TestParentMarkerCollectionFromMultipleParents(t *testing.T) { + testCases := []struct { + name string + spentVtxos []Vtxo + expectedMarkers []string + }{ + { + name: "single parent with one marker", + spentVtxos: []Vtxo{ + {MarkerIDs: []string{"marker-A"}}, + }, + expectedMarkers: []string{"marker-A"}, + }, + { + name: "two parents with distinct markers", + spentVtxos: []Vtxo{ + {MarkerIDs: []string{"marker-A"}}, + {MarkerIDs: []string{"marker-B"}}, + }, + expectedMarkers: []string{"marker-A", "marker-B"}, + }, + { + name: "three parents with overlapping markers", + spentVtxos: []Vtxo{ + {MarkerIDs: []string{"marker-A", "marker-B"}}, + {MarkerIDs: []string{"marker-B", "marker-C"}}, + {MarkerIDs: []string{"marker-A", "marker-C"}}, + }, + expectedMarkers: []string{"marker-A", "marker-B", "marker-C"}, + }, + { + name: "all parents share the same marker", + spentVtxos: []Vtxo{ + {MarkerIDs: []string{"root-marker"}}, + {MarkerIDs: []string{"root-marker"}}, + {MarkerIDs: []string{"root-marker"}}, + }, + expectedMarkers: []string{"root-marker"}, + }, + { + name: "no parents", + spentVtxos: []Vtxo{}, + expectedMarkers: []string{}, + }, + { + name: "parent with no markers", + spentVtxos: []Vtxo{ + {MarkerIDs: []string{}}, + }, + expectedMarkers: []string{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := collectParentMarkers(tc.spentVtxos) + sort.Strings(result) + sort.Strings(tc.expectedMarkers) + require.Equal(t, tc.expectedMarkers, result) + }) + } +} + +func TestParentMarkerCollectionSkipsEmptyMarkerIDs(t *testing.T) { + spentVtxos := []Vtxo{ + {MarkerIDs: []string{"marker-A", "", "marker-B"}}, + {MarkerIDs: []string{"", ""}}, + {MarkerIDs: []string{"marker-C", ""}}, + } + + result := collectParentMarkers(spentVtxos) + sort.Strings(result) + require.Equal(t, []string{"marker-A", "marker-B", "marker-C"}, result) +} + +func TestMarkerInheritanceAtNonBoundary(t *testing.T) { + testCases := []struct { + name string + parentDepths []uint32 + parentMarkerSets [][]string + expectedDepth uint32 + expectedMarkers []string + description string + }{ + { + name: "single parent at depth 0, child at depth 1", + parentDepths: []uint32{0}, + parentMarkerSets: [][]string{{"root-marker-1"}}, + expectedDepth: 1, + expectedMarkers: []string{"root-marker-1"}, + description: "child inherits single parent marker", + }, + { + name: "single parent at depth 50, child at depth 51", + parentDepths: []uint32{50}, + parentMarkerSets: [][]string{{"marker-A", "marker-B"}}, + expectedDepth: 51, + expectedMarkers: []string{"marker-A", "marker-B"}, + description: "child inherits multiple parent markers", + }, + { + name: "two parents at different depths, child not at boundary", + parentDepths: []uint32{30, 40}, + parentMarkerSets: [][]string{{"marker-X"}, {"marker-Y"}}, + expectedDepth: 41, + expectedMarkers: []string{"marker-X", "marker-Y"}, + description: "child inherits union of all parent markers", + }, + { + name: "three parents with overlapping markers", + parentDepths: []uint32{10, 20, 15}, + parentMarkerSets: [][]string{{"m1", "m2"}, {"m2", "m3"}, {"m1"}}, + expectedDepth: 21, + expectedMarkers: []string{"m1", "m2", "m3"}, + description: "child inherits deduplicated union", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + spentVtxos := make([]Vtxo, len(tc.parentDepths)) + for i, depth := range tc.parentDepths { + spentVtxos[i] = Vtxo{ + Depth: depth, + MarkerIDs: tc.parentMarkerSets[i], + } + } + + maxDepth := calculateMaxDepth(spentVtxos) + newDepth := maxDepth + 1 + require.Equal(t, tc.expectedDepth, newDepth) + + require.False(t, isAtMarkerBoundary(newDepth), + "depth %d should not be at marker boundary for this test", newDepth) + + parentMarkers := collectParentMarkers(spentVtxos) + marker, markerIDs := NewMarker("some-txid", newDepth, parentMarkers) + + require.Nil(t, marker, tc.description) + sort.Strings(markerIDs) + sort.Strings(tc.expectedMarkers) + require.Equal(t, tc.expectedMarkers, markerIDs, tc.description) + }) + } +} + +func TestMarkerCreationAtBoundary(t *testing.T) { + testCases := []struct { + name string + parentDepths []uint32 + parentMarkerSets [][]string + expectedDepth uint32 + description string + }{ + { + name: "parent at depth 99, child at depth 100", + parentDepths: []uint32{99}, + parentMarkerSets: [][]string{{"root-marker"}}, + expectedDepth: 100, + description: "first non-root boundary", + }, + { + name: "parent at depth 199, child at depth 200", + parentDepths: []uint32{199}, + parentMarkerSets: [][]string{{"marker-100", "root-marker"}}, + expectedDepth: 200, + description: "second boundary with two parent markers", + }, + { + name: "multiple parents converging at boundary", + parentDepths: []uint32{95, 99}, + parentMarkerSets: [][]string{{"marker-A"}, {"marker-B"}}, + expectedDepth: 100, + description: "boundary with multiple parent VTXOs", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + spentVtxos := make([]Vtxo, len(tc.parentDepths)) + for i, depth := range tc.parentDepths { + spentVtxos[i] = Vtxo{ + Depth: depth, + MarkerIDs: tc.parentMarkerSets[i], + } + } + + maxDepth := calculateMaxDepth(spentVtxos) + newDepth := maxDepth + 1 + require.Equal(t, tc.expectedDepth, newDepth) + + require.True(t, isAtMarkerBoundary(newDepth), + "depth %d should be at marker boundary for this test", newDepth) + + parentMarkers := collectParentMarkers(spentVtxos) + createdMarker, markerIDs := NewMarker("test-txid", newDepth, parentMarkers) + + require.NotNil(t, createdMarker, tc.description) + require.Equal(t, newDepth, createdMarker.Depth) + sort.Strings(createdMarker.ParentMarkerIDs) + sort.Strings(parentMarkers) + require.Equal(t, parentMarkers, createdMarker.ParentMarkerIDs) + require.Len(t, markerIDs, 1) + require.Equal(t, createdMarker.ID, markerIDs[0]) + }) + } +} + +func TestAllNewVtxosGetSameDepth(t *testing.T) { + testCases := []struct { + name string + parentDepths []uint32 + parentMarkerSets [][]string + numOutputVtxos int + expectedDepth uint32 + expectedMarkerLen int + description string + }{ + { + name: "3 outputs from single parent at depth 0", + parentDepths: []uint32{0}, + parentMarkerSets: [][]string{{"root-marker-1"}}, + numOutputVtxos: 3, + expectedDepth: 1, + expectedMarkerLen: 1, + description: "all 3 outputs get depth 1 and inherit root marker", + }, + { + name: "5 outputs from two parents at different depths", + parentDepths: []uint32{30, 50}, + parentMarkerSets: [][]string{{"marker-A"}, {"marker-B", "marker-C"}}, + numOutputVtxos: 5, + expectedDepth: 51, + expectedMarkerLen: 3, + description: "all 5 outputs get depth 51 (max+1) and inherit union of markers", + }, + { + name: "2 outputs at marker boundary", + parentDepths: []uint32{99}, + parentMarkerSets: [][]string{{"root-marker"}}, + numOutputVtxos: 2, + expectedDepth: 100, + expectedMarkerLen: 1, + description: "both outputs get depth 100 and the same new marker", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + spentVtxos := make([]Vtxo, len(tc.parentDepths)) + for i, depth := range tc.parentDepths { + spentVtxos[i] = Vtxo{ + Depth: depth, + MarkerIDs: tc.parentMarkerSets[i], + } + } + + maxDepth := calculateMaxDepth(spentVtxos) + newDepth := maxDepth + 1 + require.Equal(t, tc.expectedDepth, newDepth) + + parentMarkers := collectParentMarkers(spentVtxos) + _, markerIDs := NewMarker("tx-with-multiple-outputs", newDepth, parentMarkers) + + // Simulate creating multiple output VTXOs — each gets the same depth and markers + outputs := make([]Vtxo, tc.numOutputVtxos) + for i := 0; i < tc.numOutputVtxos; i++ { + outputs[i] = Vtxo{ + Outpoint: Outpoint{Txid: "tx-with-multiple-outputs", VOut: uint32(i)}, + Depth: newDepth, + MarkerIDs: markerIDs, + } + } + + for i, v := range outputs { + require.Equal(t, tc.expectedDepth, v.Depth, + "output %d has wrong depth", i) + } + + for i := 1; i < len(outputs); i++ { + sort.Strings(outputs[0].MarkerIDs) + sort.Strings(outputs[i].MarkerIDs) + require.Equal(t, outputs[0].MarkerIDs, outputs[i].MarkerIDs, + "output %d has different markers than output 0", i) + } + + require.Len(t, outputs[0].MarkerIDs, tc.expectedMarkerLen, tc.description) + }) + } +} + +func TestDepth20k_MarkerBoundaryAndInheritance(t *testing.T) { + t.Run("depth 19999 inherits markers, depth 20000 creates new marker", func(t *testing.T) { + parent := Vtxo{Depth: 19999, MarkerIDs: []string{"marker-19900"}} + parentMarkers := collectParentMarkers([]Vtxo{parent}) + + newDepth := calculateMaxDepth([]Vtxo{parent}) + 1 + require.Equal(t, uint32(20000), newDepth) + require.True(t, isAtMarkerBoundary(newDepth)) + + createdMarker, markerIDs := NewMarker("tx-at-20k", newDepth, parentMarkers) + require.NotNil(t, createdMarker, "marker should be created at depth 20000") + require.Equal(t, uint32(20000), createdMarker.Depth) + require.Equal(t, []string{"marker-19900"}, createdMarker.ParentMarkerIDs) + require.Len(t, markerIDs, 1) + require.Equal(t, createdMarker.ID, markerIDs[0]) + }) + + t.Run("depth 20001 inherits markers from boundary parent", func(t *testing.T) { + parent := Vtxo{Depth: 20000, MarkerIDs: []string{"marker-20000"}} + parentMarkers := collectParentMarkers([]Vtxo{parent}) + + newDepth := calculateMaxDepth([]Vtxo{parent}) + 1 + require.Equal(t, uint32(20001), newDepth) + require.False(t, isAtMarkerBoundary(newDepth)) + + createdMarker, markerIDs := NewMarker("tx-at-20001", newDepth, parentMarkers) + require.Nil(t, createdMarker, "no marker at non-boundary depth") + require.Equal(t, []string{"marker-20000"}, markerIDs) + }) + + t.Run("VTXO with 200 inherited markers from deep chain", func(t *testing.T) { + markers := make([]string, 200) + for i := range markers { + markers[i] = fmt.Sprintf("marker-%d", i*100) + } + + parent := Vtxo{Depth: 19950, MarkerIDs: markers} + collected := collectParentMarkers([]Vtxo{parent}) + sort.Strings(collected) + sort.Strings(markers) + require.Equal(t, markers, collected, "all 200 markers should be collected") + }) + + t.Run("multiple deep parents merge 200+ markers correctly", func(t *testing.T) { + markersA := make([]string, 100) + markersB := make([]string, 150) + for i := range markersA { + markersA[i] = fmt.Sprintf("marker-%d", i*100) + } + for i := range markersB { + markersB[i] = fmt.Sprintf("marker-%d", i*100) + } + + parents := []Vtxo{ + {Depth: 10000, MarkerIDs: markersA}, + {Depth: 15000, MarkerIDs: markersB}, + } + collected := collectParentMarkers(parents) + + require.Len(t, collected, 150) + + newDepth := calculateMaxDepth(parents) + 1 + require.Equal(t, uint32(15001), newDepth) + require.False(t, isAtMarkerBoundary(newDepth)) + + createdMarker, markerIDs := NewMarker("merge-tx", newDepth, collected) + require.Nil(t, createdMarker) + require.Len(t, markerIDs, 150, "child inherits all 150 unique markers") + }) + + t.Run("depth beyond 20k target remains valid", func(t *testing.T) { + parent := Vtxo{Depth: 20000, MarkerIDs: []string{"marker-20000"}} + newDepth := calculateMaxDepth([]Vtxo{parent}) + 1 + require.Equal(t, uint32(20001), newDepth) + + require.True(t, isAtMarkerBoundary(20100)) + require.True(t, isAtMarkerBoundary(20200)) + require.False(t, isAtMarkerBoundary(20001)) + require.False(t, isAtMarkerBoundary(20099)) + }) +} diff --git a/internal/core/domain/offchain_tx.go b/internal/core/domain/offchain_tx.go index 37da1b13d..79e76c4c7 100644 --- a/internal/core/domain/offchain_tx.go +++ b/internal/core/domain/offchain_tx.go @@ -42,6 +42,8 @@ type OffchainTx struct { CommitmentTxids map[string]string RootCommitmentTxId string ExpiryTimestamp int64 + Depth uint32 + ParentMarkerIDs []string FailReason string Version uint changes []Event @@ -97,6 +99,7 @@ func (s *OffchainTx) Request( func (s *OffchainTx) Accept( finalArkTx string, signedCheckpointTxs map[string]string, commitmentTxsByCheckpointTxid map[string]string, rootCommitmentTx string, expiryTimestamp int64, + depth uint32, parentMarkerIDs []string, ) (Event, error) { if finalArkTx == "" { return nil, fmt.Errorf("missing final ark tx") @@ -132,6 +135,8 @@ func (s *OffchainTx) Accept( CommitmentTxids: commitmentTxsByCheckpointTxid, RootCommitmentTxid: rootCommitmentTx, ExpiryTimestamp: expiryTimestamp, + Depth: depth, + ParentMarkerIDs: parentMarkerIDs, } s.raise(event) return event, nil @@ -245,6 +250,8 @@ func (s *OffchainTx) on(event Event, replayed bool) { s.CommitmentTxids = e.CommitmentTxids s.RootCommitmentTxId = e.RootCommitmentTxid s.ExpiryTimestamp = e.ExpiryTimestamp + s.Depth = e.Depth + s.ParentMarkerIDs = e.ParentMarkerIDs case OffchainTxFinalized: if s.Stage.Code != int(OffchainTxAcceptedStage) { return diff --git a/internal/core/domain/offchain_tx_event.go b/internal/core/domain/offchain_tx_event.go index b49dea04c..b057288c4 100644 --- a/internal/core/domain/offchain_tx_event.go +++ b/internal/core/domain/offchain_tx_event.go @@ -24,6 +24,8 @@ type OffchainTxAccepted struct { FinalArkTx string SignedCheckpointTxs map[string]string ExpiryTimestamp int64 + Depth uint32 + ParentMarkerIDs []string } type OffchainTxFinalized struct { diff --git a/internal/core/domain/offchain_tx_test.go b/internal/core/domain/offchain_tx_test.go index 86736a2f3..e7c129692 100644 --- a/internal/core/domain/offchain_tx_test.go +++ b/internal/core/domain/offchain_tx_test.go @@ -144,6 +144,7 @@ func testAcceptOffchainTx(t *testing.T) { commitmentTxsByCheckpointTxid, rootCommitmentTxid, expiryTimestamp, + 1, []string{"parent-marker"}, ) require.NoError(t, err) require.NotNil(t, event) @@ -156,6 +157,8 @@ func testAcceptOffchainTx(t *testing.T) { require.Equal(t, signedCheckpointTxs, offchainTx.CheckpointTxs) require.Equal(t, commitmentTxsByCheckpointTxid, offchainTx.CommitmentTxids) require.Equal(t, rootCommitmentTxid, offchainTx.RootCommitmentTxId) + require.Equal(t, uint32(1), offchainTx.Depth) + require.Equal(t, []string{"parent-marker"}, offchainTx.ParentMarkerIDs) events := offchainTx.Events() require.Len(t, events, 2) @@ -251,6 +254,7 @@ func testAcceptOffchainTx(t *testing.T) { event, err := f.offchainTx.Accept( f.finalArkTx, f.signedCheckpointTxs, f.commitmentTxids, rootCommitmentTxid, f.expiryTimestamp, + 0, nil, ) require.EqualError(t, err, f.expectedErr) require.Nil(t, event) @@ -270,6 +274,7 @@ func testFinalizeOffchainTx(t *testing.T) { event, err = offchainTx.Accept( finalArkTx, signedCheckpointTxs, commitmentTxsByCheckpointTxid, rootCommitmentTxid, expiryTimestamp, + 0, nil, ) require.NoError(t, err) require.NotNil(t, event) @@ -349,6 +354,7 @@ func testFailOffchainTx(t *testing.T) { event, err = offchainTx.Accept( finalArkTx, signedCheckpointTxs, commitmentTxsByCheckpointTxid, rootCommitmentTxid, expiryTimestamp, + 0, nil, ) require.NoError(t, err) require.NotNil(t, event) diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index e9d7392f3..8fc6f81ca 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -581,73 +581,24 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) return } - // Get spent VTXO outpoints from checkpoint txs to calculate depth - spentOutpoints := make([]domain.Outpoint, 0) - for _, tx := range offchainTx.CheckpointTxs { - _, ins, _, err := s.txDecoder.DecodeTx(tx) - if err != nil { - log.WithError(err).Warn("failed to decode checkpoint tx for depth calculation") - continue - } - spentOutpoints = append(spentOutpoints, ins...) - } - - // Get spent VTXOs to calculate new depth - var newDepth uint32 - var parentMarkerIDs []string - depthKnown := true - if len(spentOutpoints) > 0 { - spentVtxos, err := s.vtxoStore.GetVtxos(ctx, spentOutpoints) - if err != nil { - log.WithError(err). - Warn("failed to get spent vtxos for depth calculation, skipping marker creation") - // Continue with depth 0 but mark as unknown to avoid creating misleading root markers - depthKnown = false - } else { - // Calculate depth: max(parent depths) + 1 - var maxDepth uint32 - parentMarkerSet := make(map[string]struct{}) - for _, v := range spentVtxos { - if v.Depth > maxDepth { - maxDepth = v.Depth - } - // Collect ALL parent marker IDs for marker linking - for _, markerID := range v.MarkerIDs { - if markerID != "" { - parentMarkerSet[markerID] = struct{}{} - } - } - } - newDepth = maxDepth + 1 - // Convert parent marker set to slice - for id := range parentMarkerSet { - parentMarkerIDs = append(parentMarkerIDs, id) - } - } - } + // Depth and parent marker IDs are carried by the OffchainTxAccepted event, + // computed in SubmitOffchainTx from the spent VTXOs. + newDepth := offchainTx.Depth + parentMarkerIDs := offchainTx.ParentMarkerIDs - // Create marker if at boundary depth, or inherit ALL parent markers - // Skip marker creation if depth is unknown (GetVtxos failed) to avoid misleading root markers + // Create marker if at boundary depth, or inherit parent markers var markerIDs []string - - if depthKnown && domain.IsAtMarkerBoundary(newDepth) { - // Create marker ID from the first output (the ark tx id + first vtxo vout) - newMarkerID := fmt.Sprintf("%s:marker:%d", txid, newDepth) - marker := domain.Marker{ - ID: newMarkerID, - Depth: newDepth, - ParentMarkerIDs: parentMarkerIDs, - } - if err := s.markerStore.AddMarker(ctx, marker); err != nil { + marker, ids := domain.NewMarker(txid, newDepth, parentMarkerIDs) + if marker != nil { + if err := s.markerStore.AddMarker(ctx, *marker); err != nil { log.WithError(err).Warn("failed to create marker for chained vtxo") // Continue without marker - non-fatal } else { - log.Debugf("created marker %s at depth %d", newMarkerID, newDepth) - markerIDs = []string{newMarkerID} + log.Debugf("created marker %s at depth %d", marker.ID, newDepth) + markerIDs = ids } - } else if len(parentMarkerIDs) > 0 { - // Inherit ALL markers from parents at non-boundary depth - markerIDs = parentMarkerIDs + } else { + markerIDs = ids } issuances, assets, err := getAssetsFromTxOuts(txid, outs) diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index 8f68d311d..5b23d0369 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -2803,7 +2803,7 @@ func testMarkerCreationAtBoundaryDepth(t *testing.T, svc ports.RepoManager) { // Simulate offchain tx: child at depth 100 (marker boundary) newDepth := uint32(100) - require.True(t, domain.IsAtMarkerBoundary(newDepth)) + require.True(t, newDepth%domain.MarkerInterval == 0) // Collect parent markers (mimics service logic) parentMarkerIDs := parentVtxo.MarkerIDs @@ -2896,7 +2896,7 @@ func testMarkerInheritanceAtNonBoundary(t *testing.T, svc ports.RepoManager) { // Child at depth 51 (NOT a boundary) should inherit both parent markers newDepth := uint32(51) - require.False(t, domain.IsAtMarkerBoundary(newDepth)) + require.False(t, newDepth%domain.MarkerInterval == 0) inheritedMarkers := []string{markerA, markerB} childVtxo := domain.Vtxo{ From fa0bf77706d45985608daeab047f69ae6615e23a Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Wed, 18 Mar 2026 13:18:22 -0400 Subject: [PATCH 34/54] GetReadyUpdate return type fix --- internal/core/application/sweeper_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/core/application/sweeper_test.go b/internal/core/application/sweeper_test.go index 8843d3ee7..2533d7c73 100644 --- a/internal/core/application/sweeper_test.go +++ b/internal/core/application/sweeper_test.go @@ -37,7 +37,7 @@ func (m *mockWalletService) GetTransaction(ctx context.Context, txid string) (st } // Stub implementations for unused WalletService methods -func (m *mockWalletService) GetReadyUpdate(ctx context.Context) (<-chan struct{}, error) { +func (m *mockWalletService) GetReadyUpdate(ctx context.Context) (<-chan bool, error) { return nil, nil } func (m *mockWalletService) GenSeed(ctx context.Context) (string, error) { return "", nil } From 3231064e7e910c18f420fd7cb846fb9a9a6b71dd Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 19 Mar 2026 11:26:02 -0400 Subject: [PATCH 35/54] log warning on marker window error --- internal/core/application/indexer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/core/application/indexer.go b/internal/core/application/indexer.go index cedb46095..51df11287 100644 --- a/internal/core/application/indexer.go +++ b/internal/core/application/indexer.go @@ -12,6 +12,7 @@ import ( "github.com/arkade-os/arkd/internal/core/ports" "github.com/arkade-os/arkd/pkg/ark-lib/tree" "github.com/btcsuite/btcd/btcutil/psbt" + log "github.com/sirupsen/logrus" ) const ( @@ -516,6 +517,7 @@ func (i *indexerService) ensureVtxosCached( windowVtxos, err := i.repoManager.Markers().GetVtxosByMarker(ctx, markerID) if err != nil { + log.WithError(err).Warnf("failed to load marker window %s, falling back to per-VTXO lookups", markerID) continue } for _, wv := range windowVtxos { From 86f9190cff77b094dd9f94a68fecd6f1ac4d64ab Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 19 Mar 2026 12:30:23 -0400 Subject: [PATCH 36/54] db migration to use milliseconds for swept_at --- .../migration/20260210100000_add_depth_and_markers.up.sql | 2 +- .../migration/20260210000000_add_depth_and_markers.up.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql index 9c618488b..044b06c62 100644 --- a/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql +++ b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql @@ -65,7 +65,7 @@ UPDATE vtxo SET markers = jsonb_build_array(txid || ':' || vout); INSERT INTO swept_marker (marker_id, swept_at) SELECT v.txid || ':' || v.vout, - EXTRACT(EPOCH FROM NOW())::BIGINT + (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT FROM vtxo v WHERE v.swept = true ON CONFLICT (marker_id) DO NOTHING; diff --git a/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql index 1376100a3..9f18c10e2 100644 --- a/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql +++ b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql @@ -63,7 +63,7 @@ UPDATE vtxo SET markers = '["' || txid || ':' || vout || '"]'; INSERT OR IGNORE INTO swept_marker (marker_id, swept_at) SELECT v.txid || ':' || v.vout, - strftime('%s', 'now') + strftime('%s', 'now') * 1000 FROM vtxo v WHERE v.swept = 1; From ce2ef6746c0cb81aecf3d655dcb43dc45e40a709 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 19 Mar 2026 12:32:58 -0400 Subject: [PATCH 37/54] linting --- internal/core/application/indexer.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/core/application/indexer.go b/internal/core/application/indexer.go index 51df11287..360954e22 100644 --- a/internal/core/application/indexer.go +++ b/internal/core/application/indexer.go @@ -517,7 +517,8 @@ func (i *indexerService) ensureVtxosCached( windowVtxos, err := i.repoManager.Markers().GetVtxosByMarker(ctx, markerID) if err != nil { - log.WithError(err).Warnf("failed to load marker window %s, falling back to per-VTXO lookups", markerID) + log.WithError(err). + Warnf("failed to load marker window %s, falling back to per-VTXO lookups", markerID) continue } for _, wv := range windowVtxos { From f13064c7b0a909e09e3a0d204f171034776a2a0a Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Fri, 20 Mar 2026 10:11:17 -0400 Subject: [PATCH 38/54] optimize GetVtxoChain with marker-based bulk preloading (#973) * optimize GetVtxoChain with marker-based bulk preloading * Test to prove reductin in calls to GetVtxos * sorting in tests to guarantee vtxo id ordering --- internal/core/application/indexer.go | 70 ++++++ internal/core/application/indexer_test.go | 288 +++++++++++++++++++++- 2 files changed, 357 insertions(+), 1 deletion(-) diff --git a/internal/core/application/indexer.go b/internal/core/application/indexer.go index 360954e22..e5127db5c 100644 --- a/internal/core/application/indexer.go +++ b/internal/core/application/indexer.go @@ -292,6 +292,17 @@ func (i *indexerService) GetVtxoChain( vtxoCache := make(map[string]domain.Vtxo) loadedMarkers := make(map[string]bool) + // Eagerly preload VTXOs by walking the marker DAG upward. + if i.repoManager.Markers() != nil { + startVtxos, err := i.repoManager.Vtxos().GetVtxos(ctx, nextVtxos) + if err != nil { + return nil, err + } + if err := i.preloadVtxosByMarkers(ctx, startVtxos, vtxoCache); err != nil { + return nil, err + } + } + for len(nextVtxos) > 0 { if err := i.ensureVtxosCached(ctx, nextVtxos, vtxoCache, loadedMarkers); err != nil { return nil, err @@ -475,6 +486,65 @@ func decodeChainCursor(token string) ([]domain.Outpoint, error) { return outpoints, nil } +// preloadVtxosByMarkers bulk-fetches VTXOs by walking the marker DAG upward +// from the markers of startVtxos. This reduces DB round-trips from O(chain_length) +// to O(chain_length / MarkerInterval). +func (i *indexerService) preloadVtxosByMarkers( + ctx context.Context, + startVtxos []domain.Vtxo, + cache map[string]domain.Vtxo, +) error { + markerRepo := i.repoManager.Markers() + + // Seed cache and collect initial marker IDs. + currentMarkerIDs := make(map[string]bool) + for _, v := range startVtxos { + cache[v.Outpoint.String()] = v + for _, mid := range v.MarkerIDs { + currentMarkerIDs[mid] = true + } + } + + visited := make(map[string]bool) + + for len(currentMarkerIDs) > 0 { + ids := make([]string, 0, len(currentMarkerIDs)) + for id := range currentMarkerIDs { + ids = append(ids, id) + visited[id] = true + } + + // Bulk-fetch all VTXOs tagged with these markers. + vtxos, err := markerRepo.GetVtxoChainByMarkers(ctx, ids) + if err != nil { + return err + } + for _, v := range vtxos { + if _, ok := cache[v.Outpoint.String()]; !ok { + cache[v.Outpoint.String()] = v + } + } + + // Get marker objects to find parent markers. + markers, err := markerRepo.GetMarkersByIds(ctx, ids) + if err != nil { + return err + } + + nextMarkerIDs := make(map[string]bool) + for _, m := range markers { + for _, pid := range m.ParentMarkerIDs { + if !visited[pid] { + nextMarkerIDs[pid] = true + } + } + } + currentMarkerIDs = nextMarkerIDs + } + + return nil +} + // ensureVtxosCached loads the given outpoints into the cache if not already present. // For each fetched VTXO, it also loads its marker window into the cache to prefetch // nearby VTXOs that will likely be needed in subsequent iterations. diff --git a/internal/core/application/indexer_test.go b/internal/core/application/indexer_test.go index 7dd539522..0ce75f648 100644 --- a/internal/core/application/indexer_test.go +++ b/internal/core/application/indexer_test.go @@ -3,6 +3,7 @@ package application import ( "context" "fmt" + "sort" "strings" "testing" @@ -195,7 +196,11 @@ func (m *mockMarkerRepoForIndexer) GetMarkersByIds( ctx context.Context, ids []string, ) ([]domain.Marker, error) { - return nil, nil + args := m.Called(ctx, ids) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).([]domain.Marker), args.Error(1) } func (m *mockMarkerRepoForIndexer) SweepMarker( @@ -854,3 +859,284 @@ func TestGetVtxoChain_PageSizeRespected(t *testing.T) { require.Equal(t, 2, len(resp.Chain)) require.NotEmpty(t, resp.NextPageToken) } + +// matchIDs returns a mock.MatchedBy matcher that matches a []string argument +// containing exactly the given IDs, regardless of order. This avoids flakes from +// non-deterministic map iteration in preloadVtxosByMarkers. +func matchIDs(expected ...string) interface{} { + sorted := make([]string, len(expected)) + copy(sorted, expected) + sort.Strings(sorted) + return mock.MatchedBy(func(ids []string) bool { + if len(ids) != len(sorted) { + return false + } + cp := make([]string, len(ids)) + copy(cp, ids) + sort.Strings(cp) + for i := range cp { + if cp[i] != sorted[i] { + return false + } + } + return true + }) +} + +// TestPreloadVtxosByMarkers_WalksMarkerChain verifies that preloadVtxosByMarkers +// follows the marker DAG upward and populates the cache with all discovered VTXOs. +func TestPreloadVtxosByMarkers_WalksMarkerChain(t *testing.T) { + _, markerRepo, indexer := newTestIndexer() + ctx := context.Background() + + // Chain: vtxo-leaf has marker-200, which has parent marker-100, which has parent marker-0. + vtxoLeaf := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "vtxo-leaf", VOut: 0}, + Amount: 100, + MarkerIDs: []string{"marker-200"}, + } + + // GetVtxoChainByMarkers returns VTXOs for each marker level. + markerRepo.On("GetVtxoChainByMarkers", ctx, matchIDs("marker-200")). + Return([]domain.Vtxo{ + {Outpoint: domain.Outpoint{Txid: "vtxo-200a", VOut: 0}, Amount: 200}, + {Outpoint: domain.Outpoint{Txid: "vtxo-200b", VOut: 0}, Amount: 201}, + }, nil) + markerRepo.On("GetVtxoChainByMarkers", ctx, matchIDs("marker-100")). + Return([]domain.Vtxo{ + {Outpoint: domain.Outpoint{Txid: "vtxo-100a", VOut: 0}, Amount: 300}, + }, nil) + markerRepo.On("GetVtxoChainByMarkers", ctx, matchIDs("marker-0")). + Return([]domain.Vtxo{ + {Outpoint: domain.Outpoint{Txid: "vtxo-0a", VOut: 0}, Amount: 400}, + }, nil) + + // GetMarkersByIds returns marker objects with parent pointers. + markerRepo.On("GetMarkersByIds", ctx, matchIDs("marker-200")). + Return([]domain.Marker{ + {ID: "marker-200", Depth: 200, ParentMarkerIDs: []string{"marker-100"}}, + }, nil) + markerRepo.On("GetMarkersByIds", ctx, matchIDs("marker-100")). + Return([]domain.Marker{ + {ID: "marker-100", Depth: 100, ParentMarkerIDs: []string{"marker-0"}}, + }, nil) + markerRepo.On("GetMarkersByIds", ctx, matchIDs("marker-0")). + Return([]domain.Marker{ + {ID: "marker-0", Depth: 0, ParentMarkerIDs: nil}, + }, nil) + + cache := make(map[string]domain.Vtxo) + err := indexer.preloadVtxosByMarkers(ctx, []domain.Vtxo{vtxoLeaf}, cache) + require.NoError(t, err) + + // Cache should contain the seed vtxo plus all vtxos from all marker levels. + require.Contains(t, cache, "vtxo-leaf:0") + require.Contains(t, cache, "vtxo-200a:0") + require.Contains(t, cache, "vtxo-200b:0") + require.Contains(t, cache, "vtxo-100a:0") + require.Contains(t, cache, "vtxo-0a:0") + require.Len(t, cache, 5) + + markerRepo.AssertNumberOfCalls(t, "GetVtxoChainByMarkers", 3) + markerRepo.AssertNumberOfCalls(t, "GetMarkersByIds", 3) +} + +// TestPreloadVtxosByMarkers_NoCycleLoop verifies that the visited set prevents +// infinite loops when markers form a cycle. +func TestPreloadVtxosByMarkers_NoCycleLoop(t *testing.T) { + _, markerRepo, indexer := newTestIndexer() + ctx := context.Background() + + vtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "vtxo-cycle", VOut: 0}, + Amount: 100, + MarkerIDs: []string{"marker-A"}, + } + + // marker-A -> marker-B -> marker-A (cycle) + markerRepo.On("GetVtxoChainByMarkers", ctx, matchIDs("marker-A")). + Return([]domain.Vtxo{ + {Outpoint: domain.Outpoint{Txid: "vtxo-a", VOut: 0}, Amount: 100}, + }, nil) + markerRepo.On("GetVtxoChainByMarkers", ctx, matchIDs("marker-B")). + Return([]domain.Vtxo{ + {Outpoint: domain.Outpoint{Txid: "vtxo-b", VOut: 0}, Amount: 200}, + }, nil) + + markerRepo.On("GetMarkersByIds", ctx, matchIDs("marker-A")). + Return([]domain.Marker{ + {ID: "marker-A", Depth: 0, ParentMarkerIDs: []string{"marker-B"}}, + }, nil) + markerRepo.On("GetMarkersByIds", ctx, matchIDs("marker-B")). + Return([]domain.Marker{ + {ID: "marker-B", Depth: 0, ParentMarkerIDs: []string{"marker-A"}}, + }, nil) + + cache := make(map[string]domain.Vtxo) + err := indexer.preloadVtxosByMarkers(ctx, []domain.Vtxo{vtxo}, cache) + require.NoError(t, err) + + // Should terminate without looping forever. + require.Contains(t, cache, "vtxo-cycle:0") + require.Contains(t, cache, "vtxo-a:0") + require.Contains(t, cache, "vtxo-b:0") + + // Each marker queried exactly once. + markerRepo.AssertNumberOfCalls(t, "GetVtxoChainByMarkers", 2) + markerRepo.AssertNumberOfCalls(t, "GetMarkersByIds", 2) +} + +// TestGetVtxoChain_WithMarkers_UsesPreload verifies that GetVtxoChain uses +// preloadVtxosByMarkers when VTXOs have markers, and that the main loop +// hits the cache instead of making additional DB calls. +func TestGetVtxoChain_WithMarkers_UsesPreload(t *testing.T) { + vtxoRepo, markerRepo, offchainTxRepo, indexer := newTestIndexerWithOffchain() + ctx := context.Background() + + txidA := strings.Repeat("a", 64) + txidB := strings.Repeat("b", 64) + txidC := strings.Repeat("c", 64) + + vtxoA := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: txidA, VOut: 0}, + Preconfirmed: true, + ExpiresAt: 1000, + MarkerIDs: []string{"marker-200"}, + } + vtxoB := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: txidB, VOut: 0}, + Preconfirmed: true, + ExpiresAt: 2000, + MarkerIDs: []string{"marker-100"}, + } + vtxoC := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: txidC, VOut: 0}, + Preconfirmed: true, + ExpiresAt: 3000, + } + + // Initial GetVtxos call for preload (frontier = [vtxoA]). + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: txidA, VOut: 0}}). + Return([]domain.Vtxo{vtxoA}, nil) + + // Preload via marker chain: marker-200 -> marker-100 -> marker-0 (no parent). + markerRepo.On("GetVtxoChainByMarkers", ctx, matchIDs("marker-200")). + Return([]domain.Vtxo{vtxoA, vtxoB}, nil) + markerRepo.On("GetMarkersByIds", ctx, matchIDs("marker-200")). + Return([]domain.Marker{ + {ID: "marker-200", Depth: 200, ParentMarkerIDs: []string{"marker-100"}}, + }, nil) + markerRepo.On("GetVtxoChainByMarkers", ctx, matchIDs("marker-100")). + Return([]domain.Vtxo{vtxoB, vtxoC}, nil) + markerRepo.On("GetMarkersByIds", ctx, matchIDs("marker-100")). + Return([]domain.Marker{ + {ID: "marker-100", Depth: 100, ParentMarkerIDs: nil}, + }, nil) + + // ensureVtxosCached will find cache hits for B and C (preloaded), + // so no additional GetVtxos calls for them. + // Marker window loading via GetVtxosByMarker is still called for markers on + // cache misses, but since everything is preloaded there are no misses. + markerRepo.On("GetVtxosByMarker", ctx, mock.Anything). + Return([]domain.Vtxo{}, nil).Maybe() + + // Offchain tx setup for preconfirmed chain. + cpA := makeCheckpointPSBT(t, txidB, 0) + cpB := makeCheckpointPSBT(t, txidC, 0) + offchainTxRepo.On("GetOffchainTx", ctx, txidA). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{"cp-a": cpA}}, nil) + offchainTxRepo.On("GetOffchainTx", ctx, txidB). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{"cp-b": cpB}}, nil) + offchainTxRepo.On("GetOffchainTx", ctx, txidC). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{}}, nil) + + resp, err := indexer.GetVtxoChain(ctx, Outpoint{Txid: txidA, VOut: 0}, nil, "") + require.NoError(t, err) + require.Equal(t, 5, len(resp.Chain)) // A(ark+cp) + B(ark+cp) + C(ark) + + // GetVtxoChainByMarkers should have been called (preload path used). + markerRepo.AssertCalled(t, "GetVtxoChainByMarkers", ctx, matchIDs("marker-200")) + markerRepo.AssertCalled(t, "GetVtxoChainByMarkers", ctx, matchIDs("marker-100")) + + // GetVtxos should only be called once (for the initial preload fetch), + // not for B or C individually — they were already in the cache. + vtxoRepo.AssertNumberOfCalls(t, "GetVtxos", 1) +} + +// TestGetVtxoChain_PreloadReducesDBCalls builds a 500-VTXO preconfirmed chain +// with markers every 100 VTXOs and verifies that preloading reduces GetVtxos +// calls from ~500 (one per VTXO) to 1 (the initial frontier fetch). +func TestGetVtxoChain_PreloadReducesDBCalls(t *testing.T) { + const chainLen = 500 + const markersCount = chainLen / int(domain.MarkerInterval) // 5 + + vtxoRepo, markerRepo, offchainTxRepo, indexer := newTestIndexerWithOffchain() + ctx := context.Background() + + // Generate txids and VTXOs grouped by marker bucket. + txids := make([]string, chainLen) + vtxos := make([]domain.Vtxo, chainLen) + for i := 0; i < chainLen; i++ { + txids[i] = fmt.Sprintf("%064x", i) + markerID := fmt.Sprintf("m-%d", i/int(domain.MarkerInterval)) + vtxos[i] = domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: txids[i], VOut: 0}, + Preconfirmed: true, + ExpiresAt: int64(1000 + i), + MarkerIDs: []string{markerID}, + } + } + + // Preload: GetVtxos for frontier (single call). + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: txids[0], VOut: 0}}). + Return([]domain.Vtxo{vtxos[0]}, nil) + + // Preload: marker chain m-0 → m-1 → m-2 → m-3 → m-4. + for m := 0; m < markersCount; m++ { + mid := fmt.Sprintf("m-%d", m) + batch := vtxos[m*int(domain.MarkerInterval) : (m+1)*int(domain.MarkerInterval)] + + markerRepo.On("GetVtxoChainByMarkers", ctx, matchIDs(mid)). + Return(batch, nil) + + var parentIDs []string + if m+1 < markersCount { + parentIDs = []string{fmt.Sprintf("m-%d", m+1)} + } + markerRepo.On("GetMarkersByIds", ctx, matchIDs(mid)). + Return([]domain.Marker{ + {ID: mid, Depth: uint32(m * int(domain.MarkerInterval)), ParentMarkerIDs: parentIDs}, + }, nil) + } + + // Marker window (won't be called — all cache hits from preload). + markerRepo.On("GetVtxosByMarker", ctx, mock.Anything). + Return([]domain.Vtxo{}, nil).Maybe() + + // Offchain tx: each vtxo_i has a checkpoint pointing to vtxo_{i+1}. + for i := 0; i < chainLen-1; i++ { + cp := makeCheckpointPSBT(t, txids[i+1], 0) + offchainTxRepo.On("GetOffchainTx", ctx, txids[i]). + Return(&domain.OffchainTx{ + CheckpointTxs: map[string]string{fmt.Sprintf("cp-%d", i): cp}, + }, nil) + } + // Terminal VTXO (no checkpoints). + offchainTxRepo.On("GetOffchainTx", ctx, txids[chainLen-1]). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{}}, nil) + + resp, err := indexer.GetVtxoChain(ctx, Outpoint{Txid: txids[0], VOut: 0}, nil, "") + require.NoError(t, err) + + // Each non-terminal VTXO produces 2 items (ark + checkpoint), terminal produces 1. + expectedItems := (chainLen-1)*2 + 1 + require.Equal(t, expectedItems, len(resp.Chain)) + + // Key assertion: GetVtxos called only 1 time (preload frontier fetch). + // Without preloading this would be ~500 individual DB calls. + vtxoRepo.AssertNumberOfCalls(t, "GetVtxos", 1) + + // Marker-based preload: 5 bulk fetches + 5 marker lookups = 10 total queries. + markerRepo.AssertNumberOfCalls(t, "GetVtxoChainByMarkers", markersCount) + markerRepo.AssertNumberOfCalls(t, "GetMarkersByIds", markersCount) +} From f7cfde24bbe428af8415b9dea25b2621c5159a96 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Mon, 23 Mar 2026 17:41:41 +0100 Subject: [PATCH 39/54] add benchmarks and weird-tree tests for GetVtxoChain --- .../core/application/indexer_bench_test.go | 416 ++++++++++++++++++ internal/core/application/indexer_test.go | 272 ++++++++++++ 2 files changed, 688 insertions(+) create mode 100644 internal/core/application/indexer_bench_test.go diff --git a/internal/core/application/indexer_bench_test.go b/internal/core/application/indexer_bench_test.go new file mode 100644 index 000000000..b42e1c050 --- /dev/null +++ b/internal/core/application/indexer_bench_test.go @@ -0,0 +1,416 @@ +package application + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/arkade-os/arkd/internal/core/domain" + "github.com/btcsuite/btcd/btcutil/psbt" + "github.com/btcsuite/btcd/chaincfg/chainhash" + "github.com/btcsuite/btcd/wire" +) + +// Lightweight fake repos for benchmarks — no testify/mock overhead. +// Unused interface methods are satisfied by the embedded nil interface +// and will panic if called unexpectedly. + +type benchVtxoRepo struct { + domain.VtxoRepository + vtxos map[string]domain.Vtxo +} + +func (r *benchVtxoRepo) GetVtxos( + _ context.Context, outpoints []domain.Outpoint, +) ([]domain.Vtxo, error) { + result := make([]domain.Vtxo, 0, len(outpoints)) + for _, op := range outpoints { + if v, ok := r.vtxos[op.String()]; ok { + result = append(result, v) + } + } + return result, nil +} + +func (r *benchVtxoRepo) Close() {} + +type benchMarkerRepo struct { + domain.MarkerRepository + markers map[string]domain.Marker + vtxosByMarker map[string][]domain.Vtxo +} + +func (r *benchMarkerRepo) GetVtxoChainByMarkers( + _ context.Context, markerIDs []string, +) ([]domain.Vtxo, error) { + seen := make(map[string]bool) + var result []domain.Vtxo + for _, mid := range markerIDs { + for _, v := range r.vtxosByMarker[mid] { + key := v.Outpoint.String() + if !seen[key] { + seen[key] = true + result = append(result, v) + } + } + } + return result, nil +} + +func (r *benchMarkerRepo) GetMarkersByIds( + _ context.Context, ids []string, +) ([]domain.Marker, error) { + result := make([]domain.Marker, 0, len(ids)) + for _, id := range ids { + if m, ok := r.markers[id]; ok { + result = append(result, m) + } + } + return result, nil +} + +func (r *benchMarkerRepo) GetVtxosByMarker( + _ context.Context, markerID string, +) ([]domain.Vtxo, error) { + return r.vtxosByMarker[markerID], nil +} + +func (r *benchMarkerRepo) Close() {} + +type benchOffchainTxRepo struct { + domain.OffchainTxRepository + txs map[string]*domain.OffchainTx +} + +func (r *benchOffchainTxRepo) GetOffchainTx( + _ context.Context, txid string, +) (*domain.OffchainTx, error) { + if tx, ok := r.txs[txid]; ok { + return tx, nil + } + return &domain.OffchainTx{CheckpointTxs: map[string]string{}}, nil +} + +func (r *benchOffchainTxRepo) Close() {} + +type benchRepoManager struct { + vtxoRepo *benchVtxoRepo + markerRepo *benchMarkerRepo + offchainRepo *benchOffchainTxRepo +} + +func (m *benchRepoManager) Events() domain.EventRepository { return nil } +func (m *benchRepoManager) Rounds() domain.RoundRepository { return nil } +func (m *benchRepoManager) Vtxos() domain.VtxoRepository { return m.vtxoRepo } +func (m *benchRepoManager) Markers() domain.MarkerRepository { + if m.markerRepo == nil { + return nil + } + return m.markerRepo +} +func (m *benchRepoManager) ScheduledSession() domain.ScheduledSessionRepo { return nil } +func (m *benchRepoManager) OffchainTxs() domain.OffchainTxRepository { + if m.offchainRepo == nil { + return nil + } + return m.offchainRepo +} +func (m *benchRepoManager) Convictions() domain.ConvictionRepository { return nil } +func (m *benchRepoManager) Assets() domain.AssetRepository { return nil } +func (m *benchRepoManager) Fees() domain.FeeRepository { return nil } +func (m *benchRepoManager) Close() {} + +// benchTxid returns a deterministic 64-char hex txid for index i. +func benchTxid(i int) string { + return fmt.Sprintf("%064x", i) +} + +// benchCheckpointPSBT creates a base64-encoded PSBT with a single input. +func benchCheckpointPSBT(inputTxid string, inputVout uint32) string { + prevHash, err := chainhash.NewHashFromStr(inputTxid) + if err != nil { + panic(fmt.Sprintf("benchCheckpointPSBT: bad txid %q: %v", inputTxid, err)) + } + p, err := psbt.New( + []*wire.OutPoint{wire.NewOutPoint(prevHash, inputVout)}, + []*wire.TxOut{wire.NewTxOut(1000, []byte{0x51})}, + 2, 0, + []uint32{wire.MaxTxInSequenceNum}, + ) + if err != nil { + panic(err) + } + b64, err := p.B64Encode() + if err != nil { + panic(err) + } + return b64 +} + +// buildLinearChain creates a linear preconfirmed chain: +// +// V0 -> cp0 -> V1 -> cp1 -> V2 -> ... -> V{n-1} (terminal) +func buildLinearChain(n int, withMarkers bool) (*indexerService, domain.Outpoint) { + vtxoRepo := &benchVtxoRepo{vtxos: make(map[string]domain.Vtxo, n)} + offchainRepo := &benchOffchainTxRepo{txs: make(map[string]*domain.OffchainTx, n)} + + vtxos := make([]domain.Vtxo, n) + for i := 0; i < n; i++ { + tid := benchTxid(i) + var markerIDs []string + if withMarkers { + markerIDs = []string{fmt.Sprintf("m-%d", i/int(domain.MarkerInterval))} + } + vtxos[i] = domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: tid, VOut: 0}, + Preconfirmed: true, + ExpiresAt: int64(1000 + i), + MarkerIDs: markerIDs, + } + vtxoRepo.vtxos[vtxos[i].Outpoint.String()] = vtxos[i] + + if i < n-1 { + offchainRepo.txs[tid] = &domain.OffchainTx{ + CheckpointTxs: map[string]string{ + fmt.Sprintf("cp-%d", i): benchCheckpointPSBT(benchTxid(i+1), 0), + }, + } + } else { + offchainRepo.txs[tid] = &domain.OffchainTx{CheckpointTxs: map[string]string{}} + } + } + + var markerRepo *benchMarkerRepo + if withMarkers { + markerRepo = &benchMarkerRepo{ + markers: make(map[string]domain.Marker), + vtxosByMarker: make(map[string][]domain.Vtxo), + } + interval := int(domain.MarkerInterval) + markersCount := (n + interval - 1) / interval + for m := 0; m < markersCount; m++ { + mid := fmt.Sprintf("m-%d", m) + start := m * interval + end := start + interval + if end > n { + end = n + } + markerRepo.vtxosByMarker[mid] = vtxos[start:end] + + var parentIDs []string + if m+1 < markersCount { + parentIDs = []string{fmt.Sprintf("m-%d", m+1)} + } + markerRepo.markers[mid] = domain.Marker{ + ID: mid, + Depth: uint32(m * interval), + ParentMarkerIDs: parentIDs, + } + } + } + + svc := &indexerService{repoManager: &benchRepoManager{ + vtxoRepo: vtxoRepo, markerRepo: markerRepo, offchainRepo: offchainRepo, + }} + return svc, domain.Outpoint{Txid: benchTxid(0), VOut: 0} +} + +// buildFanoutTree creates a binary-tree shaped chain where each VTXO has +// 2 checkpoints pointing to 2 children. Depth d produces 2^(d+1)-1 VTXOs. +// +// V0 +// / \ +// V1 V2 +// / \ / \ +// V3 V4 V5 V6 +// ... +func buildFanoutTree(depth int) (*indexerService, domain.Outpoint, int) { + n := (1 << (depth + 1)) - 1 + vtxoRepo := &benchVtxoRepo{vtxos: make(map[string]domain.Vtxo, n)} + offchainRepo := &benchOffchainTxRepo{txs: make(map[string]*domain.OffchainTx, n)} + + for i := 0; i < n; i++ { + tid := benchTxid(i) + vtxoRepo.vtxos[fmt.Sprintf("%s:0", tid)] = domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: tid, VOut: 0}, + Preconfirmed: true, + ExpiresAt: int64(1000 + i), + } + + left := 2*i + 1 + right := 2*i + 2 + if left < n && right < n { + offchainRepo.txs[tid] = &domain.OffchainTx{ + CheckpointTxs: map[string]string{ + fmt.Sprintf("cp-l-%d", i): benchCheckpointPSBT(benchTxid(left), 0), + fmt.Sprintf("cp-r-%d", i): benchCheckpointPSBT(benchTxid(right), 0), + }, + } + } else { + offchainRepo.txs[tid] = &domain.OffchainTx{CheckpointTxs: map[string]string{}} + } + } + + svc := &indexerService{repoManager: &benchRepoManager{ + vtxoRepo: vtxoRepo, offchainRepo: offchainRepo, + }} + return svc, domain.Outpoint{Txid: benchTxid(0), VOut: 0}, n +} + +// buildDiamondChain creates a chain of diamond patterns where paths diverge +// and reconverge, stressing the visited-set deduplication: +// +// V0 --(2 checkpoints)--> V1, V2 +// V1 --(1 checkpoint)---> V3 +// V2 --(1 checkpoint)---> V3 (same V3 = convergence) +// V3 --(2 checkpoints)--> V4, V5 +// V4 --(1 checkpoint)---> V6 +// V5 --(1 checkpoint)---> V6 +// ... +// +// Each diamond uses 3 node indices; the convergence node is the next diamond's +// fan-out. Total unique VTXOs = 3*diamonds + 1. +func buildDiamondChain(diamonds int) (*indexerService, domain.Outpoint, int) { + n := 3*diamonds + 1 + vtxoRepo := &benchVtxoRepo{vtxos: make(map[string]domain.Vtxo, n)} + offchainRepo := &benchOffchainTxRepo{txs: make(map[string]*domain.OffchainTx, n)} + + for i := 0; i < n; i++ { + tid := benchTxid(i) + vtxoRepo.vtxos[fmt.Sprintf("%s:0", tid)] = domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: tid, VOut: 0}, + Preconfirmed: true, + ExpiresAt: int64(1000 + i), + } + } + + for d := 0; d < diamonds; d++ { + fanOut := 3 * d + midA := 3*d + 1 + midB := 3*d + 2 + converge := 3 * (d + 1) + + // Fan-out: 2 checkpoints -> midA, midB + offchainRepo.txs[benchTxid(fanOut)] = &domain.OffchainTx{ + CheckpointTxs: map[string]string{ + fmt.Sprintf("cp-a-%d", d): benchCheckpointPSBT(benchTxid(midA), 0), + fmt.Sprintf("cp-b-%d", d): benchCheckpointPSBT(benchTxid(midB), 0), + }, + } + // Mid A -> converge + offchainRepo.txs[benchTxid(midA)] = &domain.OffchainTx{ + CheckpointTxs: map[string]string{ + fmt.Sprintf("cp-ca-%d", d): benchCheckpointPSBT(benchTxid(converge), 0), + }, + } + // Mid B -> converge (same target) + offchainRepo.txs[benchTxid(midB)] = &domain.OffchainTx{ + CheckpointTxs: map[string]string{ + fmt.Sprintf("cp-cb-%d", d): benchCheckpointPSBT(benchTxid(converge), 0), + }, + } + } + // Terminal + offchainRepo.txs[benchTxid(3*diamonds)] = &domain.OffchainTx{ + CheckpointTxs: map[string]string{}, + } + + svc := &indexerService{repoManager: &benchRepoManager{ + vtxoRepo: vtxoRepo, offchainRepo: offchainRepo, + }} + return svc, domain.Outpoint{Txid: benchTxid(0), VOut: 0}, n +} + +func BenchmarkGetVtxoChain(b *testing.B) { + ctx := context.Background() + + for _, size := range []int{1000, 5000} { + b.Run(fmt.Sprintf("linear/%d/with_markers", size), func(b *testing.B) { + svc, start := buildLinearChain(size, true) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + resp, err := svc.GetVtxoChain(ctx, start, nil, "") + if err != nil { + b.Fatal(err) + } + // Sanity: (n-1)*2 + 1 = 2n-1 items (ark + checkpoint per non-terminal, ark for terminal). + if len(resp.Chain) != 2*size-1 { + b.Fatalf("expected %d chain items, got %d", 2*size-1, len(resp.Chain)) + } + } + }) + + b.Run(fmt.Sprintf("linear/%d/without_markers", size), func(b *testing.B) { + svc, start := buildLinearChain(size, false) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + resp, err := svc.GetVtxoChain(ctx, start, nil, "") + if err != nil { + b.Fatal(err) + } + if len(resp.Chain) != 2*size-1 { + b.Fatalf("expected %d chain items, got %d", 2*size-1, len(resp.Chain)) + } + } + }) + } + + b.Run("fanout/depth10_2047_vtxos", func(b *testing.B) { + svc, start, n := buildFanoutTree(10) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + resp, err := svc.GetVtxoChain(ctx, start, nil, "") + if err != nil { + b.Fatal(err) + } + // Internal nodes: 2^depth - 1 = 1023, each emits ark + 2 checkpoints = 3 items. + // Leaves: 2^depth = 1024, each emits 1 ark item. + // Total: 1023*3 + 1024 = 4093. + internalNodes := (1 << 10) - 1 + leaves := 1 << 10 + expected := internalNodes*3 + leaves + if len(resp.Chain) != expected { + b.Fatalf("expected %d chain items, got %d (n=%d)", expected, len(resp.Chain), n) + } + } + }) + + b.Run("diamond/500_pairs", func(b *testing.B) { + svc, start, _ := buildDiamondChain(500) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + resp, err := svc.GetVtxoChain(ctx, start, nil, "") + if err != nil { + b.Fatal(err) + } + // Each diamond's fan-out: ark + 2 checkpoints = 3 items. + // Each mid node: ark + 1 checkpoint = 2 items. + // Terminal: 1 ark item. + // Per diamond: 3 + 2 + 2 = 7 items. + // Total: 7*diamonds + 1. + expected := 7*500 + 1 + if len(resp.Chain) != expected { + b.Fatalf("expected %d chain items, got %d", expected, len(resp.Chain)) + } + } + }) +} + +// BenchmarkCheckpointPSBTParse measures the raw cost of PSBT base64 +// decode + parse, which dominates GetVtxoChain runtime. +func BenchmarkCheckpointPSBTParse(b *testing.B) { + encoded := benchCheckpointPSBT(benchTxid(1), 0) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := psbt.NewFromRawBytes(strings.NewReader(encoded), true) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/core/application/indexer_test.go b/internal/core/application/indexer_test.go index 0ce75f648..41fe7d35c 100644 --- a/internal/core/application/indexer_test.go +++ b/internal/core/application/indexer_test.go @@ -860,6 +860,32 @@ func TestGetVtxoChain_PageSizeRespected(t *testing.T) { require.NotEmpty(t, resp.NextPageToken) } +// matchOutpoints returns a mock.MatchedBy matcher that matches a []domain.Outpoint +// argument containing exactly the given outpoints, regardless of order. +func matchOutpoints(expected ...domain.Outpoint) interface{} { + sorted := make([]string, len(expected)) + for i, op := range expected { + sorted[i] = op.String() + } + sort.Strings(sorted) + return mock.MatchedBy(func(ops []domain.Outpoint) bool { + if len(ops) != len(sorted) { + return false + } + cp := make([]string, len(ops)) + for i, op := range ops { + cp[i] = op.String() + } + sort.Strings(cp) + for i := range cp { + if cp[i] != sorted[i] { + return false + } + } + return true + }) +} + // matchIDs returns a mock.MatchedBy matcher that matches a []string argument // containing exactly the given IDs, regardless of order. This avoids flakes from // non-deterministic map iteration in preloadVtxosByMarkers. @@ -1140,3 +1166,249 @@ func TestGetVtxoChain_PreloadReducesDBCalls(t *testing.T) { markerRepo.AssertNumberOfCalls(t, "GetVtxoChainByMarkers", markersCount) markerRepo.AssertNumberOfCalls(t, "GetMarkersByIds", markersCount) } + +// TestGetVtxoChain_Fanout verifies that a VTXO with 2 checkpoints pointing +// to different parents correctly traverses both branches. +// +// A --(cp1)--> B +// A --(cp2)--> C +func TestGetVtxoChain_Fanout(t *testing.T) { + vtxoRepo, markerRepo, offchainTxRepo, indexer := newTestIndexerWithOffchain() + ctx := context.Background() + + txidA := strings.Repeat("a", 64) + txidB := strings.Repeat("b", 64) + txidC := strings.Repeat("c", 64) + + vtxoA := domain.Vtxo{Outpoint: domain.Outpoint{Txid: txidA, VOut: 0}, Preconfirmed: true, ExpiresAt: 1000} + vtxoB := domain.Vtxo{Outpoint: domain.Outpoint{Txid: txidB, VOut: 0}, Preconfirmed: true, ExpiresAt: 2000} + vtxoC := domain.Vtxo{Outpoint: domain.Outpoint{Txid: txidC, VOut: 0}, Preconfirmed: true, ExpiresAt: 3000} + + // Preload frontier fetch + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{vtxoA.Outpoint}). + Return([]domain.Vtxo{vtxoA}, nil) + // ensureVtxosCached for B and C (order-independent) + vtxoRepo.On("GetVtxos", ctx, matchOutpoints(vtxoB.Outpoint, vtxoC.Outpoint)). + Return([]domain.Vtxo{vtxoB, vtxoC}, nil) + + markerRepo.On("GetVtxosByMarker", ctx, mock.Anything). + Return([]domain.Vtxo{}, nil).Maybe() + + // A has 2 checkpoints: one to B, one to C + cpB := makeCheckpointPSBT(t, txidB, 0) + cpC := makeCheckpointPSBT(t, txidC, 0) + offchainTxRepo.On("GetOffchainTx", ctx, txidA). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{"cp-b": cpB, "cp-c": cpC}}, nil) + offchainTxRepo.On("GetOffchainTx", ctx, txidB). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{}}, nil) + offchainTxRepo.On("GetOffchainTx", ctx, txidC). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{}}, nil) + + resp, err := indexer.GetVtxoChain(ctx, Outpoint{Txid: txidA, VOut: 0}, nil, "") + require.NoError(t, err) + + // A: ark + 2 checkpoints = 3. B: ark = 1. C: ark = 1. Total: 5. + require.Equal(t, 5, len(resp.Chain)) + + // A is always the first ark tx + require.Equal(t, txidA, resp.Chain[0].Txid) + require.Equal(t, IndexerChainedTxTypeArk, resp.Chain[0].Type) + require.Len(t, resp.Chain[0].Spends, 2) + + // Count chain item types + arkCount, cpCount := 0, 0 + for _, item := range resp.Chain { + switch item.Type { + case IndexerChainedTxTypeArk: + arkCount++ + case IndexerChainedTxTypeCheckpoint: + cpCount++ + } + } + require.Equal(t, 3, arkCount) + require.Equal(t, 2, cpCount) +} + +// TestGetVtxoChain_Diamond verifies that two paths converging on the same +// ancestor VTXO only process that ancestor once. +// +// A --(cp1)--> B --(cp)--> D +// A --(cp2)--> C --(cp)--> D (same D) +func TestGetVtxoChain_Diamond(t *testing.T) { + vtxoRepo, markerRepo, offchainTxRepo, indexer := newTestIndexerWithOffchain() + ctx := context.Background() + + txidA := strings.Repeat("a", 64) + txidB := strings.Repeat("b", 64) + txidC := strings.Repeat("c", 64) + txidD := strings.Repeat("d", 64) + + vtxoA := domain.Vtxo{Outpoint: domain.Outpoint{Txid: txidA, VOut: 0}, Preconfirmed: true, ExpiresAt: 1000} + vtxoB := domain.Vtxo{Outpoint: domain.Outpoint{Txid: txidB, VOut: 0}, Preconfirmed: true, ExpiresAt: 2000} + vtxoC := domain.Vtxo{Outpoint: domain.Outpoint{Txid: txidC, VOut: 0}, Preconfirmed: true, ExpiresAt: 3000} + vtxoD := domain.Vtxo{Outpoint: domain.Outpoint{Txid: txidD, VOut: 0}, Preconfirmed: true, ExpiresAt: 4000} + + // Preload frontier + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{vtxoA.Outpoint}). + Return([]domain.Vtxo{vtxoA}, nil) + // B and C fetched together (order varies due to map iteration) + vtxoRepo.On("GetVtxos", ctx, matchOutpoints(vtxoB.Outpoint, vtxoC.Outpoint)). + Return([]domain.Vtxo{vtxoB, vtxoC}, nil) + // D appears as [D, D] because both B and C point to it before D is visited. + vtxoRepo.On("GetVtxos", ctx, mock.MatchedBy(func(ops []domain.Outpoint) bool { + for _, op := range ops { + if op.String() != vtxoD.Outpoint.String() { + return false + } + } + return len(ops) > 0 + })).Return([]domain.Vtxo{vtxoD}, nil) + + markerRepo.On("GetVtxosByMarker", ctx, mock.Anything). + Return([]domain.Vtxo{}, nil).Maybe() + + // A fans out to B and C + cpB := makeCheckpointPSBT(t, txidB, 0) + cpC := makeCheckpointPSBT(t, txidC, 0) + offchainTxRepo.On("GetOffchainTx", ctx, txidA). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{"cp-b": cpB, "cp-c": cpC}}, nil) + + // B converges to D + cpBD := makeCheckpointPSBT(t, txidD, 0) + offchainTxRepo.On("GetOffchainTx", ctx, txidB). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{"cp-bd": cpBD}}, nil) + + // C converges to same D + cpCD := makeCheckpointPSBT(t, txidD, 0) + offchainTxRepo.On("GetOffchainTx", ctx, txidC). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{"cp-cd": cpCD}}, nil) + + // D is terminal + offchainTxRepo.On("GetOffchainTx", ctx, txidD). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{}}, nil) + + resp, err := indexer.GetVtxoChain(ctx, Outpoint{Txid: txidA, VOut: 0}, nil, "") + require.NoError(t, err) + + // A: ark + 2cp = 3. B: ark + 1cp = 2. C: ark + 1cp = 2. D: ark = 1. Total: 8. + require.Equal(t, 8, len(resp.Chain)) + + // D must appear exactly once despite convergence from B and C. + dCount := 0 + for _, item := range resp.Chain { + if item.Txid == txidD { + dCount++ + } + } + require.Equal(t, 1, dCount, "converged VTXO D should appear exactly once") +} + +// TestGetVtxoChain_MarkerBoundaryStart verifies that a chain starting exactly +// at marker boundary depth 0 preloads correctly (no parents to walk). +func TestGetVtxoChain_MarkerBoundaryStart(t *testing.T) { + vtxoRepo, markerRepo, offchainTxRepo, indexer := newTestIndexerWithOffchain() + ctx := context.Background() + + txidA := strings.Repeat("a", 64) + txidB := strings.Repeat("b", 64) + + vtxoA := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: txidA, VOut: 0}, Preconfirmed: true, + ExpiresAt: 1000, MarkerIDs: []string{"m-0"}, + } + vtxoB := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: txidB, VOut: 0}, Preconfirmed: true, + ExpiresAt: 2000, MarkerIDs: []string{"m-0"}, + } + + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{vtxoA.Outpoint}). + Return([]domain.Vtxo{vtxoA}, nil) + + // Preload: marker m-0 at depth 0 with no parents. + markerRepo.On("GetVtxoChainByMarkers", ctx, matchIDs("m-0")). + Return([]domain.Vtxo{vtxoA, vtxoB}, nil) + markerRepo.On("GetMarkersByIds", ctx, matchIDs("m-0")). + Return([]domain.Marker{ + {ID: "m-0", Depth: 0, ParentMarkerIDs: nil}, + }, nil) + markerRepo.On("GetVtxosByMarker", ctx, mock.Anything). + Return([]domain.Vtxo{}, nil).Maybe() + + cpB := makeCheckpointPSBT(t, txidB, 0) + offchainTxRepo.On("GetOffchainTx", ctx, txidA). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{"cp-b": cpB}}, nil) + offchainTxRepo.On("GetOffchainTx", ctx, txidB). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{}}, nil) + + resp, err := indexer.GetVtxoChain(ctx, Outpoint{Txid: txidA, VOut: 0}, nil, "") + require.NoError(t, err) + require.Equal(t, 3, len(resp.Chain)) // A(ark) + cp + B(ark) + + // Both VTXOs were preloaded via marker — only the frontier fetch needed. + vtxoRepo.AssertNumberOfCalls(t, "GetVtxos", 1) +} + +// TestGetVtxoChain_OverlappingMarkers verifies correct deduplication when a +// VTXO has multiple markers and one marker is both directly attached AND +// a parent of another marker. +// +// A (markers: m-a, m-b) -> B (marker: m-b) -> C (no markers) +// m-a has parent m-b, so m-b is already visited when discovered as parent. +func TestGetVtxoChain_OverlappingMarkers(t *testing.T) { + vtxoRepo, markerRepo, offchainTxRepo, indexer := newTestIndexerWithOffchain() + ctx := context.Background() + + txidA := strings.Repeat("a", 64) + txidB := strings.Repeat("b", 64) + txidC := strings.Repeat("c", 64) + + vtxoA := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: txidA, VOut: 0}, Preconfirmed: true, + ExpiresAt: 1000, MarkerIDs: []string{"m-a", "m-b"}, + } + vtxoB := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: txidB, VOut: 0}, Preconfirmed: true, + ExpiresAt: 2000, MarkerIDs: []string{"m-b"}, + } + vtxoC := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: txidC, VOut: 0}, Preconfirmed: true, + ExpiresAt: 3000, + } + + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{vtxoA.Outpoint}). + Return([]domain.Vtxo{vtxoA}, nil) + + // Preload: m-a and m-b fetched together. m-a's parent m-b is already visited. + markerRepo.On("GetVtxoChainByMarkers", ctx, matchIDs("m-a", "m-b")). + Return([]domain.Vtxo{vtxoA, vtxoB}, nil) + markerRepo.On("GetMarkersByIds", ctx, matchIDs("m-a", "m-b")). + Return([]domain.Marker{ + {ID: "m-a", Depth: 200, ParentMarkerIDs: []string{"m-b"}}, + {ID: "m-b", Depth: 100, ParentMarkerIDs: nil}, + }, nil) + markerRepo.On("GetVtxosByMarker", ctx, mock.Anything). + Return([]domain.Vtxo{}, nil).Maybe() + + // C not in any marker group — cache miss triggers DB fetch. + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{vtxoC.Outpoint}). + Return([]domain.Vtxo{vtxoC}, nil) + + cpB := makeCheckpointPSBT(t, txidB, 0) + cpC := makeCheckpointPSBT(t, txidC, 0) + offchainTxRepo.On("GetOffchainTx", ctx, txidA). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{"cp-b": cpB}}, nil) + offchainTxRepo.On("GetOffchainTx", ctx, txidB). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{"cp-c": cpC}}, nil) + offchainTxRepo.On("GetOffchainTx", ctx, txidC). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{}}, nil) + + resp, err := indexer.GetVtxoChain(ctx, Outpoint{Txid: txidA, VOut: 0}, nil, "") + require.NoError(t, err) + require.Equal(t, 5, len(resp.Chain)) + + // 1 preload frontier + 1 for C (cache miss). A and B were preloaded via markers. + vtxoRepo.AssertNumberOfCalls(t, "GetVtxos", 2) + // Only 1 batch of marker fetches (m-a + m-b together; m-b's parent already visited). + markerRepo.AssertNumberOfCalls(t, "GetVtxoChainByMarkers", 1) + markerRepo.AssertNumberOfCalls(t, "GetMarkersByIds", 1) +} From 373eeecf2b6ef739cbaf4815ee3045e717879cda Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 2 Apr 2026 13:35:02 -0400 Subject: [PATCH 40/54] lint --- internal/core/application/indexer.go | 6 ++++-- internal/interface/grpc/handlers/indexer.go | 8 +++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/core/application/indexer.go b/internal/core/application/indexer.go index d3d455004..5c8c428fd 100644 --- a/internal/core/application/indexer.go +++ b/internal/core/application/indexer.go @@ -322,7 +322,6 @@ func (i *indexerService) GetVtxosByOutpoint( }, nil } - func (i *indexerService) GetVtxoChain( ctx context.Context, authToken string, vtxoKey Outpoint, page *Page, pageToken string, ) (*VtxoChainResp, error) { @@ -629,7 +628,10 @@ func (i *indexerService) walkVtxoChain( for _, b64 := range offchainTx.CheckpointTxs { ptx, err := psbt.NewFromRawBytes(strings.NewReader(b64), true) if err != nil { - return nil, nil, "", fmt.Errorf("failed to deserialize checkpoint tx: %s", err) + return nil, nil, "", fmt.Errorf( + "failed to deserialize checkpoint tx: %s", + err, + ) } txid := ptx.UnsignedTx.TxID() diff --git a/internal/interface/grpc/handlers/indexer.go b/internal/interface/grpc/handlers/indexer.go index dc51213d8..a47ce1e91 100644 --- a/internal/interface/grpc/handlers/indexer.go +++ b/internal/interface/grpc/handlers/indexer.go @@ -336,7 +336,13 @@ func (e *indexerService) GetVtxoChain( if parseErr != nil { return nil, status.Error(codes.InvalidArgument, parseErr.Error()) } - resp, err = e.indexerSvc.GetVtxoChain(ctx, request.GetToken(), *outpoint, page, request.GetPageToken()) + resp, err = e.indexerSvc.GetVtxoChain( + ctx, + request.GetToken(), + *outpoint, + page, + request.GetPageToken(), + ) } if err != nil { return nil, status.Errorf(codes.Internal, "%s", err.Error()) From 453417eaa9a29d2f28d71173fad12657fb59ef65 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 2 Apr 2026 14:17:39 -0400 Subject: [PATCH 41/54] use offchainTxCache to handle test reace condition --- internal/config/config.go | 6 ++++ internal/core/application/indexer.go | 43 +++++++++++++++++++--------- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 049d0acad..a6b90144f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -738,8 +738,14 @@ func (c *Config) IndexerService() (application.IndexerService, error) { return nil, fmt.Errorf("failed to get server signing pubkey: %w", err) } + var offchainTxCache ports.OffChainTxStore + if c.liveStore != nil { + offchainTxCache = c.liveStore.OffchainTxs() + } + return application.NewIndexerService( c.repo, c.wallet, privkey, signerPubkey, c.IndexerExposure, c.IndexerAuthTokenExpiry, + offchainTxCache, ) } diff --git a/internal/core/application/indexer.go b/internal/core/application/indexer.go index 5c8c428fd..a637cff16 100644 --- a/internal/core/application/indexer.go +++ b/internal/core/application/indexer.go @@ -80,13 +80,14 @@ type IndexerService interface { } type indexerService struct { - repoManager ports.RepoManager - wallet ports.WalletService - authPrvkey *btcec.PrivateKey // key used to sign auth tokens - signerPubkey *btcec.PublicKey // server's signing key, used for stripping signatures from txs - txExposure exposure - authTokenTTL time.Duration - tokenCache *tokenCache + repoManager ports.RepoManager + wallet ports.WalletService + authPrvkey *btcec.PrivateKey // key used to sign auth tokens + signerPubkey *btcec.PublicKey // server's signing key, used for stripping signatures from txs + txExposure exposure + authTokenTTL time.Duration + tokenCache *tokenCache + offchainTxCache ports.OffChainTxStore } func NewIndexerService( @@ -96,6 +97,7 @@ func NewIndexerService( signerPubkey *btcec.PublicKey, txExposure string, authTokenExpirySec int64, + offchainTxCache ports.OffChainTxStore, ) (IndexerService, error) { // validate txExposure switch exposure(txExposure) { @@ -110,12 +112,13 @@ func NewIndexerService( } svc := &indexerService{ - repoManager: repoManager, - wallet: wallet, - authPrvkey: privkey, - txExposure: exposure(txExposure), - authTokenTTL: ttl, - tokenCache: newTokenCache(ttl), + repoManager: repoManager, + wallet: wallet, + authPrvkey: privkey, + txExposure: exposure(txExposure), + authTokenTTL: ttl, + tokenCache: newTokenCache(ttl), + offchainTxCache: offchainTxCache, } if signerPubkey != nil { @@ -271,6 +274,20 @@ func (i *indexerService) GetVtxos( return nil, err } + // Mark vtxos that are pending-spent in the offchain tx cache. + // The DB projection updates asynchronously, so without this check + // clients can see stale spendable vtxos and build duplicate txs. + if i.offchainTxCache != nil { + for idx := range allVtxos { + if allVtxos[idx].Spent { + continue + } + if spent, _ := i.offchainTxCache.Includes(ctx, allVtxos[idx].Outpoint); spent { + allVtxos[idx].Spent = true + } + } + } + if spendableOnly { spendableVtxos := make([]domain.Vtxo, 0, len(allVtxos)) for _, vtxo := range allVtxos { From 69879d04771a690c7dd1add030839b730735c143 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Wed, 15 Apr 2026 14:39:58 -0400 Subject: [PATCH 42/54] bulk-fetch offchain txs in walkVtxoChain to reduce DB round-trips (#1005) * bulk-fetch offchain txs in walkVtxoChain to reduce DB round-trips * Add benchmark and test for bulk offchain tx fetch improvement * index idx_checkpoint_tx_offchain_txid * testing our performance times for GetOffchainTxsByTxids * chunk SQLite bulk offchain tx fetch + multi-txid test * lint * preload offchain txs via marker DAG in walkVtxoChain, nbxplorer version bump * offchainTxRepo check, may be nil in test helpers * move walkVtxoChain timing out of prod code into in-process test * clarify simulated latency in timing breakdown and panic on unwired repo accessors --- docker-compose.regtest.yml | 2 +- internal/core/application/indexer.go | 92 +++- .../core/application/indexer_bench_test.go | 463 +++++++++++++++++- .../core/application/indexer_exposure_test.go | 118 +++++ internal/core/application/indexer_test.go | 48 +- internal/core/domain/offchain_tx_repo.go | 1 + internal/infrastructure/db/badger/ark_repo.go | 23 + ...checkpoint_tx_offchain_txid_index.down.sql | 1 + ...0_checkpoint_tx_offchain_txid_index.up.sql | 2 + .../db/postgres/offchain_tx_repo.go | 56 +++ .../db/postgres/sqlc/queries/query.sql.go | 44 ++ .../infrastructure/db/postgres/sqlc/query.sql | 3 + internal/infrastructure/db/service_test.go | 61 +++ ...checkpoint_tx_offchain_txid_index.down.sql | 1 + ...0_checkpoint_tx_offchain_txid_index.up.sql | 2 + .../db/sqlite/offchain_tx_repo.go | 66 +++ .../db/sqlite/sqlc/queries/query.sql.go | 54 ++ .../infrastructure/db/sqlite/sqlc/query.sql | 3 + internal/test/e2e/vtxo_chain_test.go | 176 +++++++ 19 files changed, 1190 insertions(+), 26 deletions(-) create mode 100644 internal/infrastructure/db/postgres/migration/20260409140000_checkpoint_tx_offchain_txid_index.down.sql create mode 100644 internal/infrastructure/db/postgres/migration/20260409140000_checkpoint_tx_offchain_txid_index.up.sql create mode 100644 internal/infrastructure/db/sqlite/migration/20260409140000_checkpoint_tx_offchain_txid_index.down.sql create mode 100644 internal/infrastructure/db/sqlite/migration/20260409140000_checkpoint_tx_offchain_txid_index.up.sql create mode 100644 internal/test/e2e/vtxo_chain_test.go diff --git a/docker-compose.regtest.yml b/docker-compose.regtest.yml index 562ee309f..1bfc818f5 100644 --- a/docker-compose.regtest.yml +++ b/docker-compose.regtest.yml @@ -20,7 +20,7 @@ services: container_name: nbxplorer ports: - 32838:32838 - image: nicolasdorier/nbxplorer:2.5.30 + image: nicolasdorier/nbxplorer:2.5.30-1 environment: - NBXPLORER_NETWORK=regtest - NBXPLORER_CHAINS=btc diff --git a/internal/core/application/indexer.go b/internal/core/application/indexer.go index a637cff16..f7e926e36 100644 --- a/internal/core/application/indexer.go +++ b/internal/core/application/indexer.go @@ -569,19 +569,20 @@ func (i *indexerService) walkVtxoChain( chain := make([]ChainTx, 0) nextVtxos := frontier visited := make(map[string]bool) + offchainTxCache := make(map[string]*domain.OffchainTx) allOutpoints := make([]Outpoint, 0) // Lazy cache for VTXOs loaded during this page. vtxoCache := make(map[string]domain.Vtxo) loadedMarkers := make(map[string]bool) - // Eagerly preload VTXOs by walking the marker DAG upward. + // Eagerly preload VTXOs and offchain txs by walking the marker DAG upward. if i.repoManager.Markers() != nil { startVtxos, err := i.repoManager.Vtxos().GetVtxos(ctx, nextVtxos) if err != nil { return nil, nil, "", err } - if err := i.preloadVtxosByMarkers(ctx, startVtxos, vtxoCache); err != nil { + if err := i.preloadByMarkers(ctx, startVtxos, vtxoCache, offchainTxCache); err != nil { return nil, nil, "", err } } @@ -601,6 +602,33 @@ func (i *indexerService) walkVtxoChain( return nil, nil, "", fmt.Errorf("vtxo not found for outpoint: %v", nextVtxos) } + missingOffchainTxids := make(map[string]struct{}) + for _, vtxo := range vtxos { + if !vtxo.Preconfirmed { + continue + } + if _, ok := offchainTxCache[vtxo.Txid]; ok { + continue + } + missingOffchainTxids[vtxo.Txid] = struct{}{} + } + + if len(missingOffchainTxids) > 0 { + txids := make([]string, 0, len(missingOffchainTxids)) + for txid := range missingOffchainTxids { + txids = append(txids, txid) + } + + offchainTxs, err := i.repoManager.OffchainTxs().GetOffchainTxsByTxids(ctx, txids) + if err != nil { + return nil, nil, "", fmt.Errorf("failed to retrieve offchain txs: %s", err) + } + + for _, tx := range offchainTxs { + offchainTxCache[tx.ArkTxid] = tx + } + } + newNextVtxos := make([]domain.Outpoint, 0) for _, vtxo := range vtxos { key := vtxo.Outpoint.String() @@ -630,9 +658,14 @@ func (i *indexerService) walkVtxoChain( // also, we have to populate the newNextVtxos with the checkpoints inputs // in order to continue the chain in the next iteration if vtxo.Preconfirmed { - offchainTx, err := i.repoManager.OffchainTxs().GetOffchainTx(ctx, vtxo.Txid) - if err != nil { - return nil, nil, "", fmt.Errorf("failed to retrieve offchain tx: %s", err) + offchainTx, ok := offchainTxCache[vtxo.Txid] + if !ok { + var err error + offchainTx, err = i.repoManager.OffchainTxs().GetOffchainTx(ctx, vtxo.Txid) + if err != nil { + return nil, nil, "", fmt.Errorf("failed to retrieve offchain tx: %s", err) + } + offchainTxCache[vtxo.Txid] = offchainTx } chainTx := ChainTx{ @@ -741,7 +774,6 @@ func (i *indexerService) walkVtxoChain( nextVtxos = newNextVtxos } - // Chain exhausted — no more pages. return chain, allOutpoints, "", nil } @@ -775,20 +807,22 @@ func decodeChainCursor(token string) ([]domain.Outpoint, error) { return outpoints, nil } -// preloadVtxosByMarkers bulk-fetches VTXOs by walking the marker DAG upward -// from the markers of startVtxos. This reduces DB round-trips from O(chain_length) -// to O(chain_length / MarkerInterval). -func (i *indexerService) preloadVtxosByMarkers( +// preloadByMarkers bulk-fetches VTXOs and their offchain txs by walking the +// marker DAG upward from the markers of startVtxos. This reduces DB round-trips +// from O(chain_length) to O(chain_length / MarkerInterval) for both layers. +func (i *indexerService) preloadByMarkers( ctx context.Context, startVtxos []domain.Vtxo, - cache map[string]domain.Vtxo, + vtxoCache map[string]domain.Vtxo, + offchainTxCache map[string]*domain.OffchainTx, ) error { markerRepo := i.repoManager.Markers() + offchainTxRepo := i.repoManager.OffchainTxs() // Seed cache and collect initial marker IDs. currentMarkerIDs := make(map[string]bool) for _, v := range startVtxos { - cache[v.Outpoint.String()] = v + vtxoCache[v.Outpoint.String()] = v for _, mid := range v.MarkerIDs { currentMarkerIDs[mid] = true } @@ -809,8 +843,38 @@ func (i *indexerService) preloadVtxosByMarkers( return err } for _, v := range vtxos { - if _, ok := cache[v.Outpoint.String()]; !ok { - cache[v.Outpoint.String()] = v + if _, ok := vtxoCache[v.Outpoint.String()]; !ok { + vtxoCache[v.Outpoint.String()] = v + } + } + + // Piggyback: bulk-fetch the offchain txs for the preconfirmed VTXOs + // in this window, so the walk loop never has to hit the DB per-hop. + missingTxids := make([]string, 0, len(vtxos)) + seen := make(map[string]bool, len(vtxos)) + for _, v := range vtxos { + if !v.Preconfirmed { + continue + } + if seen[v.Txid] { + continue + } + seen[v.Txid] = true + if _, ok := offchainTxCache[v.Txid]; ok { + continue + } + missingTxids = append(missingTxids, v.Txid) + } + // offchainTxRepo may be nil in test helpers that do not wire up the + // offchain-tx repo. Skip the piggyback in that case — the walk loop + // will fall back to its own in-loop bulk fetch for any cache misses. + if len(missingTxids) > 0 && offchainTxRepo != nil { + offchainTxs, err := offchainTxRepo.GetOffchainTxsByTxids(ctx, missingTxids) + if err != nil { + return err + } + for _, tx := range offchainTxs { + offchainTxCache[tx.ArkTxid] = tx } } diff --git a/internal/core/application/indexer_bench_test.go b/internal/core/application/indexer_bench_test.go index 312edf668..acb9ac8b3 100644 --- a/internal/core/application/indexer_bench_test.go +++ b/internal/core/application/indexer_bench_test.go @@ -3,13 +3,18 @@ package application import ( "context" "fmt" + "sort" "strings" + "sync" + "sync/atomic" "testing" + "time" "github.com/arkade-os/arkd/internal/core/domain" "github.com/btcsuite/btcd/btcutil/psbt" "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/wire" + "github.com/stretchr/testify/require" ) // Lightweight fake repos for benchmarks — no testify/mock overhead. @@ -92,12 +97,24 @@ func (r *benchOffchainTxRepo) GetOffchainTx( return &domain.OffchainTx{CheckpointTxs: map[string]string{}}, nil } +func (r *benchOffchainTxRepo) GetOffchainTxsByTxids( + _ context.Context, txids []string, +) ([]*domain.OffchainTx, error) { + result := make([]*domain.OffchainTx, 0, len(txids)) + for _, txid := range txids { + if tx, ok := r.txs[txid]; ok { + result = append(result, tx) + } + } + return result, nil +} + func (r *benchOffchainTxRepo) Close() {} type benchRepoManager struct { vtxoRepo *benchVtxoRepo markerRepo *benchMarkerRepo - offchainRepo *benchOffchainTxRepo + offchainRepo domain.OffchainTxRepository } func (m *benchRepoManager) Events() domain.EventRepository { return nil } @@ -172,12 +189,16 @@ func buildLinearChain(n int, withMarkers bool) (*indexerService, domain.Outpoint if i < n-1 { offchainRepo.txs[tid] = &domain.OffchainTx{ + ArkTxid: tid, CheckpointTxs: map[string]string{ fmt.Sprintf("cp-%d", i): benchCheckpointPSBT(benchTxid(i+1), 0), }, } } else { - offchainRepo.txs[tid] = &domain.OffchainTx{CheckpointTxs: map[string]string{}} + offchainRepo.txs[tid] = &domain.OffchainTx{ + ArkTxid: tid, + CheckpointTxs: map[string]string{}, + } } } @@ -414,3 +435,441 @@ func BenchmarkCheckpointPSBTParse(b *testing.B) { } } } + +// countingOffchainTxRepo wraps benchOffchainTxRepo and counts calls. +type countingOffchainTxRepo struct { + inner *benchOffchainTxRepo + singleCalls atomic.Int64 + bulkCalls atomic.Int64 + latencyPerCall time.Duration +} + +func (r *countingOffchainTxRepo) GetOffchainTx( + ctx context.Context, txid string, +) (*domain.OffchainTx, error) { + r.singleCalls.Add(1) + if r.latencyPerCall > 0 { + time.Sleep(r.latencyPerCall) + } + return r.inner.GetOffchainTx(ctx, txid) +} + +func (r *countingOffchainTxRepo) GetOffchainTxsByTxids( + ctx context.Context, txids []string, +) ([]*domain.OffchainTx, error) { + r.bulkCalls.Add(1) + if r.latencyPerCall > 0 { + time.Sleep(r.latencyPerCall) // one round-trip regardless of batch size + } + return r.inner.GetOffchainTxsByTxids(ctx, txids) +} + +func (r *countingOffchainTxRepo) AddOrUpdateOffchainTx( + _ context.Context, _ *domain.OffchainTx, +) error { + return nil +} + +func (r *countingOffchainTxRepo) Close() {} + +func (r *countingOffchainTxRepo) reset() { + r.singleCalls.Store(0) + r.bulkCalls.Store(0) +} + +// noBulkOffchainTxRepo is like benchOffchainTxRepo but GetOffchainTxsByTxids +// always returns empty, forcing the fallback to individual GetOffchainTx calls. +// This simulates the pre-optimization behavior. +type noBulkOffchainTxRepo struct { + countingOffchainTxRepo +} + +func (r *noBulkOffchainTxRepo) GetOffchainTxsByTxids( + _ context.Context, _ []string, +) ([]*domain.OffchainTx, error) { + r.bulkCalls.Add(1) + return []*domain.OffchainTx{}, nil +} + +// TestBulkOffchainTxReducesDBCalls verifies that the bulk prefetch reduces the +// number of DB round-trips. Uses a fanout tree where each iteration processes +// multiple VTXOs — bulk fetches all offchain txs in one call per iteration +// instead of one call per VTXO. +func TestBulkOffchainTxReducesDBCalls(t *testing.T) { + const depth = 8 // 2^9 - 1 = 511 VTXOs + ctx := context.Background() + + // Build fanout tree data (reuse the helper's repo setup). + n := (1 << (depth + 1)) - 1 + vtxoRepo := &benchVtxoRepo{vtxos: make(map[string]domain.Vtxo, n)} + innerRepo := &benchOffchainTxRepo{txs: make(map[string]*domain.OffchainTx, n)} + + for i := 0; i < n; i++ { + tid := benchTxid(i) + vtxoRepo.vtxos[fmt.Sprintf("%s:0", tid)] = domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: tid, VOut: 0}, + Preconfirmed: true, + ExpiresAt: int64(1000 + i), + } + left := 2*i + 1 + right := 2*i + 2 + if left < n && right < n { + innerRepo.txs[tid] = &domain.OffchainTx{ + ArkTxid: tid, + CheckpointTxs: map[string]string{ + fmt.Sprintf("cp-l-%d", i): benchCheckpointPSBT(benchTxid(left), 0), + fmt.Sprintf("cp-r-%d", i): benchCheckpointPSBT(benchTxid(right), 0), + }, + } + } else { + innerRepo.txs[tid] = &domain.OffchainTx{ + ArkTxid: tid, + CheckpointTxs: map[string]string{}, + } + } + } + + start := Outpoint{Txid: benchTxid(0), VOut: 0} + + // With bulk fetch (current behavior). + bulkRepo := &countingOffchainTxRepo{inner: innerRepo} + svc := &indexerService{repoManager: &benchRepoManager{ + vtxoRepo: vtxoRepo, offchainRepo: bulkRepo, + }} + resp, err := svc.GetVtxoChain(ctx, "", start, nil, "") + require.NoError(t, err) + + bulkSingle := bulkRepo.singleCalls.Load() + bulkBulk := bulkRepo.bulkCalls.Load() + + // Without bulk fetch (simulated pre-optimization: bulk returns empty). + noBulkRepo := &noBulkOffchainTxRepo{countingOffchainTxRepo{inner: innerRepo}} + svc2 := &indexerService{repoManager: &benchRepoManager{ + vtxoRepo: vtxoRepo, offchainRepo: noBulkRepo, + }} + resp2, err := svc2.GetVtxoChain(ctx, "", start, nil, "") + require.NoError(t, err) + require.Equal(t, len(resp.Chain), len(resp2.Chain)) + + noBulkSingle := noBulkRepo.singleCalls.Load() + + t.Logf("fanout tree: depth=%d, %d VTXOs", depth, n) + t.Logf("WITH bulk: %d bulk calls, %d individual calls (total round-trips: %d)", + bulkBulk, bulkSingle, bulkBulk+bulkSingle) + t.Logf("WITHOUT bulk: %d individual calls (total round-trips: %d)", + noBulkSingle, noBulkSingle) + + // With bulk fetch, individual calls should be 0 (all served from cache). + require.Zero(t, bulkSingle, "bulk prefetch should eliminate individual GetOffchainTx calls") + // Bulk calls = depth+1 iterations (one per tree level), much fewer than N VTXOs. + require.LessOrEqual(t, bulkBulk, int64(depth+1), + "bulk calls should equal tree depth (one per iteration)") + // Without bulk, individual calls == N (one per preconfirmed VTXO). + require.Equal(t, int64(n), noBulkSingle, + "without bulk, every VTXO triggers an individual call") +} + +// BenchmarkOffchainTxBulkVsSingle compares chain traversal with and without +// the bulk offchain tx prefetch, using simulated DB latency to make the +// round-trip reduction visible in wall-clock time. Uses a fanout tree +// (depth 8, 511 VTXOs) where each iteration processes an exponentially +// growing number of VTXOs — the bulk path does 9 round-trips vs 511 +// individual calls without it. +func BenchmarkOffchainTxBulkVsSingle(b *testing.B) { + const depth = 8 + const simulatedLatency = 50 * time.Microsecond + + n := (1 << (depth + 1)) - 1 + vtxoRepo := &benchVtxoRepo{vtxos: make(map[string]domain.Vtxo, n)} + innerRepo := &benchOffchainTxRepo{txs: make(map[string]*domain.OffchainTx, n)} + + for i := 0; i < n; i++ { + tid := benchTxid(i) + vtxoRepo.vtxos[fmt.Sprintf("%s:0", tid)] = domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: tid, VOut: 0}, + Preconfirmed: true, + ExpiresAt: int64(1000 + i), + } + left := 2*i + 1 + right := 2*i + 2 + if left < n && right < n { + innerRepo.txs[tid] = &domain.OffchainTx{ + ArkTxid: tid, + CheckpointTxs: map[string]string{ + fmt.Sprintf("cp-l-%d", i): benchCheckpointPSBT(benchTxid(left), 0), + fmt.Sprintf("cp-r-%d", i): benchCheckpointPSBT(benchTxid(right), 0), + }, + } + } else { + innerRepo.txs[tid] = &domain.OffchainTx{ + ArkTxid: tid, + CheckpointTxs: map[string]string{}, + } + } + } + + start := Outpoint{Txid: benchTxid(0), VOut: 0} + ctx := context.Background() + + b.Run(fmt.Sprintf("bulk_prefetch/%d_vtxos", n), func(b *testing.B) { + repo := &countingOffchainTxRepo{inner: innerRepo, latencyPerCall: simulatedLatency} + svc := &indexerService{repoManager: &benchRepoManager{ + vtxoRepo: vtxoRepo, offchainRepo: repo, + }} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + repo.reset() + _, err := svc.GetVtxoChain(ctx, "", start, nil, "") + if err != nil { + b.Fatal(err) + } + } + b.StopTimer() + b.ReportMetric(float64(repo.bulkCalls.Load())/float64(b.N), "bulk_calls/op") + b.ReportMetric(float64(repo.singleCalls.Load())/float64(b.N), "single_calls/op") + }) + + b.Run(fmt.Sprintf("no_bulk_fallback/%d_vtxos", n), func(b *testing.B) { + repo := &noBulkOffchainTxRepo{countingOffchainTxRepo{inner: innerRepo, latencyPerCall: simulatedLatency}} + svc := &indexerService{repoManager: &benchRepoManager{ + vtxoRepo: vtxoRepo, offchainRepo: repo, + }} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + repo.reset() + _, err := svc.GetVtxoChain(ctx, "", start, nil, "") + if err != nil { + b.Fatal(err) + } + } + b.StopTimer() + b.ReportMetric(float64(repo.bulkCalls.Load())/float64(b.N), "bulk_calls/op") + b.ReportMetric(float64(repo.singleCalls.Load())/float64(b.N), "single_calls/op") + }) +} + +// phaseTimings accumulates per-phase wall-clock time and call counts across +// the wrapped repo methods. Safe for concurrent recording. +type phaseTimings struct { + mu sync.Mutex + totals map[string]time.Duration + counts map[string]int +} + +func newPhaseTimings() *phaseTimings { + return &phaseTimings{ + totals: make(map[string]time.Duration), + counts: make(map[string]int), + } +} + +func (p *phaseTimings) record(phase string, d time.Duration) { + p.mu.Lock() + p.totals[phase] += d + p.counts[phase]++ + p.mu.Unlock() +} + +func (p *phaseTimings) log(t *testing.T, header string, wall time.Duration) { + t.Helper() + p.mu.Lock() + defer p.mu.Unlock() + + phases := make([]string, 0, len(p.totals)) + var repoTotal time.Duration + for name, d := range p.totals { + phases = append(phases, name) + repoTotal += d + } + sort.Strings(phases) + + t.Logf("%s", header) + t.Logf(" %-32s %12s", "wall clock (GetVtxoChain)", wall) + for _, name := range phases { + t.Logf(" %-32s %12s (%d calls)", name, p.totals[name], p.counts[name]) + } + t.Logf(" %-32s %12s", "sum of repo phases", repoTotal) + t.Logf(" %-32s %12s", "other (psbt parse + overhead)", wall-repoTotal) +} + +// timingVtxoRepo wraps a VtxoRepository and records per-call latency into a +// shared phaseTimings. An optional per-call latency simulates DB round-trip +// cost so the relative phase times are visible when running against fakes. +type timingVtxoRepo struct { + domain.VtxoRepository + inner domain.VtxoRepository + t *phaseTimings + latencyPerCall time.Duration +} + +func (r *timingVtxoRepo) GetVtxos( + ctx context.Context, outpoints []domain.Outpoint, +) ([]domain.Vtxo, error) { + start := time.Now() + defer func() { r.t.record("Vtxos.GetVtxos", time.Since(start)) }() + if r.latencyPerCall > 0 { + time.Sleep(r.latencyPerCall) + } + return r.inner.GetVtxos(ctx, outpoints) +} + +func (r *timingVtxoRepo) Close() {} + +type timingMarkerRepo struct { + domain.MarkerRepository + inner domain.MarkerRepository + t *phaseTimings + latencyPerCall time.Duration +} + +func (r *timingMarkerRepo) GetVtxoChainByMarkers( + ctx context.Context, markerIDs []string, +) ([]domain.Vtxo, error) { + start := time.Now() + defer func() { r.t.record("Markers.GetVtxoChainByMarkers", time.Since(start)) }() + if r.latencyPerCall > 0 { + time.Sleep(r.latencyPerCall) + } + return r.inner.GetVtxoChainByMarkers(ctx, markerIDs) +} + +func (r *timingMarkerRepo) GetMarkersByIds( + ctx context.Context, ids []string, +) ([]domain.Marker, error) { + start := time.Now() + defer func() { r.t.record("Markers.GetMarkersByIds", time.Since(start)) }() + if r.latencyPerCall > 0 { + time.Sleep(r.latencyPerCall) + } + return r.inner.GetMarkersByIds(ctx, ids) +} + +func (r *timingMarkerRepo) GetVtxosByMarker( + ctx context.Context, markerID string, +) ([]domain.Vtxo, error) { + start := time.Now() + defer func() { r.t.record("Markers.GetVtxosByMarker", time.Since(start)) }() + if r.latencyPerCall > 0 { + time.Sleep(r.latencyPerCall) + } + return r.inner.GetVtxosByMarker(ctx, markerID) +} + +func (r *timingMarkerRepo) Close() {} + +type timingOffchainTxRepo struct { + domain.OffchainTxRepository + inner domain.OffchainTxRepository + t *phaseTimings + latencyPerCall time.Duration +} + +func (r *timingOffchainTxRepo) GetOffchainTx( + ctx context.Context, txid string, +) (*domain.OffchainTx, error) { + start := time.Now() + defer func() { r.t.record("OffchainTxs.GetOffchainTx", time.Since(start)) }() + if r.latencyPerCall > 0 { + time.Sleep(r.latencyPerCall) + } + return r.inner.GetOffchainTx(ctx, txid) +} + +func (r *timingOffchainTxRepo) GetOffchainTxsByTxids( + ctx context.Context, txids []string, +) ([]*domain.OffchainTx, error) { + start := time.Now() + defer func() { r.t.record("OffchainTxs.GetOffchainTxsByTxids", time.Since(start)) }() + if r.latencyPerCall > 0 { + time.Sleep(r.latencyPerCall) + } + return r.inner.GetOffchainTxsByTxids(ctx, txids) +} + +func (r *timingOffchainTxRepo) AddOrUpdateOffchainTx( + _ context.Context, _ *domain.OffchainTx, +) error { + return nil +} + +func (r *timingOffchainTxRepo) Close() {} + +// TestVtxoChainTimingBreakdown builds a deep linear chain and runs +// GetVtxoChain against it with timing-decorated repos, logging a per-phase +// wall-clock breakdown. This is the in-process replacement for the server-side +// timing log that previously lived in walkVtxoChain. +// +// The repos use an in-memory backing store and inject a fixed per-call +// simulatedLatency via time.Sleep, so the absolute numbers in the breakdown +// do NOT reflect real DB cost — they are only meaningful as relative phase +// proportions under a uniform latency assumption. +// +// Run with: +// +// go test -v -run TestVtxoChainTimingBreakdown ./internal/core/application/... +func TestVtxoChainTimingBreakdown(t *testing.T) { + const ( + chainLen = 10000 + simulatedLatency = 50 * time.Microsecond + ) + + ctx := context.Background() + + // Reuse buildLinearChain to get the same data layout the perf test produces, + // then swap its repo manager for a timing-decorated one. + svc, start := buildLinearChain(chainLen, true) + inner := svc.repoManager.(*benchRepoManager) + + timings := newPhaseTimings() + svc.repoManager = &wrappedRepoManager{ + vtxos: &timingVtxoRepo{ + inner: inner.vtxoRepo, t: timings, latencyPerCall: simulatedLatency, + }, + markers: &timingMarkerRepo{ + inner: inner.markerRepo, t: timings, latencyPerCall: simulatedLatency, + }, + offchainTxs: &timingOffchainTxRepo{ + inner: inner.offchainRepo, t: timings, latencyPerCall: simulatedLatency, + }, + } + + wallStart := time.Now() + resp, err := svc.GetVtxoChain(ctx, "", start, nil, "") + wall := time.Since(wallStart) + require.NoError(t, err) + require.Equal(t, 2*chainLen-1, len(resp.Chain)) + + timings.log(t, fmt.Sprintf( + "GetVtxoChain timing breakdown: linear chain n=%d, simulated repo latency=%s", + chainLen, simulatedLatency, + ), wall) +} + +// wrappedRepoManager is a minimal RepoManager that exposes only the repos +// walkVtxoChain touches. Unwired accessors panic with a descriptive message +// instead of returning nil, so an accidental dependency on one of them +// surfaces as a clear failure rather than a nil-pointer dereference. +type wrappedRepoManager struct { + vtxos domain.VtxoRepository + markers domain.MarkerRepository + offchainTxs domain.OffchainTxRepository +} + +func (m *wrappedRepoManager) Events() domain.EventRepository { panic("Events: not wired") } +func (m *wrappedRepoManager) Rounds() domain.RoundRepository { panic("Rounds: not wired") } +func (m *wrappedRepoManager) Vtxos() domain.VtxoRepository { return m.vtxos } +func (m *wrappedRepoManager) Markers() domain.MarkerRepository { + return m.markers +} +func (m *wrappedRepoManager) ScheduledSession() domain.ScheduledSessionRepo { + panic("ScheduledSession: not wired") +} +func (m *wrappedRepoManager) OffchainTxs() domain.OffchainTxRepository { return m.offchainTxs } +func (m *wrappedRepoManager) Convictions() domain.ConvictionRepository { + panic("Convictions: not wired") +} +func (m *wrappedRepoManager) Assets() domain.AssetRepository { panic("Assets: not wired") } +func (m *wrappedRepoManager) Fees() domain.FeeRepository { panic("Fees: not wired") } +func (m *wrappedRepoManager) Close() {} diff --git a/internal/core/application/indexer_exposure_test.go b/internal/core/application/indexer_exposure_test.go index 9ca69d6f6..0edf9cdaa 100644 --- a/internal/core/application/indexer_exposure_test.go +++ b/internal/core/application/indexer_exposure_test.go @@ -585,6 +585,71 @@ func TestGetVtxoChain(t *testing.T) { rounds.AssertExpectations(t) vtxos.AssertExpectations(t) }) + + t.Run("preconfirmed chain bulk-loads offchain txs", func(t *testing.T) { + vtxoOutpoint := Outpoint{Txid: testTxids[0], VOut: 0} + offchainTxid := vtxoOutpoint.Txid + checkpointB64 := buildCheckpointTxSpending(t, vtxoOutpoint.Txid, vtxoOutpoint.VOut) + + vtxos := &mockedVtxoRepo{} + vtxos.On("GetVtxos", mock.Anything, []domain.Outpoint{vtxoOutpoint}). + Return([]domain.Vtxo{{ + Outpoint: domain.Outpoint{Txid: vtxoOutpoint.Txid, VOut: vtxoOutpoint.VOut}, + Preconfirmed: true, + }}, nil) + + offchainRepo := &mockedOffchainTxRepo{} + offchainRepo.On("GetOffchainTxsByTxids", mock.Anything, []string{offchainTxid}). + Return([]*domain.OffchainTx{{ + ArkTxid: offchainTxid, + CheckpointTxs: map[string]string{ + "cp": checkpointB64, + }, + }}, nil) + + indexer := newTestIndexer(t, privkey, exposurePrivate, nil, vtxos, nil, offchainRepo) + + chain, _, _, err := indexer.walkVtxoChain(t.Context(), []domain.Outpoint{vtxoOutpoint}, 1000) + require.NoError(t, err) + require.NotEmpty(t, chain) + + offchainRepo.AssertNotCalled(t, "GetOffchainTx", mock.Anything, offchainTxid) + offchainRepo.AssertExpectations(t) + vtxos.AssertExpectations(t) + }) + + t.Run("preconfirmed chain falls back to single fetch on cache miss", func(t *testing.T) { + vtxoOutpoint := Outpoint{Txid: testTxids[0], VOut: 0} + offchainTxid := vtxoOutpoint.Txid + checkpointB64 := buildCheckpointTxSpending(t, vtxoOutpoint.Txid, vtxoOutpoint.VOut) + + vtxos := &mockedVtxoRepo{} + vtxos.On("GetVtxos", mock.Anything, []domain.Outpoint{vtxoOutpoint}). + Return([]domain.Vtxo{{ + Outpoint: domain.Outpoint{Txid: vtxoOutpoint.Txid, VOut: vtxoOutpoint.VOut}, + Preconfirmed: true, + }}, nil) + + offchainRepo := &mockedOffchainTxRepo{} + offchainRepo.On("GetOffchainTxsByTxids", mock.Anything, []string{offchainTxid}). + Return([]*domain.OffchainTx{}, nil) + offchainRepo.On("GetOffchainTx", mock.Anything, offchainTxid). + Return(&domain.OffchainTx{ + ArkTxid: offchainTxid, + CheckpointTxs: map[string]string{ + "cp": checkpointB64, + }, + }, nil) + + indexer := newTestIndexer(t, privkey, exposurePrivate, nil, vtxos, nil, offchainRepo) + + chain, _, _, err := indexer.walkVtxoChain(t.Context(), []domain.Outpoint{vtxoOutpoint}, 1000) + require.NoError(t, err) + require.NotEmpty(t, chain) + + offchainRepo.AssertExpectations(t) + vtxos.AssertExpectations(t) + }) }) t.Run("invalid", func(t *testing.T) { @@ -927,6 +992,7 @@ func TestStripSignerSignatures(t *testing.T) { func newTestIndexer( t *testing.T, privkey *btcec.PrivateKey, exposure exposure, rounds *mockedRoundRepo, vtxos *mockedVtxoRepo, wallet *mockedWallet, + offchainRepos ...*mockedOffchainTxRepo, ) *indexerService { t.Helper() @@ -940,6 +1006,9 @@ func newTestIndexer( if vtxos != nil { repo.On("Vtxos").Return(vtxos) } + if len(offchainRepos) > 0 && offchainRepos[0] != nil { + repo.On("OffchainTxs").Return(offchainRepos[0]) + } cache := newTokenCache(defaultAuthTokenTTL) t.Cleanup(cache.close) @@ -995,6 +1064,25 @@ func buildTestTreeTxs(t *testing.T) (rootTxid, leafTxid string, flatTree arktree return } +func buildCheckpointTxSpending(t *testing.T, prevTxid string, prevVout uint32) string { + t.Helper() + + prevHash, err := chainhash.NewHashFromStr(prevTxid) + require.NoError(t, err) + + ptx, err := psbt.New( + []*wire.OutPoint{{Hash: *prevHash, Index: prevVout}}, + []*wire.TxOut{{Value: 1000, PkScript: []byte{txscript.OP_TRUE}}}, + 2, 0, []uint32{wire.MaxTxInSequenceNum}, + ) + require.NoError(t, err) + + b64, err := ptx.B64Encode() + require.NoError(t, err) + + return b64 +} + // buildTestIntent creates a valid signed intent proof that passes intent.Verify. // It builds a MultisigClosure with vtxoKey, derives the taproot output key from // that closure, signs input 1, and returns the intent plus the taproot key (so @@ -1152,6 +1240,36 @@ func (m *mockedRepoManager) Vtxos() domain.VtxoRepository { return nil } +func (m *mockedRepoManager) OffchainTxs() domain.OffchainTxRepository { + if v := m.Called().Get(0); v != nil { + return v.(domain.OffchainTxRepository) + } + return nil +} + +type mockedOffchainTxRepo struct { + mock.Mock + domain.OffchainTxRepository // unimplemented methods panic on call +} + +func (m *mockedOffchainTxRepo) GetOffchainTx(ctx context.Context, txid string) (*domain.OffchainTx, error) { + args := m.Called(ctx, txid) + if v := args.Get(0); v != nil { + return v.(*domain.OffchainTx), args.Error(1) + } + return nil, args.Error(1) +} + +func (m *mockedOffchainTxRepo) GetOffchainTxsByTxids( + ctx context.Context, txids []string, +) ([]*domain.OffchainTx, error) { + args := m.Called(ctx, txids) + if v := args.Get(0); v != nil { + return v.([]*domain.OffchainTx), args.Error(1) + } + return nil, args.Error(1) +} + type mockedWallet struct { mock.Mock ports.WalletService // unimplemented methods panic on call diff --git a/internal/core/application/indexer_test.go b/internal/core/application/indexer_test.go index 3b3ed31b5..d5861ebc8 100644 --- a/internal/core/application/indexer_test.go +++ b/internal/core/application/indexer_test.go @@ -303,6 +303,16 @@ func (m *mockOffchainTxRepoForIndexer) GetOffchainTx( return args.Get(0).(*domain.OffchainTx), args.Error(1) } +func (m *mockOffchainTxRepoForIndexer) GetOffchainTxsByTxids( + ctx context.Context, txids []string, +) ([]*domain.OffchainTx, error) { + args := m.Called(ctx, txids) + if args.Get(0) == nil { + return nil, args.Error(1) + } + return args.Get(0).([]*domain.OffchainTx), args.Error(1) +} + func (m *mockOffchainTxRepoForIndexer) AddOrUpdateOffchainTx( ctx context.Context, offchainTx *domain.OffchainTx, ) error { @@ -361,6 +371,10 @@ func newChainTestIndexerWithOffchain() ( vtxoRepo := &mockVtxoRepoForIndexer{} markerRepo := &mockMarkerRepoForIndexer{} offchainTxRepo := &mockOffchainTxRepoForIndexer{} + // Default: bulk fetch returns empty so the fallback to GetOffchainTx is used. + // Tests that want to verify bulk behavior can override with a more specific expectation. + offchainTxRepo.On("GetOffchainTxsByTxids", mock.Anything, mock.Anything). + Return([]*domain.OffchainTx{}, nil).Maybe() repoManager := &mockRepoManagerForIndexer{ vtxos: vtxoRepo, markers: markerRepo, offchainTxs: offchainTxRepo, } @@ -737,12 +751,23 @@ func setupPreconfirmedChain( cpA := makeCheckpointPSBT(t, txidB, 0) cpB := makeCheckpointPSBT(t, txidC, 0) + offchainTxA := &domain.OffchainTx{ArkTxid: txidA, CheckpointTxs: map[string]string{"cp-a": cpA}} + offchainTxB := &domain.OffchainTx{ArkTxid: txidB, CheckpointTxs: map[string]string{"cp-b": cpB}} + offchainTxC := &domain.OffchainTx{ArkTxid: txidC, CheckpointTxs: map[string]string{}} + + offchainTxRepo.On("GetOffchainTxsByTxids", ctx, []string{txidA}). + Return([]*domain.OffchainTx{offchainTxA}, nil).Maybe() + offchainTxRepo.On("GetOffchainTxsByTxids", ctx, []string{txidB}). + Return([]*domain.OffchainTx{offchainTxB}, nil).Maybe() + offchainTxRepo.On("GetOffchainTxsByTxids", ctx, []string{txidC}). + Return([]*domain.OffchainTx{offchainTxC}, nil).Maybe() + offchainTxRepo.On("GetOffchainTx", ctx, txidA). - Return(&domain.OffchainTx{CheckpointTxs: map[string]string{"cp-a": cpA}}, nil) + Return(offchainTxA, nil).Maybe() offchainTxRepo.On("GetOffchainTx", ctx, txidB). - Return(&domain.OffchainTx{CheckpointTxs: map[string]string{"cp-b": cpB}}, nil) + Return(offchainTxB, nil).Maybe() offchainTxRepo.On("GetOffchainTx", ctx, txidC). - Return(&domain.OffchainTx{CheckpointTxs: map[string]string{}}, nil) + Return(offchainTxC, nil).Maybe() return Outpoint{Txid: txidA, VOut: 0} } @@ -826,8 +851,11 @@ func TestGetVtxoChain_ShortChainNoToken(t *testing.T) { Return([]domain.Vtxo{vtxo}, nil) markerRepo.On("GetVtxosByMarker", ctx, mock.Anything). Return([]domain.Vtxo{}, nil).Maybe() + offchainTxA := &domain.OffchainTx{ArkTxid: txidA, CheckpointTxs: map[string]string{}} + offchainTxRepo.On("GetOffchainTxsByTxids", ctx, []string{txidA}). + Return([]*domain.OffchainTx{offchainTxA}, nil) offchainTxRepo.On("GetOffchainTx", ctx, txidA). - Return(&domain.OffchainTx{CheckpointTxs: map[string]string{}}, nil) + Return(offchainTxA, nil).Maybe() // Page size larger than chain page := &Page{PageSize: 100} @@ -888,7 +916,7 @@ func matchOutpoints(expected ...domain.Outpoint) interface{} { // matchIDs returns a mock.MatchedBy matcher that matches a []string argument // containing exactly the given IDs, regardless of order. This avoids flakes from -// non-deterministic map iteration in preloadVtxosByMarkers. +// non-deterministic map iteration in preloadByMarkers. func matchIDs(expected ...string) interface{} { sorted := make([]string, len(expected)) copy(sorted, expected) @@ -909,7 +937,7 @@ func matchIDs(expected ...string) interface{} { }) } -// TestPreloadVtxosByMarkers_WalksMarkerChain verifies that preloadVtxosByMarkers +// TestPreloadVtxosByMarkers_WalksMarkerChain verifies that preloadByMarkers // follows the marker DAG upward and populates the cache with all discovered VTXOs. func TestPreloadVtxosByMarkers_WalksMarkerChain(t *testing.T) { _, markerRepo, indexer := newChainTestIndexer() @@ -952,7 +980,8 @@ func TestPreloadVtxosByMarkers_WalksMarkerChain(t *testing.T) { }, nil) cache := make(map[string]domain.Vtxo) - err := indexer.preloadVtxosByMarkers(ctx, []domain.Vtxo{vtxoLeaf}, cache) + offchainCache := make(map[string]*domain.OffchainTx) + err := indexer.preloadByMarkers(ctx, []domain.Vtxo{vtxoLeaf}, cache, offchainCache) require.NoError(t, err) // Cache should contain the seed vtxo plus all vtxos from all marker levels. @@ -999,7 +1028,8 @@ func TestPreloadVtxosByMarkers_NoCycleLoop(t *testing.T) { }, nil) cache := make(map[string]domain.Vtxo) - err := indexer.preloadVtxosByMarkers(ctx, []domain.Vtxo{vtxo}, cache) + offchainCache := make(map[string]*domain.OffchainTx) + err := indexer.preloadByMarkers(ctx, []domain.Vtxo{vtxo}, cache, offchainCache) require.NoError(t, err) // Should terminate without looping forever. @@ -1013,7 +1043,7 @@ func TestPreloadVtxosByMarkers_NoCycleLoop(t *testing.T) { } // TestGetVtxoChain_WithMarkers_UsesPreload verifies that GetVtxoChain uses -// preloadVtxosByMarkers when VTXOs have markers, and that the main loop +// preloadByMarkers when VTXOs have markers, and that the main loop // hits the cache instead of making additional DB calls. func TestGetVtxoChain_WithMarkers_UsesPreload(t *testing.T) { vtxoRepo, markerRepo, offchainTxRepo, indexer := newChainTestIndexerWithOffchain() diff --git a/internal/core/domain/offchain_tx_repo.go b/internal/core/domain/offchain_tx_repo.go index 87615d7a1..094bad62c 100644 --- a/internal/core/domain/offchain_tx_repo.go +++ b/internal/core/domain/offchain_tx_repo.go @@ -5,5 +5,6 @@ import "context" type OffchainTxRepository interface { AddOrUpdateOffchainTx(ctx context.Context, offchainTx *OffchainTx) error GetOffchainTx(ctx context.Context, txid string) (*OffchainTx, error) + GetOffchainTxsByTxids(ctx context.Context, txids []string) ([]*OffchainTx, error) Close() } diff --git a/internal/infrastructure/db/badger/ark_repo.go b/internal/infrastructure/db/badger/ark_repo.go index 3bb6c34ef..0ca0a92cd 100644 --- a/internal/infrastructure/db/badger/ark_repo.go +++ b/internal/infrastructure/db/badger/ark_repo.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path/filepath" + "strings" "time" "github.com/arkade-os/arkd/internal/core/domain" @@ -234,6 +235,28 @@ func (r *arkRepository) GetOffchainTx( return r.getOffchainTx(ctx, txid) } +func (r *arkRepository) GetOffchainTxsByTxids( + ctx context.Context, txids []string, +) ([]*domain.OffchainTx, error) { + if len(txids) == 0 { + return []*domain.OffchainTx{}, nil + } + + txs := make([]*domain.OffchainTx, 0, len(txids)) + for _, txid := range txids { + tx, err := r.getOffchainTx(ctx, txid) + if err != nil { + if strings.Contains(err.Error(), "not found") { + continue + } + return nil, err + } + txs = append(txs, tx) + } + + return txs, nil +} + func (r *arkRepository) Close() { // nolint r.store.Close() diff --git a/internal/infrastructure/db/postgres/migration/20260409140000_checkpoint_tx_offchain_txid_index.down.sql b/internal/infrastructure/db/postgres/migration/20260409140000_checkpoint_tx_offchain_txid_index.down.sql new file mode 100644 index 000000000..3bf97317f --- /dev/null +++ b/internal/infrastructure/db/postgres/migration/20260409140000_checkpoint_tx_offchain_txid_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_checkpoint_tx_offchain_txid; diff --git a/internal/infrastructure/db/postgres/migration/20260409140000_checkpoint_tx_offchain_txid_index.up.sql b/internal/infrastructure/db/postgres/migration/20260409140000_checkpoint_tx_offchain_txid_index.up.sql new file mode 100644 index 000000000..4fddcadc2 --- /dev/null +++ b/internal/infrastructure/db/postgres/migration/20260409140000_checkpoint_tx_offchain_txid_index.up.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS idx_checkpoint_tx_offchain_txid + ON checkpoint_tx (offchain_txid); diff --git a/internal/infrastructure/db/postgres/offchain_tx_repo.go b/internal/infrastructure/db/postgres/offchain_tx_repo.go index 9f41ea76c..a45be1dbc 100644 --- a/internal/infrastructure/db/postgres/offchain_tx_repo.go +++ b/internal/infrastructure/db/postgres/offchain_tx_repo.go @@ -114,6 +114,62 @@ func (v *offchainTxRepository) GetOffchainTx( }, nil } +func (v *offchainTxRepository) GetOffchainTxsByTxids( + ctx context.Context, txids []string, +) ([]*domain.OffchainTx, error) { + if len(txids) == 0 { + return []*domain.OffchainTx{}, nil + } + + rows, err := v.querier.SelectOffchainTxsByTxids(ctx, txids) + if err != nil { + return nil, err + } + + grouped := make(map[string][]queries.OffchainTxVw) + for _, row := range rows { + grouped[row.OffchainTxVw.Txid] = append(grouped[row.OffchainTxVw.Txid], row.OffchainTxVw) + } + + txs := make([]*domain.OffchainTx, 0, len(grouped)) + for _, vws := range grouped { + vt := vws[0] + checkpointTxs := make(map[string]string) + commitmentTxids := make(map[string]string) + rootCommitmentTxId := "" + for _, vw := range vws { + if vw.CheckpointTxid.Valid && vw.CheckpointTx.Valid { + checkpointTxs[vw.CheckpointTxid.String] = vw.CheckpointTx.String + commitmentTxids[vw.CheckpointTxid.String] = vw.CommitmentTxid.String + if vw.IsRootCommitmentTxid.Valid && vw.IsRootCommitmentTxid.Bool { + rootCommitmentTxId = vw.CommitmentTxid.String + } + } + } + stage := domain.Stage{Code: int(vt.StageCode)} + if vt.FailReason.String != "" { + stage.Failed = true + } + if domain.OffchainTxStage(vt.StageCode) == domain.OffchainTxFinalizedStage { + stage.Ended = true + } + txs = append(txs, &domain.OffchainTx{ + ArkTxid: vt.Txid, + ArkTx: vt.Tx, + StartingTimestamp: vt.StartingTimestamp, + EndingTimestamp: vt.EndingTimestamp, + ExpiryTimestamp: vt.ExpiryTimestamp, + FailReason: vt.FailReason.String, + Stage: stage, + CheckpointTxs: checkpointTxs, + CommitmentTxids: commitmentTxids, + RootCommitmentTxId: rootCommitmentTxId, + }) + } + + return txs, nil +} + func (v *offchainTxRepository) Close() { _ = v.db.Close() } diff --git a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go index 67b7b1060..c3d8c4dc5 100644 --- a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go @@ -876,6 +876,50 @@ func (q *Queries) SelectOffchainTx(ctx context.Context, txid string) ([]SelectOf return items, nil } +const selectOffchainTxsByTxids = `-- name: SelectOffchainTxsByTxids :many +SELECT offchain_tx_vw.txid, offchain_tx_vw.tx, offchain_tx_vw.starting_timestamp, offchain_tx_vw.ending_timestamp, offchain_tx_vw.expiry_timestamp, offchain_tx_vw.fail_reason, offchain_tx_vw.stage_code, offchain_tx_vw.checkpoint_txid, offchain_tx_vw.checkpoint_tx, offchain_tx_vw.commitment_txid, offchain_tx_vw.is_root_commitment_txid, offchain_tx_vw.offchain_txid FROM offchain_tx_vw WHERE txid = ANY($1::varchar[]) AND COALESCE(fail_reason, '') = '' +` + +type SelectOffchainTxsByTxidsRow struct { + OffchainTxVw OffchainTxVw +} + +func (q *Queries) SelectOffchainTxsByTxids(ctx context.Context, txids []string) ([]SelectOffchainTxsByTxidsRow, error) { + rows, err := q.db.QueryContext(ctx, selectOffchainTxsByTxids, pq.Array(txids)) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SelectOffchainTxsByTxidsRow + for rows.Next() { + var i SelectOffchainTxsByTxidsRow + if err := rows.Scan( + &i.OffchainTxVw.Txid, + &i.OffchainTxVw.Tx, + &i.OffchainTxVw.StartingTimestamp, + &i.OffchainTxVw.EndingTimestamp, + &i.OffchainTxVw.ExpiryTimestamp, + &i.OffchainTxVw.FailReason, + &i.OffchainTxVw.StageCode, + &i.OffchainTxVw.CheckpointTxid, + &i.OffchainTxVw.CheckpointTx, + &i.OffchainTxVw.CommitmentTxid, + &i.OffchainTxVw.IsRootCommitmentTxid, + &i.OffchainTxVw.OffchainTxid, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const selectPendingSpentVtxo = `-- name: SelectPendingSpentVtxo :many SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.markers, v.commitments, v.swept, v.asset_id, v.asset_amount FROM vtxo_vw v diff --git a/internal/infrastructure/db/postgres/sqlc/query.sql b/internal/infrastructure/db/postgres/sqlc/query.sql index 47c335993..33747e6e6 100644 --- a/internal/infrastructure/db/postgres/sqlc/query.sql +++ b/internal/infrastructure/db/postgres/sqlc/query.sql @@ -277,6 +277,9 @@ WHERE EXISTS ( -- name: SelectOffchainTx :many SELECT sqlc.embed(offchain_tx_vw) FROM offchain_tx_vw WHERE txid = @txid AND COALESCE(fail_reason, '') = ''; +-- name: SelectOffchainTxsByTxids :many +SELECT sqlc.embed(offchain_tx_vw) FROM offchain_tx_vw WHERE txid = ANY(@txids::varchar[]) AND COALESCE(fail_reason, '') = ''; + -- name: SelectLatestScheduledSession :one SELECT * FROM scheduled_session ORDER BY updated_at DESC LIMIT 1; diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index 5b23d0369..5862a12be 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -3434,6 +3434,67 @@ func testOffchainTxRepository(t *testing.T, svc ports.RepoManager) { require.NotNil(t, offchainTx) require.True(t, gotOffchainTx.IsFinalized()) require.Condition(t, offchainTxMatch(*offchainTx, *gotOffchainTx)) + + bulkFetchedTxs, err := repo.GetOffchainTxsByTxids(ctx, []string{arkTxid}) + require.NoError(t, err) + require.Len(t, bulkFetchedTxs, 1) + require.Equal(t, arkTxid, bulkFetchedTxs[0].ArkTxid) + + bulkFetchedTxs, err = repo.GetOffchainTxsByTxids(ctx, []string{"missing-txid"}) + require.NoError(t, err) + require.Empty(t, bulkFetchedTxs) + + // Insert a second offchain tx so we can exercise multi-txid bulk fetch. + secondArkTxid := txidb + secondCheckpointTxid := "0000000000000000000000000000000000000000000000000000000000000005" + secondCheckpointPtx := "cHNldP8BAgQCAAAAAQQBAAEFAQABBgEDAfsEAgAAAAA=signed-2" + secondEvents := []domain.Event{ + domain.OffchainTxRequested{ + OffchainTxEvent: domain.OffchainTxEvent{ + Id: secondArkTxid, + Type: domain.EventTypeOffchainTxRequested, + }, + StartingTimestamp: now.Unix(), + }, + domain.OffchainTxAccepted{ + OffchainTxEvent: domain.OffchainTxEvent{ + Id: secondArkTxid, + Type: domain.EventTypeOffchainTxAccepted, + }, + CommitmentTxids: map[string]string{ + secondCheckpointTxid: rootCommitmentTxid, + }, + SignedCheckpointTxs: map[string]string{ + secondCheckpointTxid: secondCheckpointPtx, + }, + RootCommitmentTxid: rootCommitmentTxid, + }, + } + secondOffchainTx := domain.NewOffchainTxFromEvents(secondEvents) + require.NoError(t, repo.AddOrUpdateOffchainTx(ctx, secondOffchainTx)) + + // Multi-txid fetch returns both, plus tolerates a missing entry. + bulkFetchedTxs, err = repo.GetOffchainTxsByTxids( + ctx, []string{arkTxid, secondArkTxid, "missing-txid"}, + ) + require.NoError(t, err) + require.Len(t, bulkFetchedTxs, 2) + + got := make(map[string]*domain.OffchainTx, len(bulkFetchedTxs)) + for _, tx := range bulkFetchedTxs { + got[tx.ArkTxid] = tx + } + require.Contains(t, got, arkTxid) + require.Contains(t, got, secondArkTxid) + + // Each result must carry its own checkpoint mapping — guards the + // row-grouping logic against cross-txid contamination. + require.Contains(t, got[arkTxid].CheckpointTxs, checkpointTxid1) + require.Contains(t, got[arkTxid].CheckpointTxs, checkpointTxid2) + require.NotContains(t, got[arkTxid].CheckpointTxs, secondCheckpointTxid) + require.Contains(t, got[secondArkTxid].CheckpointTxs, secondCheckpointTxid) + require.NotContains(t, got[secondArkTxid].CheckpointTxs, checkpointTxid1) + require.NotContains(t, got[secondArkTxid].CheckpointTxs, checkpointTxid2) }) } diff --git a/internal/infrastructure/db/sqlite/migration/20260409140000_checkpoint_tx_offchain_txid_index.down.sql b/internal/infrastructure/db/sqlite/migration/20260409140000_checkpoint_tx_offchain_txid_index.down.sql new file mode 100644 index 000000000..3bf97317f --- /dev/null +++ b/internal/infrastructure/db/sqlite/migration/20260409140000_checkpoint_tx_offchain_txid_index.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS idx_checkpoint_tx_offchain_txid; diff --git a/internal/infrastructure/db/sqlite/migration/20260409140000_checkpoint_tx_offchain_txid_index.up.sql b/internal/infrastructure/db/sqlite/migration/20260409140000_checkpoint_tx_offchain_txid_index.up.sql new file mode 100644 index 000000000..4fddcadc2 --- /dev/null +++ b/internal/infrastructure/db/sqlite/migration/20260409140000_checkpoint_tx_offchain_txid_index.up.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS idx_checkpoint_tx_offchain_txid + ON checkpoint_tx (offchain_txid); diff --git a/internal/infrastructure/db/sqlite/offchain_tx_repo.go b/internal/infrastructure/db/sqlite/offchain_tx_repo.go index c9df73897..3f30b7d85 100644 --- a/internal/infrastructure/db/sqlite/offchain_tx_repo.go +++ b/internal/infrastructure/db/sqlite/offchain_tx_repo.go @@ -9,6 +9,11 @@ import ( "github.com/arkade-os/arkd/internal/infrastructure/db/sqlite/sqlc/queries" ) +// sqliteMaxBulkTxids caps the per-query batch for GetOffchainTxsByTxids to stay +// well under SQLITE_MAX_VARIABLE_NUMBER (default 999 on SQLite < 3.32). The +// SLICE expansion in the generated query emits one bound parameter per txid. +const sqliteMaxBulkTxids = 500 + type offchainTxRepository struct { db *sql.DB querier *queries.Queries @@ -114,6 +119,67 @@ func (v *offchainTxRepository) GetOffchainTx( }, nil } +func (v *offchainTxRepository) GetOffchainTxsByTxids( + ctx context.Context, txids []string, +) ([]*domain.OffchainTx, error) { + if len(txids) == 0 { + return []*domain.OffchainTx{}, nil + } + + grouped := make(map[string][]queries.OffchainTxVw) + for start := 0; start < len(txids); start += sqliteMaxBulkTxids { + end := min(start+sqliteMaxBulkTxids, len(txids)) + rows, err := v.querier.SelectOffchainTxsByTxids(ctx, txids[start:end]) + if err != nil { + return nil, err + } + for _, row := range rows { + grouped[row.OffchainTxVw.Txid] = append( + grouped[row.OffchainTxVw.Txid], + row.OffchainTxVw, + ) + } + } + + txs := make([]*domain.OffchainTx, 0, len(grouped)) + for _, vws := range grouped { + vt := vws[0] + checkpointTxs := make(map[string]string) + commitmentTxids := make(map[string]string) + rootCommitmentTxId := "" + for _, vw := range vws { + if vw.CheckpointTxid != "" && vw.CheckpointTx != "" { + checkpointTxs[vw.CheckpointTxid] = vw.CheckpointTx + commitmentTxids[vw.CheckpointTxid] = vw.CommitmentTxid.String + if vw.IsRootCommitmentTxid.Bool { + rootCommitmentTxId = vw.CommitmentTxid.String + } + } + } + stage := domain.Stage{Code: int(vt.StageCode)} + if vt.FailReason.String != "" { + stage.Failed = true + } + if domain.OffchainTxStage(vt.StageCode) == domain.OffchainTxFinalizedStage { + stage.Ended = true + } + txs = append(txs, &domain.OffchainTx{ + ArkTxid: vt.Txid, + ArkTx: vt.Tx, + StartingTimestamp: vt.StartingTimestamp, + EndingTimestamp: vt.EndingTimestamp, + ExpiryTimestamp: vt.ExpiryTimestamp, + FailReason: vt.FailReason.String, + Stage: stage, + CheckpointTxs: checkpointTxs, + CommitmentTxids: commitmentTxids, + RootCommitmentTxId: rootCommitmentTxId, + }) + } + + return txs, nil +} + func (v *offchainTxRepository) Close() { _ = v.db.Close() } diff --git a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go index 7bf84f46b..15adcd901 100644 --- a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go @@ -936,6 +936,60 @@ func (q *Queries) SelectOffchainTx(ctx context.Context, txid string) ([]SelectOf return items, nil } +const selectOffchainTxsByTxids = `-- name: SelectOffchainTxsByTxids :many +SELECT offchain_tx_vw.txid, offchain_tx_vw.tx, offchain_tx_vw.starting_timestamp, offchain_tx_vw.ending_timestamp, offchain_tx_vw.expiry_timestamp, offchain_tx_vw.fail_reason, offchain_tx_vw.stage_code, offchain_tx_vw.checkpoint_txid, offchain_tx_vw.checkpoint_tx, offchain_tx_vw.commitment_txid, offchain_tx_vw.is_root_commitment_txid, offchain_tx_vw.offchain_txid FROM offchain_tx_vw WHERE txid IN (/*SLICE:txids*/?) AND COALESCE(fail_reason, '') = '' +` + +type SelectOffchainTxsByTxidsRow struct { + OffchainTxVw OffchainTxVw +} + +func (q *Queries) SelectOffchainTxsByTxids(ctx context.Context, txids []string) ([]SelectOffchainTxsByTxidsRow, error) { + query := selectOffchainTxsByTxids + var queryParams []interface{} + if len(txids) > 0 { + for _, v := range txids { + queryParams = append(queryParams, v) + } + query = strings.Replace(query, "/*SLICE:txids*/?", strings.Repeat(",?", len(txids))[1:], 1) + } else { + query = strings.Replace(query, "/*SLICE:txids*/?", "NULL", 1) + } + rows, err := q.db.QueryContext(ctx, query, queryParams...) + if err != nil { + return nil, err + } + defer rows.Close() + var items []SelectOffchainTxsByTxidsRow + for rows.Next() { + var i SelectOffchainTxsByTxidsRow + if err := rows.Scan( + &i.OffchainTxVw.Txid, + &i.OffchainTxVw.Tx, + &i.OffchainTxVw.StartingTimestamp, + &i.OffchainTxVw.EndingTimestamp, + &i.OffchainTxVw.ExpiryTimestamp, + &i.OffchainTxVw.FailReason, + &i.OffchainTxVw.StageCode, + &i.OffchainTxVw.CheckpointTxid, + &i.OffchainTxVw.CheckpointTx, + &i.OffchainTxVw.CommitmentTxid, + &i.OffchainTxVw.IsRootCommitmentTxid, + &i.OffchainTxVw.OffchainTxid, + ); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const selectPendingSpentVtxo = `-- name: SelectPendingSpentVtxo :many SELECT v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.preconfirmed, v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.markers, v.commitments, v.swept, v.asset_id, v.asset_amount FROM vtxo_vw v diff --git a/internal/infrastructure/db/sqlite/sqlc/query.sql b/internal/infrastructure/db/sqlite/sqlc/query.sql index 47b4badea..204c96fe8 100644 --- a/internal/infrastructure/db/sqlite/sqlc/query.sql +++ b/internal/infrastructure/db/sqlite/sqlc/query.sql @@ -283,6 +283,9 @@ WHERE EXISTS ( -- name: SelectOffchainTx :many SELECT sqlc.embed(offchain_tx_vw) FROM offchain_tx_vw WHERE txid = @txid AND COALESCE(fail_reason, '') = ''; +-- name: SelectOffchainTxsByTxids :many +SELECT sqlc.embed(offchain_tx_vw) FROM offchain_tx_vw WHERE txid IN (sqlc.slice('txids')) AND COALESCE(fail_reason, '') = ''; + -- name: SelectLatestScheduledSession :one SELECT * FROM scheduled_session ORDER BY updated_at DESC LIMIT 1; diff --git a/internal/test/e2e/vtxo_chain_test.go b/internal/test/e2e/vtxo_chain_test.go new file mode 100644 index 000000000..575590588 --- /dev/null +++ b/internal/test/e2e/vtxo_chain_test.go @@ -0,0 +1,176 @@ +package e2e_test + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "net/http" + "sync" + "testing" + "time" + + arksdk "github.com/arkade-os/arkd/pkg/client-lib" + grpcindexer "github.com/arkade-os/arkd/pkg/client-lib/indexer/grpc" + "github.com/arkade-os/arkd/pkg/client-lib/store" + "github.com/arkade-os/arkd/pkg/client-lib/types" + "github.com/btcsuite/btcd/btcec/v2" + "github.com/stretchr/testify/require" +) + +var ( + chainLength = flag.Int("chain-length", 10, "Number of self-send hops in the VTXO chain") + initialAmount = flag.Int("initial-amount", 1000, "Initial funding amount in satoshis") + arkServerUrl = flag.String("server-url", serverUrl, "Ark server gRPC address") + arkAdminUrl = flag.String("admin-url", adminUrl, "Ark admin HTTP address") + walletSeed = flag.String("seed", "", "Wallet private key hex (random if empty)") + skipChain = flag.Bool("skip-chain", false, "Skip chain creation, only run GetVtxoChain on existing wallet") +) + +// TestVtxoChain creates a long VTXO chain by repeatedly self-sending. +// Run with: +// +// go test -v -run TestVtxoChain -args -chain-length=50 -initial-amount=10000 +func TestVtxoChain(t *testing.T) { + if !flag.Parsed() { + flag.Parse() + } + + ctx := t.Context() + + appDataStore, err := store.NewStore(store.Config{ + ConfigStoreType: types.InMemoryStore, + }) + require.NoError(t, err) + + client, err := arksdk.NewArkClient(appDataStore) + require.NoError(t, err) + t.Cleanup(client.Stop) + + seed := *walletSeed + if seed == "" { + privkey, err := btcec.NewPrivateKey() + require.NoError(t, err) + seed = hex.EncodeToString(privkey.Serialize()) + } + t.Logf("wallet seed: %s", seed) + + err = client.Init(ctx, arksdk.InitArgs{ + WalletType: arksdk.SingleKeyWallet, + ServerUrl: *arkServerUrl, + Password: password, + Seed: seed, + ExplorerURL: explorerUrl, + }) + require.NoError(t, err) + + err = client.Unlock(ctx, password) + require.NoError(t, err) + + _, offchainAddr, _, err := client.Receive(ctx) + require.NoError(t, err) + + if !*skipChain { + // Fund the client offchain via admin note. + note := chainGenerateNote(t, uint64(*initialAmount)) + + wg := &sync.WaitGroup{} + var notifyErr error + wg.Go(func() { + _, notifyErr = client.NotifyIncomingFunds(ctx, offchainAddr.Address) + }) + + redeemTxid, err := client.RedeemNotes(ctx, []string{note}) + require.NoError(t, err) + require.NotEmpty(t, redeemTxid) + + wg.Wait() + require.NoError(t, notifyErr) + + time.Sleep(time.Second) + + spendable, _, err := client.ListVtxos(ctx) + require.NoError(t, err) + require.NotEmpty(t, spendable, "no spendable VTXOs after faucet") + + start := time.Now() + hops := 0 + + for i := range *chainLength { + spendable, _, err = client.ListVtxos(ctx) + require.NoError(t, err) + for len(spendable) == 0 { + spendable, _, err = client.ListVtxos(ctx) + require.NoError(t, err) + } + require.Len(t, spendable, 1) + tip := spendable[0] + + wg := &sync.WaitGroup{} + var notifyErr error + wg.Go(func() { + _, notifyErr = client.NotifyIncomingFunds(ctx, offchainAddr.Address) + }) + + res, err := client.SendOffChain(ctx, []types.Receiver{{ + To: offchainAddr.Address, + Amount: tip.Amount, + }}) + require.NoError(t, err) + + wg.Wait() + require.NoError(t, notifyErr) + + hops++ + t.Logf("hop %d: txid=%s", i, res.Txid) + } + + chainElapsed := time.Since(start) + t.Logf("chain built: %d hops in %s", hops, chainElapsed) + + time.Sleep(2 * time.Second) + } + + spendable, _, err := client.ListVtxos(ctx) + require.NoError(t, err) + tip := spendable[0] + + // Benchmark GetVtxoChain on the last VTXO in the chain. + last := types.Outpoint{Txid: tip.Txid, VOut: tip.VOut} + idx, err := grpcindexer.NewClient(*arkServerUrl) + require.NoError(t, err) + + getChainStart := time.Now() + resp, err := idx.GetVtxoChain(ctx, last) + getChainElapsed := time.Since(getChainStart) + require.NoError(t, err) + + t.Logf("GetVtxoChain: %d entries in %s (tip=%s:%d)", len(resp.Chain), getChainElapsed, last.Txid, last.VOut) +} + +func chainGenerateNote(t *testing.T, amount uint64) string { + t.Helper() + + httpClient := &http.Client{Timeout: 15 * time.Second} + + reqBody := bytes.NewReader([]byte(fmt.Sprintf(`{"amount": "%d"}`, amount))) + req, err := http.NewRequest("POST", *arkAdminUrl+"/v1/admin/note", reqBody) + require.NoError(t, err) + + req.Header.Set("Authorization", "Basic YWRtaW46YWRtaW4=") + req.Header.Set("Content-Type", "application/json") + + resp, err := httpClient.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + var noteResp struct { + Notes []string `json:"notes"` + } + err = json.NewDecoder(resp.Body).Decode(¬eResp) + require.NoError(t, err) + require.NotEmpty(t, noteResp.Notes) + + return noteResp.Notes[0] +} From 7bb2a6a868eb3a1c68dde447df282124f4592c0c Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Wed, 15 Apr 2026 21:45:44 -0400 Subject: [PATCH 43/54] fix badger fragile error matching --- internal/infrastructure/db/badger/ark_repo.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/infrastructure/db/badger/ark_repo.go b/internal/infrastructure/db/badger/ark_repo.go index 0ca0a92cd..f0d98a8ec 100644 --- a/internal/infrastructure/db/badger/ark_repo.go +++ b/internal/infrastructure/db/badger/ark_repo.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "path/filepath" - "strings" "time" "github.com/arkade-os/arkd/internal/core/domain" @@ -246,7 +245,7 @@ func (r *arkRepository) GetOffchainTxsByTxids( for _, txid := range txids { tx, err := r.getOffchainTx(ctx, txid) if err != nil { - if strings.Contains(err.Error(), "not found") { + if errors.Is(err, badgerhold.ErrNotFound) { continue } return nil, err @@ -369,10 +368,10 @@ func (r *arkRepository) getOffchainTx( err = r.store.Get(txid, &offchainTx) } if err != nil && err == badgerhold.ErrNotFound { - return nil, fmt.Errorf("offchain tx %s not found", txid) + return nil, fmt.Errorf("offchain tx %s: %w", txid, badgerhold.ErrNotFound) } if offchainTx.Stage.Code == int(domain.OffchainTxUndefinedStage) { - return nil, fmt.Errorf("offchain tx %s not found", txid) + return nil, fmt.Errorf("offchain tx %s: %w", txid, badgerhold.ErrNotFound) } return &offchainTx, nil From a266382a28764fde946e14301a156f147ae08c19 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 16 Apr 2026 11:50:48 -0400 Subject: [PATCH 44/54] fix marker sweep, SQLite atomicity, token TTL, and retry bounds --- internal/core/application/indexer.go | 89 ++++++++++---- internal/core/application/token_cache.go | 35 ++++++ internal/infrastructure/db/service.go | 30 +++-- internal/infrastructure/db/service_test.go | 109 ++++++++++++++++++ .../infrastructure/db/sqlite/marker_repo.go | 34 +++--- ...0260210000000_add_depth_and_markers.up.sql | 4 +- .../infrastructure/db/sqlite/vtxo_repo.go | 15 +-- 7 files changed, 263 insertions(+), 53 deletions(-) diff --git a/internal/core/application/indexer.go b/internal/core/application/indexer.go index 3cb114e22..751696a5f 100644 --- a/internal/core/application/indexer.go +++ b/internal/core/application/indexer.go @@ -353,33 +353,15 @@ func (i *indexerService) GetVtxoChain( case exposureWithheld: // Auth token is optional, validate it only if provided if authToken != "" { - hash, err := i.validateAuthToken(authToken) - if err != nil { + if err := i.validateChainAuth(authToken, vtxoKey, pageToken != ""); err != nil { return nil, err } - - outpoints, _, ok := i.tokenCache.getOutpoints(hash) - if !ok { - return nil, fmt.Errorf("auth token not found") - } - if _, ok := outpoints[vtxoKey.String()]; !ok { - return nil, fmt.Errorf("auth token is not for outpoint %s", vtxoKey) - } } case exposurePrivate: // Auth token is mandatory, always validate it - hash, err := i.validateAuthToken(authToken) - if err != nil { + if err := i.validateChainAuth(authToken, vtxoKey, pageToken != ""); err != nil { return nil, err } - - outpoints, _, ok := i.tokenCache.getOutpoints(hash) - if !ok { - return nil, fmt.Errorf("auth token not found") - } - if _, ok := outpoints[vtxoKey.String()]; !ok { - return nil, fmt.Errorf("auth token is not for outpoint %s", vtxoKey) - } } // Determine page size. @@ -417,6 +399,40 @@ func (i *indexerService) GetVtxoChain( }, nil } +// validateChainAuth validates the auth token for GetVtxoChain. On pagination +// continuations (isPaginating=true), if the signed timestamp has expired but +// the session is still active in the token cache (kept alive by touch on each +// page request), the token is accepted based on signature verification alone. +func (i *indexerService) validateChainAuth( + authToken string, vtxoKey Outpoint, isPaginating bool, +) error { + hash, err := i.validateAuthToken(authToken) + if err != nil && isPaginating { + // Token timestamp expired, but this is a pagination continuation. + // Verify signature only and check if the session is still live. + hash, err = i.verifyAuthTokenSignature(authToken) + if err != nil { + return err + } + if !i.tokenCache.isActive(hash) { + return fmt.Errorf("auth token expired") + } + } else if err != nil { + return err + } + + outpoints, _, ok := i.tokenCache.getOutpoints(hash) + if !ok { + return fmt.Errorf("auth token not found") + } + if _, ok := outpoints[vtxoKey.String()]; !ok { + return fmt.Errorf("auth token is not for outpoint %s", vtxoKey) + } + // Keep the session alive for pagination continuations. + i.tokenCache.touch(hash) + return nil +} + func (i *indexerService) GetVtxoChainByIntent( ctx context.Context, intent Intent, page *Page, ) (*VtxoChainResp, error) { @@ -1248,6 +1264,39 @@ func (i *indexerService) validateAuthToken(authToken string) (string, error) { return hex.EncodeToString(msg[:32]), nil } +// verifyAuthTokenSignature validates the auth token signature without checking +// the embedded timestamp. Used for pagination continuations where the session +// is kept alive via tokenCache.touch instead of the signed timestamp. +func (i *indexerService) verifyAuthTokenSignature(authToken string) (string, error) { + if authToken == "" { + return "", fmt.Errorf("missing auth") + } + + tokenBytes, err := base64.StdEncoding.DecodeString(authToken) + if err != nil { + return "", fmt.Errorf("invalid auth token format, must be base64") + } + + if len(tokenBytes) != 40+64 { + return "", fmt.Errorf("invalid auth token length") + } + + msg := tokenBytes[0:40] + sigBytes := tokenBytes[40:] + + msgHash := chainhash.HashB(msg) + sig, err := schnorr.ParseSignature(sigBytes) + if err != nil { + return "", fmt.Errorf("failed to parse auth token signature: %w", err) + } + + if !sig.Verify(msgHash, i.authPrvkey.PubKey()) { + return "", fmt.Errorf("signature verification failed") + } + + return hex.EncodeToString(msg[:32]), nil +} + // extractTokenHash decodes an auth token and returns the outpoints hash // without checking expiry. Signature is still verified. func (i *indexerService) extractTokenHash(authToken string) (string, error) { diff --git a/internal/core/application/token_cache.go b/internal/core/application/token_cache.go index c35385d3d..49ad34eda 100644 --- a/internal/core/application/token_cache.go +++ b/internal/core/application/token_cache.go @@ -61,6 +61,41 @@ func (c *tokenCache) close() { close(c.stop) } +// touch extends the expiry of an existing cache entry by invalidationDuration +// from now. Auth tokens embed a signed timestamp that expires after authTokenTTL +// (5 min), but paginating a long VTXO chain can span many requests over a longer +// period. Each successful GetVtxoChain page calls touch so the cache entry stays +// live; validateChainAuth then accepts expired-timestamp tokens as long as the +// cache entry is still active, proving the session was recently used. +func (c *tokenCache) touch(hash string) { + c.mu.Lock() + defer c.mu.Unlock() + + outpoints, ok := c.outpointsByHash[hash] + if !ok { + return + } + newExpiry := time.Now().Add(c.invalidationDuration) + for op := range outpoints { + outpoints[op] = newExpiry + } +} + +// isActive returns true if the hash has a non-expired cache entry. +func (c *tokenCache) isActive(hash string) bool { + c.mu.RLock() + defer c.mu.RUnlock() + + outpoints, ok := c.outpointsByHash[hash] + if !ok { + return false + } + for _, expiresAt := range outpoints { + return time.Now().Before(expiresAt) + } + return false +} + func (c *tokenCache) add(hash string, outpoints []Outpoint, now time.Time) { c.mu.Lock() defer c.mu.Unlock() diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index 2bea09111..9c15c053b 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -86,7 +86,8 @@ var ( ) const ( - sqliteDbFile = "sqlite.db" + sqliteDbFile = "sqlite.db" + maxProjectionRetry = 5 ) type ServiceConfig struct { @@ -504,9 +505,11 @@ func (s *service) updateProjectionsAfterRoundEvents(events []domain.Event) { newVtxos := getNewVtxosFromRound(*round, s.txDecoder) if len(spentVtxos) > 0 { - for { + for attempt := range maxProjectionRetry { if err := repo.SettleVtxos(ctx, spentVtxos, round.CommitmentTxid); err != nil { - log.WithError(err).Warn("failed to spend vtxos, retrying...") + log.WithError(err).Warnf( + "failed to spend vtxos (attempt %d/%d)", attempt+1, maxProjectionRetry, + ) time.Sleep(100 * time.Millisecond) continue } @@ -516,10 +519,12 @@ func (s *service) updateProjectionsAfterRoundEvents(events []domain.Event) { } if len(newVtxos) > 0 { - for { + for attempt := range maxProjectionRetry { // this will take care of updating asset projections as well if err := repo.AddVtxos(ctx, newVtxos); err != nil { - log.WithError(err).Warn("failed to add new vtxos, retrying soon") + log.WithError(err).Warnf( + "failed to add new vtxos (attempt %d/%d)", attempt+1, maxProjectionRetry, + ) time.Sleep(100 * time.Millisecond) continue } @@ -529,10 +534,11 @@ func (s *service) updateProjectionsAfterRoundEvents(events []domain.Event) { } // Create root markers for batch VTXOs (depth 0 is always at marker boundary) - for { + for attempt := range maxProjectionRetry { if err := s.markerStore.CreateRootMarkersForVtxos(ctx, newVtxos); err != nil { log.WithError(err).Warnf( - "failed to create root markers for %d vtxos, retrying soon", len(newVtxos), + "failed to create root markers for %d vtxos (attempt %d/%d)", + len(newVtxos), attempt+1, maxProjectionRetry, ) time.Sleep(100 * time.Millisecond) continue @@ -591,8 +597,14 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) marker, ids := domain.NewMarker(txid, newDepth, parentMarkerIDs) if marker != nil { if err := s.markerStore.AddMarker(ctx, *marker); err != nil { - log.WithError(err).Warn("failed to create marker for chained vtxo") - // Continue without marker - non-fatal + log.WithError(err). + Warn("failed to create marker for chained vtxo, falling back to parent markers") + // Fall back to parent markers so VTXOs are still sweepable. + // Without this, markerIDs stays nil and the VTXOs become + // permanently unsweepable — the swept column was removed and + // swept status is now derived from whether any of a VTXO's + // markers appear in the swept_marker table. + markerIDs = parentMarkerIDs } else { log.Debugf("created marker %s at depth %d", marker.ID, newDepth) markerIDs = ids diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index 5862a12be..c7674eab5 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -207,6 +207,7 @@ func TestService(t *testing.T) { testDeepChain20kMarkers(t, svc) testPartialMarkerSweep(t, svc) testListVtxosMarkerSweptFiltering(t, svc) + testAddMarkerFailureFallbackToParentMarkers(t, svc) testSweepableUnrolledExcludesMarkerSwept(t, svc) testConvergentMultiParentMarkerDAG(t, svc) testSweepMarkerWithDescendantsDeepChain(t, svc) @@ -4575,6 +4576,114 @@ func testListVtxosMarkerSweptFiltering(t *testing.T, svc ports.RepoManager) { }) } +// testAddMarkerFailureFallbackToParentMarkers verifies the fix for the AddMarker +// failure path in service.go:593. When AddMarker fails at a boundary depth, VTXOs +// should fall back to inheriting parentMarkerIDs instead of getting nil markers. +// This test simulates that fallback and proves the VTXOs remain sweepable via the +// parent marker. +func testAddMarkerFailureFallbackToParentMarkers(t *testing.T, svc ports.RepoManager) { + t.Run("test_add_marker_failure_fallback_to_parent_markers", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + suffix := randomString(16) + testPubkey := "fallback-pk-" + suffix + + // Create a finalized round. + roundId := uuid.New().String() + commitmentTxid := randomString(32) + round := domain.NewRoundFromEvents([]domain.Event{ + domain.RoundStarted{ + RoundEvent: domain.RoundEvent{Id: roundId, Type: domain.EventTypeRoundStarted}, + Timestamp: time.Now().Unix(), + }, + domain.RoundFinalizationStarted{ + RoundEvent: domain.RoundEvent{ + Id: roundId, + Type: domain.EventTypeRoundFinalizationStarted, + }, + CommitmentTxid: commitmentTxid, + CommitmentTx: emptyTx, + VtxoTree: vtxoTree, + Connectors: connectorsTree, + VtxoTreeExpiration: 3600, + }, + domain.RoundFinalized{ + RoundEvent: domain.RoundEvent{ + Id: roundId, + Type: domain.EventTypeRoundFinalized, + }, + FinalCommitmentTx: emptyTx, + Timestamp: time.Now().Unix(), + }, + }) + require.NoError(t, svc.Rounds().AddOrUpdateRound(ctx, *round)) + + // Create a parent marker (depth 0) — this is the marker the parent VTXO carries. + parentMarkerID := "fallback-parent-m-" + suffix + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: parentMarkerID, Depth: 0, + })) + + // Simulate the fix: at boundary depth 100, AddMarker failed, so we fall + // back to parentMarkerIDs. The child VTXO inherits the parent marker + // instead of getting nil. + parentMarkerIDs := []string{parentMarkerID} + + // Reproduce the fixed code path from service.go: + // marker, ids := domain.NewMarker(txid, 100, parentMarkerIDs) + // // AddMarker fails... + // markerIDs = parentMarkerIDs <-- the fix + marker, _ := domain.NewMarker("some-txid", 100, parentMarkerIDs) + require.NotNil(t, marker, "depth 100 is a boundary, should produce a marker") + // We intentionally skip AddMarker (simulating failure) and fall back: + markerIDs := parentMarkerIDs + + childVtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "fallback-child-" + suffix, VOut: 0}, + PubKey: testPubkey, + Amount: 4000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + CreatedAt: time.Now().Unix(), + ExpiresAt: time.Now().Add(time.Hour).Unix(), + Depth: 100, + MarkerIDs: markerIDs, + } + + require.NoError(t, svc.Vtxos().AddVtxos(ctx, []domain.Vtxo{childVtxo})) + require.NoError(t, svc.Markers().UpdateVtxoMarkers(ctx, childVtxo.Outpoint, childVtxo.MarkerIDs)) + + // Verify the child VTXO inherited the parent marker. + vtxos, err := svc.Vtxos().GetVtxos(ctx, []domain.Outpoint{childVtxo.Outpoint}) + require.NoError(t, err) + require.Len(t, vtxos, 1) + require.Equal(t, parentMarkerIDs, vtxos[0].MarkerIDs, + "child VTXO should carry parent markers after AddMarker failure fallback") + + // Sweep the parent marker. + sweptAt := time.Now().UnixMilli() + require.NoError(t, svc.Markers().BulkSweepMarkers(ctx, []string{parentMarkerID}, sweptAt)) + + // Verify the child VTXO is now swept — the fix works. + unspent, spent, err := svc.Vtxos().GetAllNonUnrolledVtxos(ctx, testPubkey) + require.NoError(t, err) + + spentTxids := make(map[string]bool) + for _, v := range spent { + spentTxids[v.Txid] = true + } + require.True(t, spentTxids[childVtxo.Outpoint.Txid], + "child VTXO with inherited parent markers should be swept") + + for _, v := range unspent { + require.NotEqual(t, childVtxo.Outpoint.Txid, v.Txid, + "child VTXO should not appear in unspent list after parent marker sweep") + } + }) +} + // testSweepableUnrolledExcludesMarkerSwept verifies that GetAllSweepableUnrolledVtxos // excludes VTXOs whose markers have been swept. Creates 3 spent+unrolled VTXOs across // two markers, sweeps one marker, and confirms only the unswept VTXOs appear as sweepable. diff --git a/internal/infrastructure/db/sqlite/marker_repo.go b/internal/infrastructure/db/sqlite/marker_repo.go index 37c1d1943..e7875e10b 100644 --- a/internal/infrastructure/db/sqlite/marker_repo.go +++ b/internal/infrastructure/db/sqlite/marker_repo.go @@ -166,25 +166,29 @@ func (m *markerRepository) SweepMarkerWithDescendants( markerID string, sweptAt int64, ) (int64, error) { - // Get all descendant marker IDs (including the root marker) that are not already swept - descendantIDs, err := m.querier.GetDescendantMarkerIds(ctx, markerID) - if err != nil { - return 0, fmt.Errorf("failed to get descendant markers: %w", err) - } - - // Insert each descendant into swept_marker var count int64 - for _, id := range descendantIDs { - err := m.querier.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ - MarkerID: id, - SweptAt: sweptAt, - }) + txBody := func(qtx *queries.Queries) error { + // Get all descendant marker IDs (including the root marker) that are not already swept + descendantIDs, err := qtx.GetDescendantMarkerIds(ctx, markerID) if err != nil { - return count, fmt.Errorf("failed to sweep marker %s: %w", id, err) + return fmt.Errorf("failed to get descendant markers: %w", err) } - count++ - } + // Insert each descendant into swept_marker + for _, id := range descendantIDs { + if err := qtx.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ + MarkerID: id, + SweptAt: sweptAt, + }); err != nil { + return fmt.Errorf("failed to sweep marker %s: %w", id, err) + } + count++ + } + return nil + } + if err := execTx(ctx, m.db, txBody); err != nil { + return 0, err + } return count, nil } diff --git a/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql index 9f18c10e2..1fedbd68b 100644 --- a/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql +++ b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS swept_marker ( ); -- Add markers column (JSON array, not single marker_id) -ALTER TABLE vtxo ADD COLUMN markers TEXT; +ALTER TABLE vtxo ADD COLUMN markers TEXT NOT NULL DEFAULT '[]'; CREATE INDEX IF NOT EXISTS idx_vtxo_markers ON vtxo(markers); -- Recreate views to include the new columns @@ -86,7 +86,7 @@ CREATE TABLE vtxo_new ( intent_id TEXT, updated_at INTEGER, depth INTEGER NOT NULL DEFAULT 0, - markers TEXT, + markers TEXT NOT NULL DEFAULT '[]', PRIMARY KEY (txid, vout), FOREIGN KEY (intent_id) REFERENCES intent(id) ); diff --git a/internal/infrastructure/db/sqlite/vtxo_repo.go b/internal/infrastructure/db/sqlite/vtxo_repo.go index 6b0a490ff..434000d6f 100644 --- a/internal/infrastructure/db/sqlite/vtxo_repo.go +++ b/internal/infrastructure/db/sqlite/vtxo_repo.go @@ -43,14 +43,15 @@ func (v *vtxoRepository) AddVtxos(ctx context.Context, vtxos []domain.Vtxo) erro for i := range vtxos { vtxo := vtxos[i] - var markersJSON sql.NullString - if len(vtxo.MarkerIDs) > 0 { - data, err := json.Marshal(vtxo.MarkerIDs) - if err != nil { - return fmt.Errorf("failed to marshal markers: %w", err) - } - markersJSON = sql.NullString{String: string(data), Valid: true} + markersToMarshal := vtxo.MarkerIDs + if markersToMarshal == nil { + markersToMarshal = []string{} + } + markersData, err := json.Marshal(markersToMarshal) + if err != nil { + return fmt.Errorf("failed to marshal markers: %w", err) } + markersJSON := sql.NullString{String: string(markersData), Valid: true} if err := querierWithTx.UpsertVtxo( ctx, queries.UpsertVtxoParams{ From b9f58d85ed5f35caa4972378f958eef2c1e7fcff Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 16 Apr 2026 13:48:36 -0400 Subject: [PATCH 45/54] HMAC-sign pagination cursors to prevent auth bypass --- internal/core/application/indexer.go | 63 ++++++++++--- internal/core/application/indexer_test.go | 109 ++++++++++++++++++++-- 2 files changed, 151 insertions(+), 21 deletions(-) diff --git a/internal/core/application/indexer.go b/internal/core/application/indexer.go index 751696a5f..9593bda63 100644 --- a/internal/core/application/indexer.go +++ b/internal/core/application/indexer.go @@ -3,6 +3,7 @@ package application import ( "bytes" "context" + "crypto/hmac" "crypto/sha256" "encoding/base64" "encoding/binary" @@ -88,6 +89,7 @@ type indexerService struct { repoManager ports.RepoManager wallet ports.WalletService authPrvkey *btcec.PrivateKey // key used to sign auth tokens + cursorHMACKey []byte // HMAC key for signing pagination cursors signerPubkey *btcec.PublicKey // server's signing key, used for stripping signatures from txs txExposure exposure authTokenTTL time.Duration @@ -116,10 +118,19 @@ func NewIndexerService( ttl = time.Duration(authTokenExpirySec) * time.Second } + // Derive HMAC key for pagination cursors from the auth private key. + // This prevents clients from forging cursors with arbitrary outpoints. + var cursorKey []byte + if privkey != nil { + h := sha256.Sum256(append(privkey.Serialize(), []byte("cursor-hmac")...)) + cursorKey = h[:] + } + svc := &indexerService{ repoManager: repoManager, wallet: wallet, authPrvkey: privkey, + cursorHMACKey: cursorKey, txExposure: exposure(txExposure), authTokenTTL: ttl, tokenCache: newTokenCache(ttl), @@ -379,7 +390,7 @@ func (i *indexerService) GetVtxoChain( // Determine frontier: decode pageToken, or use [vtxoKey] for first page. var frontier []domain.Outpoint if pageToken != "" { - decoded, err := decodeChainCursor(pageToken) + decoded, err := i.decodeChainCursor(pageToken) if err != nil { return nil, fmt.Errorf("invalid page_token: %w", err) } @@ -667,7 +678,7 @@ func (i *indexerService) walkVtxoChain( } } remaining = append(remaining, newNextVtxos...) - token := encodeChainCursor(remaining) + token := i.encodeChainCursor(remaining) return chain, allOutpoints, token, nil } @@ -798,32 +809,56 @@ func (i *indexerService) walkVtxoChain( return chain, allOutpoints, "", nil } -// encodeChainCursor encodes a frontier of outpoints into an opaque page token. -func encodeChainCursor(frontier []domain.Outpoint) string { +// encodeChainCursor encodes a frontier of outpoints into an HMAC-signed opaque +// page token. The HMAC prevents clients from forging cursors with arbitrary +// outpoints, which would bypass auth validation in exposurePrivate mode. +func (i *indexerService) encodeChainCursor(frontier []domain.Outpoint) string { if len(frontier) == 0 { return "" } cur := vtxoChainCursor{Frontier: make([]Outpoint, len(frontier))} - for i, op := range frontier { - cur.Frontier[i] = Outpoint(op) + for idx, op := range frontier { + cur.Frontier[idx] = Outpoint(op) } - data, _ := json.Marshal(cur) - return base64.RawURLEncoding.EncodeToString(data) + payload, _ := json.Marshal(cur) + + if len(i.cursorHMACKey) > 0 { + mac := hmac.New(sha256.New, i.cursorHMACKey) + mac.Write(payload) + payload = append(payload, mac.Sum(nil)...) + } + return base64.RawURLEncoding.EncodeToString(payload) } -// decodeChainCursor decodes a page token back into a frontier of outpoints. -func decodeChainCursor(token string) ([]domain.Outpoint, error) { - data, err := base64.RawURLEncoding.DecodeString(token) +// decodeChainCursor decodes and verifies an HMAC-signed page token. +func (i *indexerService) decodeChainCursor(token string) ([]domain.Outpoint, error) { + raw, err := base64.RawURLEncoding.DecodeString(token) if err != nil { return nil, fmt.Errorf("invalid base64: %w", err) } + + payload := raw + if len(i.cursorHMACKey) > 0 { + if len(raw) < sha256.Size { + return nil, fmt.Errorf("invalid cursor: too short") + } + payload = raw[:len(raw)-sha256.Size] + sig := raw[len(raw)-sha256.Size:] + + mac := hmac.New(sha256.New, i.cursorHMACKey) + mac.Write(payload) + if !hmac.Equal(sig, mac.Sum(nil)) { + return nil, fmt.Errorf("invalid cursor: signature mismatch") + } + } + var cur vtxoChainCursor - if err := json.Unmarshal(data, &cur); err != nil { + if err := json.Unmarshal(payload, &cur); err != nil { return nil, fmt.Errorf("invalid JSON: %w", err) } outpoints := make([]domain.Outpoint, len(cur.Frontier)) - for i, op := range cur.Frontier { - outpoints[i] = domain.Outpoint(op) + for idx, op := range cur.Frontier { + outpoints[idx] = domain.Outpoint(op) } return outpoints, nil } diff --git a/internal/core/application/indexer_test.go b/internal/core/application/indexer_test.go index d5861ebc8..0d80437d8 100644 --- a/internal/core/application/indexer_test.go +++ b/internal/core/application/indexer_test.go @@ -2,6 +2,7 @@ package application import ( "context" + "encoding/base64" "fmt" "sort" "strings" @@ -408,16 +409,17 @@ func makeCheckpointPSBT(t *testing.T, inputTxid string, inputVout uint32) string // TestEncodeDecodeChainCursor_RoundTrip verifies that encoding then decoding // a frontier of outpoints returns the same outpoints. func TestEncodeDecodeChainCursor_RoundTrip(t *testing.T) { + svc := &indexerService{} frontier := []domain.Outpoint{ {Txid: "abc123", VOut: 0}, {Txid: "def456", VOut: 2}, {Txid: "ghi789", VOut: 1}, } - token := encodeChainCursor(frontier) + token := svc.encodeChainCursor(frontier) require.NotEmpty(t, token) - decoded, err := decodeChainCursor(token) + decoded, err := svc.decodeChainCursor(token) require.NoError(t, err) require.Equal(t, frontier, decoded) } @@ -425,16 +427,18 @@ func TestEncodeDecodeChainCursor_RoundTrip(t *testing.T) { // TestEncodeDecodeChainCursor_EmptyFrontier verifies that an empty frontier // encodes to an empty string. func TestEncodeDecodeChainCursor_EmptyFrontier(t *testing.T) { - token := encodeChainCursor(nil) + svc := &indexerService{} + token := svc.encodeChainCursor(nil) require.Empty(t, token) - token = encodeChainCursor([]domain.Outpoint{}) + token = svc.encodeChainCursor([]domain.Outpoint{}) require.Empty(t, token) } // TestDecodeChainCursor_InvalidBase64 verifies that invalid base64 returns an error. func TestDecodeChainCursor_InvalidBase64(t *testing.T) { - _, err := decodeChainCursor("not-valid-base64!!!") + svc := &indexerService{} + _, err := svc.decodeChainCursor("not-valid-base64!!!") require.Error(t, err) require.Contains(t, err.Error(), "invalid base64") } @@ -442,11 +446,102 @@ func TestDecodeChainCursor_InvalidBase64(t *testing.T) { // TestDecodeChainCursor_InvalidJSON verifies that valid base64 but invalid JSON // returns an error. func TestDecodeChainCursor_InvalidJSON(t *testing.T) { + svc := &indexerService{} // Encode something that is not valid JSON token := "bm90LWpzb24" // base64url of "not-json" - _, err := decodeChainCursor(token) + _, err := svc.decodeChainCursor(token) require.Error(t, err) - require.Contains(t, err.Error(), "invalid JSON") +} + +// TestDecodeChainCursor_HMACRejectsForgery verifies that a cursor signed with +// one key is rejected by a service with a different key. +func TestDecodeChainCursor_HMACRejectsForgery(t *testing.T) { + svc := &indexerService{cursorHMACKey: []byte("server-secret-key")} + frontier := []domain.Outpoint{{Txid: "abc123", VOut: 0}} + + token := svc.encodeChainCursor(frontier) + require.NotEmpty(t, token) + + // Valid decode with same key works. + decoded, err := svc.decodeChainCursor(token) + require.NoError(t, err) + require.Equal(t, frontier, decoded) + + // Forge a cursor with a different key — should be rejected. + forger := &indexerService{cursorHMACKey: []byte("attacker-key")} + forgedToken := forger.encodeChainCursor([]domain.Outpoint{{Txid: "victim-vtxo", VOut: 0}}) + _, err = svc.decodeChainCursor(forgedToken) + require.Error(t, err) + require.Contains(t, err.Error(), "signature mismatch") + + // Tampered cursor — modify one byte of a valid token. + rawToken, _ := base64.RawURLEncoding.DecodeString(token) + rawToken[0] ^= 0xff + tampered := base64.RawURLEncoding.EncodeToString(rawToken) + _, err = svc.decodeChainCursor(tampered) + require.Error(t, err) +} + +// TestDecodeChainCursor_HMACEdgeCases covers malicious and accidental misuse of +// the cursor field: truncated tokens, empty strings, unsigned cursors sent to a +// signing server, replaying a valid cursor after the HMAC portion is stripped, etc. +func TestDecodeChainCursor_HMACEdgeCases(t *testing.T) { + svc := &indexerService{cursorHMACKey: []byte("server-secret-key")} + frontier := []domain.Outpoint{{Txid: "abc123", VOut: 0}} + validToken := svc.encodeChainCursor(frontier) + + t.Run("empty string", func(t *testing.T) { + _, err := svc.decodeChainCursor("") + require.Error(t, err) + }) + + t.Run("truncated token missing HMAC bytes", func(t *testing.T) { + raw, err := base64.RawURLEncoding.DecodeString(validToken) + require.NoError(t, err) + // Strip the 32-byte HMAC, leaving only the JSON payload. + truncated := base64.RawURLEncoding.EncodeToString(raw[:len(raw)-32]) + _, err = svc.decodeChainCursor(truncated) + require.Error(t, err) + }) + + t.Run("unsigned cursor rejected by signing server", func(t *testing.T) { + // A server with no HMAC key produces unsigned cursors. + noKey := &indexerService{} + unsigned := noKey.encodeChainCursor(frontier) + // A server WITH an HMAC key must reject it. + _, err := svc.decodeChainCursor(unsigned) + require.Error(t, err) + }) + + t.Run("hand-crafted JSON without HMAC", func(t *testing.T) { + // Attacker builds raw JSON and base64-encodes it, no HMAC. + raw := []byte(`{"frontier":[{"txid":"victim","vout":0}]}`) + crafted := base64.RawURLEncoding.EncodeToString(raw) + _, err := svc.decodeChainCursor(crafted) + require.Error(t, err) + }) + + t.Run("cursor from restarted server with new key", func(t *testing.T) { + oldServer := &indexerService{cursorHMACKey: []byte("old-key")} + oldToken := oldServer.encodeChainCursor(frontier) + newServer := &indexerService{cursorHMACKey: []byte("new-key-after-restart")} + _, err := newServer.decodeChainCursor(oldToken) + require.Error(t, err) + require.Contains(t, err.Error(), "signature mismatch") + }) + + t.Run("swapped payload same length", func(t *testing.T) { + // Take a valid token, replace the JSON payload but keep the + // original HMAC — should fail because HMAC won't match. + raw, err := base64.RawURLEncoding.DecodeString(validToken) + require.NoError(t, err) + origHMAC := raw[len(raw)-32:] + newPayload := []byte(`{"frontier":[{"txid":"other","vout":0}]}`) + tampered := append(newPayload, origHMAC...) + token := base64.RawURLEncoding.EncodeToString(tampered) + _, err = svc.decodeChainCursor(token) + require.Error(t, err) + }) } // TestEnsureVtxosCached_AllCacheHits verifies that when all outpoints are already From 6833a6cdc078eccdbacf1999182520deb3027cb6 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Thu, 16 Apr 2026 17:28:39 -0400 Subject: [PATCH 46/54] fix checkpoint sweep over-reach with per-outpoint tracking --- internal/core/application/indexer_test.go | 8 + internal/core/application/sweeper.go | 37 +- internal/core/application/sweeper_test.go | 521 ++---------------- internal/core/domain/marker_repo.go | 5 + .../infrastructure/db/badger/marker_repo.go | 21 + .../infrastructure/db/postgres/marker_repo.go | 30 +- .../20260416120000_add_swept_vtxo.down.sql | 54 ++ .../20260416120000_add_swept_vtxo.up.sql | 95 ++++ .../db/postgres/sqlc/queries/models.go | 8 +- .../db/postgres/sqlc/queries/query.sql.go | 36 +- .../infrastructure/db/postgres/sqlc/query.sql | 10 + .../infrastructure/db/postgres/vtxo_repo.go | 2 +- internal/infrastructure/db/service_test.go | 231 ++++++++ .../infrastructure/db/sqlite/marker_repo.go | 41 +- .../20260416120000_add_swept_vtxo.down.sql | 30 + .../20260416120000_add_swept_vtxo.up.sql | 82 +++ .../infrastructure/db/sqlite/round_repo.go | 2 +- .../db/sqlite/sqlc/queries/models.go | 14 +- .../db/sqlite/sqlc/queries/query.sql.go | 20 +- .../infrastructure/db/sqlite/sqlc/query.sql | 4 + .../infrastructure/db/sqlite/vtxo_repo.go | 21 +- 21 files changed, 736 insertions(+), 536 deletions(-) create mode 100644 internal/infrastructure/db/postgres/migration/20260416120000_add_swept_vtxo.down.sql create mode 100644 internal/infrastructure/db/postgres/migration/20260416120000_add_swept_vtxo.up.sql create mode 100644 internal/infrastructure/db/sqlite/migration/20260416120000_add_swept_vtxo.down.sql create mode 100644 internal/infrastructure/db/sqlite/migration/20260416120000_add_swept_vtxo.up.sql diff --git a/internal/core/application/indexer_test.go b/internal/core/application/indexer_test.go index 0d80437d8..5407009c8 100644 --- a/internal/core/application/indexer_test.go +++ b/internal/core/application/indexer_test.go @@ -288,6 +288,14 @@ func (m *mockMarkerRepoForIndexer) GetVtxosByArkTxid( ) ([]domain.Vtxo, error) { return nil, nil } +func (m *mockMarkerRepoForIndexer) SweepVtxoOutpoints( + ctx context.Context, + outpoints []domain.Outpoint, + sweptAt int64, +) error { + return nil +} + func (m *mockMarkerRepoForIndexer) Close() {} type mockOffchainTxRepoForIndexer struct { diff --git a/internal/core/application/sweeper.go b/internal/core/application/sweeper.go index 53e8cf4ba..62c1dff28 100644 --- a/internal/core/application/sweeper.go +++ b/internal/core/application/sweeper.go @@ -759,45 +759,28 @@ func (s *sweeper) createCheckpointSweepTask( log.Debugf("sweeper: checkpoint %s swept by: %s", checkpointTxid, txid) } - // mark all vtxos linked to the unrolled vtxo as swept + // Mark all vtxos linked to the unrolled vtxo as swept. + // Use per-outpoint sweeping instead of marker-based sweeping here + // because markers can be shared across independent subtrees when + // offchain txs consolidate inputs from different lineages. Sweeping + // by marker would over-reach and incorrectly mark unrelated VTXOs. childrenVtxos, err := s.repoManager.Vtxos().GetAllChildrenVtxos(ctx, vtxo.Txid) if err != nil { return err } - // Get the VTXOs to find their markers - vtxos, err := s.repoManager.Vtxos().GetVtxos(ctx, childrenVtxos) - if err != nil { - return err - } - - // Collect all unique markers from all VTXOs - // Every VTXO is guaranteed to have at least 1 marker after migration - uniqueMarkers := make(map[string]struct{}) - for _, v := range vtxos { - for _, markerID := range v.MarkerIDs { - uniqueMarkers[markerID] = struct{}{} - } - } - - if len(uniqueMarkers) == 0 { + if len(childrenVtxos) == 0 { return nil } - // Convert marker set to slice for bulk sweeping - markerIDs := make([]string, 0, len(uniqueMarkers)) - for markerID := range uniqueMarkers { - markerIDs = append(markerIDs, markerID) - } - sweptAt := time.Now().UnixMilli() - markerStore := s.repoManager.Markers() - if err := markerStore.BulkSweepMarkers(ctx, markerIDs, sweptAt); err != nil { - log.WithError(err).Warn("failed to bulk sweep markers") + if err := s.repoManager.Markers(). + SweepVtxoOutpoints(ctx, childrenVtxos, sweptAt); err != nil { + log.WithError(err).Warn("failed to sweep vtxo outpoints") return err } - log.Debugf("bulk swept %d markers affecting %d vtxos", len(markerIDs), len(vtxos)) + log.Debugf("swept %d vtxo outpoints for checkpoint %s", len(childrenVtxos), checkpointTxid) return nil } } diff --git a/internal/core/application/sweeper_test.go b/internal/core/application/sweeper_test.go index 2533d7c73..07fb7241c 100644 --- a/internal/core/application/sweeper_test.go +++ b/internal/core/application/sweeper_test.go @@ -402,6 +402,15 @@ func (m *mockMarkerRepository) GetVtxoChainByMarkers( ) ([]domain.Vtxo, error) { return nil, nil } +func (m *mockMarkerRepository) SweepVtxoOutpoints( + ctx context.Context, + outpoints []domain.Outpoint, + sweptAt int64, +) error { + args := m.Called(ctx, outpoints, sweptAt) + return args.Error(0) +} + func (m *mockMarkerRepository) Close() {} type mockTxBuilder struct { @@ -492,50 +501,23 @@ func newTestSweeper() ( return wallet, vtxoRepo, markerRepo, builder, s } -// TestCreateCheckpointSweepTask_BulkSweepsMarkers verifies that when a checkpoint -// is swept, the sweeper correctly collects all unique marker IDs from the affected -// VTXOs and calls BulkSweepMarkers with the deduplicated set. This tests the core -// optimization where multiple VTXOs sharing markers result in fewer marker sweep -// operations (3 VTXOs with overlapping markers should yield only 3 unique markers). -func TestCreateCheckpointSweepTask_BulkSweepsMarkers(t *testing.T) { +// TestCreateCheckpointSweepTask_SweepsVtxoOutpoints verifies that checkpoint +// sweeps use per-outpoint sweeping (SweepVtxoOutpoints) instead of marker-based +// sweeping. This prevents over-reach when markers are shared across independent +// subtrees due to offchain tx consolidation. +func TestCreateCheckpointSweepTask_SweepsVtxoOutpoints(t *testing.T) { wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() - // Test data checkpointTxid := "checkpoint123" vtxoOutpoint := domain.Outpoint{Txid: "vtxo123", VOut: 0} - // Child VTXOs that will be returned by GetAllChildrenVtxos childOutpoints := []domain.Outpoint{ {Txid: "child1", VOut: 0}, {Txid: "child2", VOut: 0}, {Txid: "child3", VOut: 0}, } - // VTXOs with markers - note some share markers to test deduplication - vtxosWithMarkers := []domain.Vtxo{ - { - Outpoint: childOutpoints[0], - MarkerIDs: []string{"marker-A", "marker-B"}, - Depth: 50, - }, - { - Outpoint: childOutpoints[1], - MarkerIDs: []string{"marker-B", "marker-C"}, // marker-B is shared - Depth: 75, - }, - { - Outpoint: childOutpoints[2], - MarkerIDs: []string{"marker-A"}, // marker-A is shared - Depth: 100, - }, - } - - // Setup mock expectations - toSweep := ports.TxInput{ - Txid: checkpointTxid, - Index: 0, - Value: 10000, - } + toSweep := ports.TxInput{Txid: checkpointTxid, Index: 0, Value: 10000} builder.On("BuildSweepTx", []ports.TxInput{toSweep}). Return("sweeptxid123", "sweeptx_hex", nil) @@ -546,249 +528,31 @@ func TestCreateCheckpointSweepTask_BulkSweepsMarkers(t *testing.T) { vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). Return(childOutpoints, nil) - vtxoRepo.On("GetVtxos", mock.Anything, childOutpoints). - Return(vtxosWithMarkers, nil) - - // Expect BulkSweepMarkers to be called with deduplicated markers - // Should have: marker-A, marker-B, marker-C (3 unique markers) - markerRepo.On("BulkSweepMarkers", mock.Anything, mock.MatchedBy(func(markerIDs []string) bool { - // Verify we have exactly 3 unique markers - if len(markerIDs) != 3 { - return false - } - // Verify all expected markers are present - markerSet := make(map[string]bool) - for _, id := range markerIDs { - markerSet[id] = true - } - return markerSet["marker-A"] && markerSet["marker-B"] && markerSet["marker-C"] - }), mock.AnythingOfType("int64")).Return(nil) - - // Execute the sweep task + // SweepVtxoOutpoints should be called with the exact child outpoints + markerRepo.On("SweepVtxoOutpoints", mock.Anything, childOutpoints, mock.AnythingOfType("int64")). + Return(nil) + task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) err := task() - // Verify require.NoError(t, err) wallet.AssertExpectations(t) vtxoRepo.AssertExpectations(t) markerRepo.AssertExpectations(t) builder.AssertExpectations(t) -} - -// TestCreateCheckpointSweepTask_NoMarkersSkipsSweep verifies that when VTXOs have -// no markers (empty MarkerIDs slice), the sweeper does not call BulkSweepMarkers. -// This is an edge case that could occur with legacy VTXOs or during error recovery, -// and ensures the sweeper handles it gracefully without attempting empty bulk operations. -func TestCreateCheckpointSweepTask_NoMarkersSkipsSweep(t *testing.T) { - wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() - - // Test data - checkpointTxid := "checkpoint456" - vtxoOutpoint := domain.Outpoint{Txid: "vtxo456", VOut: 0} - - // Child VTXOs with no markers (empty MarkerIDs) - childOutpoints := []domain.Outpoint{ - {Txid: "child1", VOut: 0}, - } - - vtxosWithoutMarkers := []domain.Vtxo{ - { - Outpoint: childOutpoints[0], - MarkerIDs: []string{}, // No markers - Depth: 0, - }, - } - - // Setup mock expectations - toSweep := ports.TxInput{ - Txid: checkpointTxid, - Index: 0, - Value: 10000, - } - - builder.On("BuildSweepTx", []ports.TxInput{toSweep}). - Return("sweeptxid456", "sweeptx_hex", nil) - - wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). - Return("sweeptxid456", nil) - - vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). - Return(childOutpoints, nil) - - vtxoRepo.On("GetVtxos", mock.Anything, childOutpoints). - Return(vtxosWithoutMarkers, nil) - - // BulkSweepMarkers should NOT be called since there are no markers - - // Execute the sweep task - task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) - err := task() - - // Verify - require.NoError(t, err) - wallet.AssertExpectations(t) - vtxoRepo.AssertExpectations(t) - // Verify BulkSweepMarkers was never called + // BulkSweepMarkers should NOT be called — checkpoint sweeps use per-outpoint markerRepo.AssertNotCalled(t, "BulkSweepMarkers", mock.Anything, mock.Anything, mock.Anything) } -// TestCreateCheckpointSweepTask_SingleMarkerPerVtxo verifies the typical post-migration -// state where each VTXO has exactly one marker (its own outpoint as the marker ID). -// This represents the common case after the database migration that assigns a unique -// marker to every existing VTXO, ensuring backward compatibility with the new marker system. -func TestCreateCheckpointSweepTask_SingleMarkerPerVtxo(t *testing.T) { - // Test case: each VTXO has exactly one marker (post-migration state) - wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() - - checkpointTxid := "checkpoint789" - vtxoOutpoint := domain.Outpoint{Txid: "vtxo789", VOut: 0} - - // Each VTXO has its own unique marker (typical post-migration state) - childOutpoints := []domain.Outpoint{ - {Txid: "child1", VOut: 0}, - {Txid: "child2", VOut: 0}, - } - - vtxosWithUniqueMarkers := []domain.Vtxo{ - { - Outpoint: childOutpoints[0], - MarkerIDs: []string{"child1:0"}, // Marker ID matches outpoint - Depth: 0, - }, - { - Outpoint: childOutpoints[1], - MarkerIDs: []string{"child2:0"}, - Depth: 0, - }, - } - - toSweep := ports.TxInput{ - Txid: checkpointTxid, - Index: 0, - Value: 20000, - } - - builder.On("BuildSweepTx", []ports.TxInput{toSweep}). - Return("sweeptxid789", "sweeptx_hex", nil) - - wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). - Return("sweeptxid789", nil) - - vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). - Return(childOutpoints, nil) - - vtxoRepo.On("GetVtxos", mock.Anything, childOutpoints). - Return(vtxosWithUniqueMarkers, nil) - - // Expect exactly 2 unique markers - markerRepo.On("BulkSweepMarkers", mock.Anything, mock.MatchedBy(func(markerIDs []string) bool { - if len(markerIDs) != 2 { - return false - } - markerSet := make(map[string]bool) - for _, id := range markerIDs { - markerSet[id] = true - } - return markerSet["child1:0"] && markerSet["child2:0"] - }), mock.AnythingOfType("int64")).Return(nil) - - task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) - err := task() - - require.NoError(t, err) - wallet.AssertExpectations(t) - vtxoRepo.AssertExpectations(t) - markerRepo.AssertExpectations(t) -} - -// TestCreateCheckpointSweepTask_ManyVtxosWithSharedMarkers verifies that the bulk sweep -// optimization works correctly for deep VTXO chains where many VTXOs share the same -// markers. This simulates a chain spanning depths 0-196 where all VTXOs share a root -// marker, and VTXOs at depth >= 100 also share an additional marker. Despite having -// 50 VTXOs, only 2 unique markers should be swept, demonstrating the efficiency gain. -func TestCreateCheckpointSweepTask_ManyVtxosWithSharedMarkers(t *testing.T) { - // Test case: many VTXOs share markers (chain with depth > 100) - wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() - - checkpointTxid := "checkpoint_deep" - vtxoOutpoint := domain.Outpoint{Txid: "vtxo_deep", VOut: 0} - - // Simulate a deep chain where many VTXOs share the same root marker - childOutpoints := make([]domain.Outpoint, 50) - vtxosWithSharedMarkers := make([]domain.Vtxo, 50) - - for i := 0; i < 50; i++ { - childOutpoints[i] = domain.Outpoint{Txid: "child" + string(rune('A'+i)), VOut: 0} - // All VTXOs at depth < 100 share the root marker - // VTXOs at depth >= 100 also have a depth-100 marker - depth := uint32(i * 4) // depths: 0, 4, 8, ... 196 (spans beyond 100) - markers := []string{"root-marker"} - if depth >= 100 { - markers = append(markers, "marker-100") - } - vtxosWithSharedMarkers[i] = domain.Vtxo{ - Outpoint: childOutpoints[i], - MarkerIDs: markers, - Depth: depth, - } - } - - toSweep := ports.TxInput{ - Txid: checkpointTxid, - Index: 0, - Value: 500000, - } - - builder.On("BuildSweepTx", []ports.TxInput{toSweep}). - Return("sweeptxid_deep", "sweeptx_hex", nil) - - wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). - Return("sweeptxid_deep", nil) - - vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). - Return(childOutpoints, nil) - - vtxoRepo.On("GetVtxos", mock.Anything, childOutpoints). - Return(vtxosWithSharedMarkers, nil) - - // Even with 50 VTXOs, we should only have 2 unique markers - markerRepo.On("BulkSweepMarkers", mock.Anything, mock.MatchedBy(func(markerIDs []string) bool { - if len(markerIDs) != 2 { - return false - } - markerSet := make(map[string]bool) - for _, id := range markerIDs { - markerSet[id] = true - } - return markerSet["root-marker"] && markerSet["marker-100"] - }), mock.AnythingOfType("int64")).Return(nil) - - task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) - err := task() - - require.NoError(t, err) - markerRepo.AssertExpectations(t) -} - -// TestCreateCheckpointSweepTask_SweptAtTimestamp verifies that the sweptAt timestamp -// passed to BulkSweepMarkers is accurate and falls within the execution window. -// This ensures that swept marker records have correct timestamps for auditing and -// debugging purposes, and that the timestamp is generated at execution time rather -// than being a stale or incorrect value. +// TestCreateCheckpointSweepTask_SweptAtTimestamp verifies that the sweptAt +// timestamp passed to SweepVtxoOutpoints is accurate. func TestCreateCheckpointSweepTask_SweptAtTimestamp(t *testing.T) { - // Test that the sweptAt timestamp is reasonable (within a few seconds of now) wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() checkpointTxid := "checkpoint_timestamp" vtxoOutpoint := domain.Outpoint{Txid: "vtxo_timestamp", VOut: 0} childOutpoints := []domain.Outpoint{{Txid: "child_ts", VOut: 0}} - vtxos := []domain.Vtxo{{ - Outpoint: childOutpoints[0], - MarkerIDs: []string{"marker-ts"}, - Depth: 0, - }} toSweep := ports.TxInput{Txid: checkpointTxid, Index: 0, Value: 1000} @@ -801,45 +565,31 @@ func TestCreateCheckpointSweepTask_SweptAtTimestamp(t *testing.T) { vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). Return(childOutpoints, nil) - vtxoRepo.On("GetVtxos", mock.Anything, childOutpoints). - Return(vtxos, nil) - - // Capture the sweptAt timestamp beforeExec := time.Now().UnixMilli() var capturedSweptAt int64 - markerRepo.On("BulkSweepMarkers", mock.Anything, mock.Anything, mock.MatchedBy(func(sweptAt int64) bool { + markerRepo.On("SweepVtxoOutpoints", mock.Anything, childOutpoints, mock.MatchedBy(func(sweptAt int64) bool { capturedSweptAt = sweptAt return true - })). - Return(nil) + })).Return(nil) task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) err := task() afterExec := time.Now().UnixMilli() require.NoError(t, err) - // Verify timestamp is within the execution window require.GreaterOrEqual(t, capturedSweptAt, beforeExec) require.LessOrEqual(t, capturedSweptAt, afterExec) } -// TestCreateCheckpointSweepTask_BulkSweepMarkersError verifies that when BulkSweepMarkers -// returns an error, the sweep task propagates the error back to the caller. This ensures -// that marker sweep failures are not silently ignored and can be properly handled by -// the calling code for retry logic or alerting. -func TestCreateCheckpointSweepTask_BulkSweepMarkersError(t *testing.T) { +// TestCreateCheckpointSweepTask_SweepVtxoOutpointsError verifies error propagation. +func TestCreateCheckpointSweepTask_SweepVtxoOutpointsError(t *testing.T) { wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() checkpointTxid := "checkpoint_error" vtxoOutpoint := domain.Outpoint{Txid: "vtxo_error", VOut: 0} childOutpoints := []domain.Outpoint{{Txid: "child_err", VOut: 0}} - vtxos := []domain.Vtxo{{ - Outpoint: childOutpoints[0], - MarkerIDs: []string{"marker-err"}, - Depth: 0, - }} toSweep := ports.TxInput{Txid: checkpointTxid, Index: 0, Value: 1000} @@ -852,62 +602,19 @@ func TestCreateCheckpointSweepTask_BulkSweepMarkersError(t *testing.T) { vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). Return(childOutpoints, nil) - vtxoRepo.On("GetVtxos", mock.Anything, childOutpoints). - Return(vtxos, nil) - - // Simulate a database error during bulk sweep dbError := fmt.Errorf("database connection failed") - markerRepo.On("BulkSweepMarkers", mock.Anything, mock.Anything, mock.Anything). + markerRepo.On("SweepVtxoOutpoints", mock.Anything, childOutpoints, mock.AnythingOfType("int64")). Return(dbError) task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) err := task() - // Verify the error is propagated require.Error(t, err) require.Contains(t, err.Error(), "database connection failed") } -// TestCreateCheckpointSweepTask_GetVtxosError verifies that when GetVtxos fails to -// retrieve the VTXOs associated with child outpoints, the error is properly propagated. -// This tests the error handling path before marker collection even begins. -func TestCreateCheckpointSweepTask_GetVtxosError(t *testing.T) { - wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() - - checkpointTxid := "checkpoint_vtxo_err" - vtxoOutpoint := domain.Outpoint{Txid: "vtxo_vtxo_err", VOut: 0} - - childOutpoints := []domain.Outpoint{{Txid: "child_vtxo_err", VOut: 0}} - - toSweep := ports.TxInput{Txid: checkpointTxid, Index: 0, Value: 1000} - - builder.On("BuildSweepTx", []ports.TxInput{toSweep}). - Return("sweeptxid_vtxo_err", "sweeptx_hex", nil) - - wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). - Return("sweeptxid_vtxo_err", nil) - - vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). - Return(childOutpoints, nil) - - // Simulate error when fetching VTXOs - vtxoRepo.On("GetVtxos", mock.Anything, childOutpoints). - Return(nil, fmt.Errorf("vtxo not found in database")) - - task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) - err := task() - - // Verify the error is propagated - require.Error(t, err) - require.Contains(t, err.Error(), "vtxo not found") - - // BulkSweepMarkers should never be called since we failed earlier - markerRepo.AssertNotCalled(t, "BulkSweepMarkers", mock.Anything, mock.Anything, mock.Anything) -} - // TestCreateCheckpointSweepTask_GetAllChildrenVtxosError verifies that when -// GetAllChildrenVtxos fails to retrieve child outpoints, the error is propagated. -// This tests the earliest error handling path in the sweep task. +// GetAllChildrenVtxos fails, the error is propagated. func TestCreateCheckpointSweepTask_GetAllChildrenVtxosError(t *testing.T) { wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() @@ -922,25 +629,19 @@ func TestCreateCheckpointSweepTask_GetAllChildrenVtxosError(t *testing.T) { wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). Return("sweeptxid_children_err", nil) - // Simulate error when fetching children vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). Return(nil, fmt.Errorf("failed to query children vtxos")) task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) err := task() - // Verify the error is propagated require.Error(t, err) require.Contains(t, err.Error(), "failed to query children") - - // Neither GetVtxos nor BulkSweepMarkers should be called - vtxoRepo.AssertNotCalled(t, "GetVtxos", mock.Anything, mock.Anything) - markerRepo.AssertNotCalled(t, "BulkSweepMarkers", mock.Anything, mock.Anything, mock.Anything) + markerRepo.AssertNotCalled(t, "SweepVtxoOutpoints", mock.Anything, mock.Anything, mock.Anything) } // TestCreateCheckpointSweepTask_BuildSweepTxError verifies that when BuildSweepTx -// fails to create the sweep transaction, the error is propagated and no marker -// operations are attempted. This tests the very first error handling path. +// fails, no sweep operations are attempted. func TestCreateCheckpointSweepTask_BuildSweepTxError(t *testing.T) { wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() @@ -949,26 +650,22 @@ func TestCreateCheckpointSweepTask_BuildSweepTxError(t *testing.T) { toSweep := ports.TxInput{Txid: checkpointTxid, Index: 0, Value: 1000} - // Simulate error when building sweep tx builder.On("BuildSweepTx", []ports.TxInput{toSweep}). Return("", "", fmt.Errorf("insufficient funds for sweep")) task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) err := task() - // Verify the error is propagated require.Error(t, err) require.Contains(t, err.Error(), "insufficient funds") - // No other operations should be called wallet.AssertNotCalled(t, "BroadcastTransaction", mock.Anything, mock.Anything) vtxoRepo.AssertNotCalled(t, "GetAllChildrenVtxos", mock.Anything, mock.Anything) - markerRepo.AssertNotCalled(t, "BulkSweepMarkers", mock.Anything, mock.Anything, mock.Anything) + markerRepo.AssertNotCalled(t, "SweepVtxoOutpoints", mock.Anything, mock.Anything, mock.Anything) } -// TestCreateCheckpointSweepTask_BroadcastError verifies that when BroadcastTransaction -// fails, the error is propagated and marker sweep operations are not attempted. -// This ensures we don't mark VTXOs as swept if the sweep transaction wasn't actually broadcast. +// TestCreateCheckpointSweepTask_BroadcastError verifies that when broadcast +// fails, VTXOs are not marked as swept. func TestCreateCheckpointSweepTask_BroadcastError(t *testing.T) { wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() @@ -980,25 +677,21 @@ func TestCreateCheckpointSweepTask_BroadcastError(t *testing.T) { builder.On("BuildSweepTx", []ports.TxInput{toSweep}). Return("sweeptxid_broadcast_err", "sweeptx_hex", nil) - // Simulate broadcast failure wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). Return("", fmt.Errorf("network timeout")) task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) err := task() - // Verify the error is propagated require.Error(t, err) require.Contains(t, err.Error(), "network timeout") - // Marker operations should not be attempted since broadcast failed vtxoRepo.AssertNotCalled(t, "GetAllChildrenVtxos", mock.Anything, mock.Anything) - markerRepo.AssertNotCalled(t, "BulkSweepMarkers", mock.Anything, mock.Anything, mock.Anything) + markerRepo.AssertNotCalled(t, "SweepVtxoOutpoints", mock.Anything, mock.Anything, mock.Anything) } -// TestCreateCheckpointSweepTask_NoChildrenVtxos verifies that when -// GetAllChildrenVtxos returns an empty slice (no children under the unrolled -// vtxo), the sweeper does not attempt to fetch VTXOs or sweep markers. +// TestCreateCheckpointSweepTask_NoChildrenVtxos verifies that an empty +// children list results in no sweep operations. func TestCreateCheckpointSweepTask_NoChildrenVtxos(t *testing.T) { wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() @@ -1013,154 +706,14 @@ func TestCreateCheckpointSweepTask_NoChildrenVtxos(t *testing.T) { wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). Return("sweeptxid_nc", nil) - // No children found vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). Return([]domain.Outpoint{}, nil) - // GetVtxos called with empty slice returns empty - vtxoRepo.On("GetVtxos", mock.Anything, []domain.Outpoint{}). - Return([]domain.Vtxo{}, nil) - task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) err := task() require.NoError(t, err) wallet.AssertExpectations(t) vtxoRepo.AssertExpectations(t) - // BulkSweepMarkers should NOT be called since there are no VTXOs/markers - markerRepo.AssertNotCalled(t, "BulkSweepMarkers", mock.Anything, mock.Anything, mock.Anything) -} - -// TestCreateCheckpointSweepTask_DuplicateMarkersAcrossVtxos verifies that when -// all VTXOs share the exact same marker set (100% overlap), only the unique -// markers are passed to BulkSweepMarkers. For example, 5 VTXOs each carrying -// {"marker-X", "marker-Y"} should result in exactly 2 markers being swept. -func TestCreateCheckpointSweepTask_DuplicateMarkersAcrossVtxos(t *testing.T) { - wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() - - checkpointTxid := "checkpoint_dup" - vtxoOutpoint := domain.Outpoint{Txid: "vtxo_dup", VOut: 0} - - // 5 children, all sharing the identical marker set - childOutpoints := []domain.Outpoint{ - {Txid: "dup1", VOut: 0}, - {Txid: "dup2", VOut: 0}, - {Txid: "dup3", VOut: 0}, - {Txid: "dup4", VOut: 0}, - {Txid: "dup5", VOut: 0}, - } - - identicalMarkers := []string{"marker-X", "marker-Y"} - vtxosWithDupMarkers := make([]domain.Vtxo, len(childOutpoints)) - for i, op := range childOutpoints { - vtxosWithDupMarkers[i] = domain.Vtxo{ - Outpoint: op, - MarkerIDs: identicalMarkers, - Depth: 50, - } - } - - toSweep := ports.TxInput{Txid: checkpointTxid, Index: 0, Value: 25000} - - builder.On("BuildSweepTx", []ports.TxInput{toSweep}). - Return("sweeptxid_dup", "sweeptx_hex", nil) - - wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). - Return("sweeptxid_dup", nil) - - vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). - Return(childOutpoints, nil) - - vtxoRepo.On("GetVtxos", mock.Anything, childOutpoints). - Return(vtxosWithDupMarkers, nil) - - // Despite 5 VTXOs, only 2 unique markers should be swept - markerRepo.On("BulkSweepMarkers", mock.Anything, mock.MatchedBy(func(markerIDs []string) bool { - if len(markerIDs) != 2 { - return false - } - markerSet := make(map[string]bool) - for _, id := range markerIDs { - markerSet[id] = true - } - return markerSet["marker-X"] && markerSet["marker-Y"] - }), mock.AnythingOfType("int64")).Return(nil) - - task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) - err := task() - - require.NoError(t, err) - markerRepo.AssertExpectations(t) -} - -// TestCreateCheckpointSweepTask_LargeMarkerSet verifies that the map-based -// deduplication works correctly at scale: 120 VTXOs carrying a mix of 60 -// unique markers. This ensures no scaling issues with the map allocation -// or iteration, and that the deduplicated set is passed correctly to -// BulkSweepMarkers. -func TestCreateCheckpointSweepTask_LargeMarkerSet(t *testing.T) { - wallet, vtxoRepo, markerRepo, builder, s := newTestSweeper() - - checkpointTxid := "checkpoint_large" - vtxoOutpoint := domain.Outpoint{Txid: "vtxo_large", VOut: 0} - - // 120 VTXOs, each with 2 markers drawn from a pool of 60 - childOutpoints := make([]domain.Outpoint, 120) - vtxosLarge := make([]domain.Vtxo, 120) - expectedMarkers := make(map[string]bool) - - for i := range 120 { - txid := fmt.Sprintf("large-child-%d", i) - childOutpoints[i] = domain.Outpoint{Txid: txid, VOut: 0} - - // Each VTXO gets two markers: marker-{i%60} and marker-{(i+1)%60} - m1 := fmt.Sprintf("marker-%d", i%60) - m2 := fmt.Sprintf("marker-%d", (i+1)%60) - expectedMarkers[m1] = true - expectedMarkers[m2] = true - - vtxosLarge[i] = domain.Vtxo{ - Outpoint: childOutpoints[i], - MarkerIDs: []string{m1, m2}, - Depth: uint32(i * 2), - } - } - - toSweep := ports.TxInput{Txid: checkpointTxid, Index: 0, Value: 1200000} - - builder.On("BuildSweepTx", []ports.TxInput{toSweep}). - Return("sweeptxid_large", "sweeptx_hex", nil) - - wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). - Return("sweeptxid_large", nil) - - vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). - Return(childOutpoints, nil) - - vtxoRepo.On("GetVtxos", mock.Anything, childOutpoints). - Return(vtxosLarge, nil) - - // Should have exactly 60 unique markers (marker-0 through marker-59) - markerRepo.On("BulkSweepMarkers", mock.Anything, mock.MatchedBy(func(markerIDs []string) bool { - if len(markerIDs) != 60 { - return false - } - seen := make(map[string]bool) - for _, id := range markerIDs { - if seen[id] { - return false // duplicate found — dedup failed - } - seen[id] = true - if !expectedMarkers[id] { - return false // unexpected marker - } - } - return true - }), mock.AnythingOfType("int64")).Return(nil) - - task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) - err := task() - - require.NoError(t, err) - markerRepo.AssertExpectations(t) + markerRepo.AssertNotCalled(t, "SweepVtxoOutpoints", mock.Anything, mock.Anything, mock.Anything) } diff --git a/internal/core/domain/marker_repo.go b/internal/core/domain/marker_repo.go index 085d8dabb..e32663308 100644 --- a/internal/core/domain/marker_repo.go +++ b/internal/core/domain/marker_repo.go @@ -38,6 +38,11 @@ type MarkerRepository interface { // in a single transaction. Each VTXO gets a marker with ID equal to its outpoint string. CreateRootMarkersForVtxos(ctx context.Context, vtxos []Vtxo) error + // SweepVtxoOutpoints marks specific VTXO outpoints as swept in the swept_vtxo + // table. Used by checkpoint sweeps where marker-based sweeping would over-reach + // across independent subtrees that share inherited markers. + SweepVtxoOutpoints(ctx context.Context, outpoints []Outpoint, sweptAt int64) error + // Chain traversal methods for GetVtxoChain optimization // GetVtxosByDepthRange retrieves VTXOs within a depth range GetVtxosByDepthRange(ctx context.Context, minDepth, maxDepth uint32) ([]Vtxo, error) diff --git a/internal/infrastructure/db/badger/marker_repo.go b/internal/infrastructure/db/badger/marker_repo.go index 700c15f08..d380ebae3 100644 --- a/internal/infrastructure/db/badger/marker_repo.go +++ b/internal/infrastructure/db/badger/marker_repo.go @@ -299,6 +299,27 @@ func (r *markerRepository) BulkSweepMarkers( return nil } +func (r *markerRepository) SweepVtxoOutpoints( + ctx context.Context, + outpoints []domain.Outpoint, + sweptAt int64, +) error { + for _, op := range outpoints { + var dto vtxoDTO + if err := r.vtxoStore.Get(op.String(), &dto); err != nil { + if err == badgerhold.ErrNotFound { + continue + } + return err + } + dto.Swept = true + if err := r.vtxoStore.Update(op.String(), dto); err != nil { + return err + } + } + return nil +} + func (r *markerRepository) SweepMarkerWithDescendants( ctx context.Context, markerID string, diff --git a/internal/infrastructure/db/postgres/marker_repo.go b/internal/infrastructure/db/postgres/marker_repo.go index 46cc620ab..0ac7a01f2 100644 --- a/internal/infrastructure/db/postgres/marker_repo.go +++ b/internal/infrastructure/db/postgres/marker_repo.go @@ -160,6 +160,27 @@ func (m *markerRepository) BulkSweepMarkers( }) } +func (m *markerRepository) SweepVtxoOutpoints( + ctx context.Context, + outpoints []domain.Outpoint, + sweptAt int64, +) error { + if len(outpoints) == 0 { + return nil + } + txids := make([]string, len(outpoints)) + vouts := make([]int32, len(outpoints)) + for i, op := range outpoints { + txids[i] = op.Txid + vouts[i] = int32(op.VOut) + } + return m.querier.BulkInsertSweptVtxos(ctx, queries.BulkInsertSweptVtxosParams{ + Txids: txids, + Vouts: vouts, + SweptAt: sweptAt, + }) +} + func (m *markerRepository) SweepMarkerWithDescendants( ctx context.Context, markerID string, @@ -343,7 +364,10 @@ func (m *markerRepository) GetVtxosByArkTxid( ctx context.Context, arkTxid string, ) ([]domain.Vtxo, error) { - rows, err := m.querier.SelectVtxosByArkTxid(ctx, arkTxid) + rows, err := m.querier.SelectVtxosByArkTxid( + ctx, + sql.NullString{String: arkTxid, Valid: arkTxid != ""}, + ) if err != nil { return nil, err } @@ -391,7 +415,7 @@ func rowToVtxoFromVtxoVw(row queries.VtxoVw) domain.Vtxo { SpentBy: row.SpentBy.String, Spent: row.Spent, Unrolled: row.Unrolled, - Swept: row.Swept, + Swept: row.Swept.Bool, Preconfirmed: row.Preconfirmed, ExpiresAt: row.ExpiresAt, CreatedAt: row.CreatedAt, @@ -430,7 +454,7 @@ func rowToVtxoFromMarkerQuery(row queries.SelectVtxosByMarkerIdRow) domain.Vtxo SpentBy: row.VtxoVw.SpentBy.String, Spent: row.VtxoVw.Spent, Unrolled: row.VtxoVw.Unrolled, - Swept: row.VtxoVw.Swept, + Swept: row.VtxoVw.Swept.Bool, Preconfirmed: row.VtxoVw.Preconfirmed, ExpiresAt: row.VtxoVw.ExpiresAt, CreatedAt: row.VtxoVw.CreatedAt, diff --git a/internal/infrastructure/db/postgres/migration/20260416120000_add_swept_vtxo.down.sql b/internal/infrastructure/db/postgres/migration/20260416120000_add_swept_vtxo.down.sql new file mode 100644 index 000000000..52e01148e --- /dev/null +++ b/internal/infrastructure/db/postgres/migration/20260416120000_add_swept_vtxo.down.sql @@ -0,0 +1,54 @@ +DROP TABLE IF EXISTS swept_vtxo; + +-- Restore views without swept_vtxo check +DROP VIEW IF EXISTS intent_with_inputs_vw; +DROP VIEW IF EXISTS vtxo_vw; + +CREATE VIEW vtxo_vw AS +SELECT v.*, + COALESCE(vc.commitments, '') AS commitments, + EXISTS ( + SELECT 1 FROM swept_marker sm + WHERE v.markers @> jsonb_build_array(sm.marker_id) + ) AS swept, + COALESCE(ap.asset_id, '') AS asset_id, + COALESCE(ap.amount, 0) AS asset_amount +FROM vtxo v +LEFT JOIN LATERAL ( + SELECT string_agg(commitment_txid, ',') AS commitments + FROM vtxo_commitment_txid + WHERE vtxo_txid = v.txid AND vtxo_vout = v.vout +) vc ON true +LEFT JOIN ( + SELECT txid, vout, asset_id, amount + FROM asset_projection + GROUP BY txid, vout, asset_id, amount +) ap +ON ap.txid = v.txid AND ap.vout = v.vout; + +CREATE VIEW intent_with_inputs_vw AS +SELECT + v.txid, v.vout, v.pubkey, v.amount, v.expires_at, v.created_at, + v.commitment_txid, v.spent_by, v.spent, v.unrolled, v.preconfirmed, + v.settled_by, v.ark_txid, v.intent_id, v.updated_at, v.depth, v.markers, + COALESCE(vc.commitments, '') AS commitments, + EXISTS ( + SELECT 1 FROM swept_marker sm + WHERE v.markers @> jsonb_build_array(sm.marker_id) + ) AS swept, + COALESCE(ap.asset_id, '') AS asset_id, + COALESCE(ap.amount, 0) AS asset_amount, + intent.id, intent.round_id, intent.proof, intent.message, + intent.txid AS intent_txid +FROM intent +LEFT OUTER JOIN vtxo v ON intent.id = v.intent_id +LEFT JOIN LATERAL ( + SELECT string_agg(commitment_txid, ',') AS commitments + FROM vtxo_commitment_txid + WHERE vtxo_txid = v.txid AND vtxo_vout = v.vout +) vc ON true +LEFT JOIN ( + SELECT txid, vout, asset_id, amount + FROM asset_projection + GROUP BY txid, vout, asset_id, amount +) ap ON ap.txid = v.txid AND ap.vout = v.vout; diff --git a/internal/infrastructure/db/postgres/migration/20260416120000_add_swept_vtxo.up.sql b/internal/infrastructure/db/postgres/migration/20260416120000_add_swept_vtxo.up.sql new file mode 100644 index 000000000..dd30fe134 --- /dev/null +++ b/internal/infrastructure/db/postgres/migration/20260416120000_add_swept_vtxo.up.sql @@ -0,0 +1,95 @@ +-- Per-outpoint sweep tracking for checkpoint sweeps. +-- Markers can be shared across independent subtrees when offchain txs +-- consolidate inputs from different lineages. BulkSweepMarkers is safe +-- for batch sweeps (entire round) but over-reaches for checkpoint sweeps +-- (single subtree). This table tracks per-outpoint sweep status for the +-- checkpoint path. +CREATE TABLE IF NOT EXISTS swept_vtxo ( + txid TEXT NOT NULL, + vout INTEGER NOT NULL, + swept_at BIGINT NOT NULL, + PRIMARY KEY (txid, vout) +); + +-- Rebuild vtxo_vw: swept if marker in swept_marker OR outpoint in swept_vtxo +DROP VIEW IF EXISTS intent_with_inputs_vw; +DROP VIEW IF EXISTS vtxo_vw; + +CREATE VIEW vtxo_vw AS +SELECT v.*, + COALESCE(vc.commitments, '') AS commitments, + ( + EXISTS ( + SELECT 1 FROM swept_marker sm + WHERE v.markers @> jsonb_build_array(sm.marker_id) + ) + OR EXISTS ( + SELECT 1 FROM swept_vtxo sv + WHERE sv.txid = v.txid AND sv.vout = v.vout + ) + ) AS swept, + COALESCE(ap.asset_id, '') AS asset_id, + COALESCE(ap.amount, 0) AS asset_amount +FROM vtxo v +LEFT JOIN LATERAL ( + SELECT string_agg(commitment_txid, ',') AS commitments + FROM vtxo_commitment_txid + WHERE vtxo_txid = v.txid AND vtxo_vout = v.vout +) vc ON true +LEFT JOIN ( + SELECT txid, vout, asset_id, amount + FROM asset_projection + GROUP BY txid, vout, asset_id, amount +) ap +ON ap.txid = v.txid AND ap.vout = v.vout; + +-- Rebuild intent_with_inputs_vw +CREATE VIEW intent_with_inputs_vw AS +SELECT + v.txid, + v.vout, + v.pubkey, + v.amount, + v.expires_at, + v.created_at, + v.commitment_txid, + v.spent_by, + v.spent, + v.unrolled, + v.preconfirmed, + v.settled_by, + v.ark_txid, + v.intent_id, + v.updated_at, + v.depth, + v.markers, + COALESCE(vc.commitments, '') AS commitments, + ( + EXISTS ( + SELECT 1 FROM swept_marker sm + WHERE v.markers @> jsonb_build_array(sm.marker_id) + ) + OR EXISTS ( + SELECT 1 FROM swept_vtxo sv + WHERE sv.txid = v.txid AND sv.vout = v.vout + ) + ) AS swept, + COALESCE(ap.asset_id, '') AS asset_id, + COALESCE(ap.amount, 0) AS asset_amount, + intent.id, + intent.round_id, + intent.proof, + intent.message, + intent.txid AS intent_txid +FROM intent +LEFT OUTER JOIN vtxo v ON intent.id = v.intent_id +LEFT JOIN LATERAL ( + SELECT string_agg(commitment_txid, ',') AS commitments + FROM vtxo_commitment_txid + WHERE vtxo_txid = v.txid AND vtxo_vout = v.vout +) vc ON true +LEFT JOIN ( + SELECT txid, vout, asset_id, amount + FROM asset_projection + GROUP BY txid, vout, asset_id, amount +) ap ON ap.txid = v.txid AND ap.vout = v.vout; diff --git a/internal/infrastructure/db/postgres/sqlc/queries/models.go b/internal/infrastructure/db/postgres/sqlc/queries/models.go index 4250198eb..e113291de 100644 --- a/internal/infrastructure/db/postgres/sqlc/queries/models.go +++ b/internal/infrastructure/db/postgres/sqlc/queries/models.go @@ -220,6 +220,12 @@ type SweptMarker struct { SweptAt int64 } +type SweptVtxo struct { + Txid string + Vout int32 + SweptAt int64 +} + type Tx struct { Txid string Tx string @@ -274,7 +280,7 @@ type VtxoVw struct { Depth int32 Markers json.RawMessage Commitments []byte - Swept bool + Swept sql.NullBool AssetID string AssetAmount string } diff --git a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go index c3d8c4dc5..309ed699a 100644 --- a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go @@ -79,6 +79,23 @@ func (q *Queries) BulkInsertSweptMarkers(ctx context.Context, arg BulkInsertSwep return err } +const bulkInsertSweptVtxos = `-- name: BulkInsertSweptVtxos :exec +INSERT INTO swept_vtxo (txid, vout, swept_at) +SELECT unnest($1::text[]), unnest($2::integer[]), $3 +ON CONFLICT(txid, vout) DO NOTHING +` + +type BulkInsertSweptVtxosParams struct { + Txids []string + Vouts []int32 + SweptAt int64 +} + +func (q *Queries) BulkInsertSweptVtxos(ctx context.Context, arg BulkInsertSweptVtxosParams) error { + _, err := q.db.ExecContext(ctx, bulkInsertSweptVtxos, pq.Array(arg.Txids), pq.Array(arg.Vouts), arg.SweptAt) + return err +} + const clearIntentFees = `-- name: ClearIntentFees :exec INSERT INTO intent_fees ( offchain_input_fee_program, @@ -194,6 +211,23 @@ func (q *Queries) InsertSweptMarker(ctx context.Context, arg InsertSweptMarkerPa return err } +const insertSweptVtxo = `-- name: InsertSweptVtxo :exec +INSERT INTO swept_vtxo (txid, vout, swept_at) +VALUES ($1, $2, $3) +ON CONFLICT(txid, vout) DO NOTHING +` + +type InsertSweptVtxoParams struct { + Txid string + Vout int32 + SweptAt int64 +} + +func (q *Queries) InsertSweptVtxo(ctx context.Context, arg InsertSweptVtxoParams) error { + _, err := q.db.ExecContext(ctx, insertSweptVtxo, arg.Txid, arg.Vout, arg.SweptAt) + return err +} + const insertVtxoAssetProjection = `-- name: InsertVtxoAssetProjection :exec INSERT INTO asset_projection (asset_id, txid, vout, amount) VALUES ($1, $2, $3, $4) @@ -1991,7 +2025,7 @@ SELECT txid, vout, pubkey, amount, expires_at, created_at, commitment_txid, spen ` // Get all VTXOs created by a specific ark tx (offchain tx) -func (q *Queries) SelectVtxosByArkTxid(ctx context.Context, arkTxid string) ([]VtxoVw, error) { +func (q *Queries) SelectVtxosByArkTxid(ctx context.Context, arkTxid sql.NullString) ([]VtxoVw, error) { rows, err := q.db.QueryContext(ctx, selectVtxosByArkTxid, arkTxid) if err != nil { return nil, err diff --git a/internal/infrastructure/db/postgres/sqlc/query.sql b/internal/infrastructure/db/postgres/sqlc/query.sql index 33747e6e6..6d3b4e33b 100644 --- a/internal/infrastructure/db/postgres/sqlc/query.sql +++ b/internal/infrastructure/db/postgres/sqlc/query.sql @@ -538,3 +538,13 @@ SELECT control_asset_id FROM asset WHERE id = $1; -- name: SelectAssetExists :one SELECT 1 FROM asset WHERE id = $1 LIMIT 1; + +-- name: InsertSweptVtxo :exec +INSERT INTO swept_vtxo (txid, vout, swept_at) +VALUES (@txid, @vout, @swept_at) +ON CONFLICT(txid, vout) DO NOTHING; + +-- name: BulkInsertSweptVtxos :exec +INSERT INTO swept_vtxo (txid, vout, swept_at) +SELECT unnest(@txids::text[]), unnest(@vouts::integer[]), @swept_at +ON CONFLICT(txid, vout) DO NOTHING; diff --git a/internal/infrastructure/db/postgres/vtxo_repo.go b/internal/infrastructure/db/postgres/vtxo_repo.go index 912264f70..24b78056e 100644 --- a/internal/infrastructure/db/postgres/vtxo_repo.go +++ b/internal/infrastructure/db/postgres/vtxo_repo.go @@ -527,7 +527,7 @@ func rowToVtxo(row queries.VtxoVw) domain.Vtxo { SpentBy: row.SpentBy.String, Spent: row.Spent, Unrolled: row.Unrolled, - Swept: row.Swept, + Swept: row.Swept.Bool, Preconfirmed: row.Preconfirmed, ExpiresAt: row.ExpiresAt, CreatedAt: row.CreatedAt, diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index c7674eab5..51a0b4678 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -209,6 +209,8 @@ func TestService(t *testing.T) { testListVtxosMarkerSweptFiltering(t, svc) testAddMarkerFailureFallbackToParentMarkers(t, svc) testSweepableUnrolledExcludesMarkerSwept(t, svc) + testSweepVtxoOutpointsNoOverreach(t, svc) + testSweepVtxoOutpointsEdgeCases(t, svc) testConvergentMultiParentMarkerDAG(t, svc) testSweepMarkerWithDescendantsDeepChain(t, svc) testScheduledSessionRepository(t, svc) @@ -4816,6 +4818,235 @@ func testSweepableUnrolledExcludesMarkerSwept(t *testing.T, svc ports.RepoManage }) } +// testSweepVtxoOutpointsNoOverreach proves that per-outpoint sweeping via +// SweepVtxoOutpoints does NOT over-reach across independent subtrees that share +// a marker. This is the scenario where marker-based sweeping (BulkSweepMarkers) +// would incorrectly sweep an unrelated sibling VTXO. +// +// Setup: two batch VTXOs (X, Y) from the same round share a marker M_root. +// Sweeping X's outpoint via SweepVtxoOutpoints should mark X as swept but +// leave Y unswept, even though both carry M_root. +func testSweepVtxoOutpointsNoOverreach(t *testing.T, svc ports.RepoManager) { + t.Run("test_sweep_vtxo_outpoints_no_overreach", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + suffix := randomString(16) + testPubkey := "overreach-pk-" + suffix + + // Create a finalized round. + roundId := uuid.New().String() + commitmentTxid := randomString(32) + round := domain.NewRoundFromEvents([]domain.Event{ + domain.RoundStarted{ + RoundEvent: domain.RoundEvent{Id: roundId, Type: domain.EventTypeRoundStarted}, + Timestamp: time.Now().Unix(), + }, + domain.RoundFinalizationStarted{ + RoundEvent: domain.RoundEvent{ + Id: roundId, + Type: domain.EventTypeRoundFinalizationStarted, + }, + CommitmentTxid: commitmentTxid, + CommitmentTx: emptyTx, + VtxoTree: vtxoTree, + Connectors: connectorsTree, + VtxoTreeExpiration: 3600, + }, + domain.RoundFinalized{ + RoundEvent: domain.RoundEvent{ + Id: roundId, + Type: domain.EventTypeRoundFinalized, + }, + FinalCommitmentTx: emptyTx, + Timestamp: time.Now().Unix(), + }, + }) + require.NoError(t, svc.Rounds().AddOrUpdateRound(ctx, *round)) + + // Create a shared marker — simulates two sibling VTXOs from the same + // offchain tx inheriting the same parent marker. + sharedMarkerID := "overreach-shared-m-" + suffix + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: sharedMarkerID, Depth: 0, + })) + + // VTXO X — will be swept via SweepVtxoOutpoints + vtxoX := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "overreach-X-" + suffix, VOut: 0}, + PubKey: testPubkey, + Amount: 5000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + CreatedAt: time.Now().Unix(), + ExpiresAt: time.Now().Add(time.Hour).Unix(), + Depth: 1, + MarkerIDs: []string{sharedMarkerID}, + } + + // VTXO Y — independent sibling sharing the same marker, should NOT be swept + vtxoY := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "overreach-Y-" + suffix, VOut: 0}, + PubKey: testPubkey, + Amount: 3000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + CreatedAt: time.Now().Unix(), + ExpiresAt: time.Now().Add(time.Hour).Unix(), + Depth: 1, + MarkerIDs: []string{sharedMarkerID}, + } + + require.NoError(t, svc.Vtxos().AddVtxos(ctx, []domain.Vtxo{vtxoX, vtxoY})) + for _, v := range []domain.Vtxo{vtxoX, vtxoY} { + require.NoError(t, svc.Markers().UpdateVtxoMarkers(ctx, v.Outpoint, v.MarkerIDs)) + } + + // Sweep ONLY vtxoX via per-outpoint sweeping. + sweptAt := time.Now().UnixMilli() + err := svc.Markers().SweepVtxoOutpoints(ctx, []domain.Outpoint{vtxoX.Outpoint}, sweptAt) + require.NoError(t, err) + + // Verify: X is swept, Y is NOT swept. + unspent, spent, err := svc.Vtxos().GetAllNonUnrolledVtxos(ctx, testPubkey) + require.NoError(t, err) + + spentTxids := make(map[string]bool) + for _, v := range spent { + spentTxids[v.Txid] = true + } + unspentTxids := make(map[string]bool) + for _, v := range unspent { + unspentTxids[v.Txid] = true + } + + require.True(t, spentTxids[vtxoX.Outpoint.Txid], + "vtxo X should be swept via SweepVtxoOutpoints") + require.True(t, unspentTxids[vtxoY.Outpoint.Txid], + "vtxo Y must NOT be swept — it shares a marker with X but was not in the sweep set") + + // Contrast: if we had used BulkSweepMarkers(sharedMarkerID) instead, + // Y would also be swept. That's the over-reach this fix prevents. + }) +} + +// testSweepVtxoOutpointsEdgeCases covers edge cases for the dual sweep tracking: +// - Double sweep: a VTXO swept via both marker AND outpoint stays swept +// - Non-existent outpoints: SweepVtxoOutpoints silently ignores them +// - Empty outpoints: no-op without error +func testSweepVtxoOutpointsEdgeCases(t *testing.T, svc ports.RepoManager) { + t.Run("test_sweep_vtxo_outpoints_edge_cases", func(t *testing.T) { + if svc.Markers() == nil { + t.Skip("marker repository not available for this data store") + } + ctx := context.Background() + suffix := randomString(16) + testPubkey := "edge-pk-" + suffix + + // Create round. + roundId := uuid.New().String() + commitmentTxid := randomString(32) + round := domain.NewRoundFromEvents([]domain.Event{ + domain.RoundStarted{ + RoundEvent: domain.RoundEvent{Id: roundId, Type: domain.EventTypeRoundStarted}, + Timestamp: time.Now().Unix(), + }, + domain.RoundFinalizationStarted{ + RoundEvent: domain.RoundEvent{ + Id: roundId, + Type: domain.EventTypeRoundFinalizationStarted, + }, + CommitmentTxid: commitmentTxid, + CommitmentTx: emptyTx, + VtxoTree: vtxoTree, + Connectors: connectorsTree, + VtxoTreeExpiration: 3600, + }, + domain.RoundFinalized{ + RoundEvent: domain.RoundEvent{ + Id: roundId, + Type: domain.EventTypeRoundFinalized, + }, + FinalCommitmentTx: emptyTx, + Timestamp: time.Now().Unix(), + }, + }) + require.NoError(t, svc.Rounds().AddOrUpdateRound(ctx, *round)) + + markerID := "edge-m-" + suffix + require.NoError(t, svc.Markers().AddMarker(ctx, domain.Marker{ + ID: markerID, Depth: 0, + })) + + vtxo := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: "edge-vtxo-" + suffix, VOut: 0}, + PubKey: testPubkey, + Amount: 5000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + CreatedAt: time.Now().Unix(), + ExpiresAt: time.Now().Add(time.Hour).Unix(), + Depth: 0, + MarkerIDs: []string{markerID}, + } + require.NoError(t, svc.Vtxos().AddVtxos(ctx, []domain.Vtxo{vtxo})) + require.NoError(t, svc.Markers().UpdateVtxoMarkers(ctx, vtxo.Outpoint, vtxo.MarkerIDs)) + + sweptAt := time.Now().UnixMilli() + + // Edge case 1: empty outpoints — should be a no-op + err := svc.Markers().SweepVtxoOutpoints(ctx, []domain.Outpoint{}, sweptAt) + require.NoError(t, err) + + // Edge case 2: non-existent outpoints — should not error + err = svc.Markers().SweepVtxoOutpoints(ctx, []domain.Outpoint{ + {Txid: "does-not-exist", VOut: 99}, + }, sweptAt) + require.NoError(t, err) + + // Verify VTXO is still unswept after those no-ops + unspent, _, err := svc.Vtxos().GetAllNonUnrolledVtxos(ctx, testPubkey) + require.NoError(t, err) + found := false + for _, v := range unspent { + if v.Txid == vtxo.Outpoint.Txid { + found = true + } + } + require.True(t, found, "vtxo should still be unswept after empty/nonexistent sweep calls") + + // Edge case 3: double sweep — sweep via marker THEN via outpoint + require.NoError(t, svc.Markers().BulkSweepMarkers(ctx, []string{markerID}, sweptAt)) + + // VTXO is now swept via marker + _, spent, err := svc.Vtxos().GetAllNonUnrolledVtxos(ctx, testPubkey) + require.NoError(t, err) + foundInSpent := false + for _, v := range spent { + if v.Txid == vtxo.Outpoint.Txid { + foundInSpent = true + } + } + require.True(t, foundInSpent, "vtxo should be swept after BulkSweepMarkers") + + // Now also sweep via outpoint — should not error (idempotent) + err = svc.Markers().SweepVtxoOutpoints(ctx, []domain.Outpoint{vtxo.Outpoint}, sweptAt) + require.NoError(t, err) + + // VTXO should still be swept (via both paths now) + _, spent2, err := svc.Vtxos().GetAllNonUnrolledVtxos(ctx, testPubkey) + require.NoError(t, err) + foundInSpent2 := false + for _, v := range spent2 { + if v.Txid == vtxo.Outpoint.Txid { + foundInSpent2 = true + } + } + require.True(t, foundInSpent2, "vtxo should remain swept after double sweep via both paths") + }) +} + // testConvergentMultiParentMarkerDAG builds a diamond-shaped marker DAG where two // independent root→mid branches converge into a single merge marker, then extend // to a leaf. Verifies GetVtxoChainByMarkers returns correct VTXOs per marker set, diff --git a/internal/infrastructure/db/sqlite/marker_repo.go b/internal/infrastructure/db/sqlite/marker_repo.go index e7875e10b..f8eb9c963 100644 --- a/internal/infrastructure/db/sqlite/marker_repo.go +++ b/internal/infrastructure/db/sqlite/marker_repo.go @@ -161,6 +161,29 @@ func (m *markerRepository) BulkSweepMarkers( return execTx(ctx, m.db, txBody) } +func (m *markerRepository) SweepVtxoOutpoints( + ctx context.Context, + outpoints []domain.Outpoint, + sweptAt int64, +) error { + if len(outpoints) == 0 { + return nil + } + txBody := func(qtx *queries.Queries) error { + for _, op := range outpoints { + if err := qtx.InsertSweptVtxo(ctx, queries.InsertSweptVtxoParams{ + Txid: op.Txid, + Vout: int64(op.VOut), + SweptAt: sweptAt, + }); err != nil { + return err + } + } + return nil + } + return execTx(ctx, m.db, txBody) +} + func (m *markerRepository) SweepMarkerWithDescendants( ctx context.Context, markerID string, @@ -233,7 +256,7 @@ func (m *markerRepository) UpdateVtxoMarkers( return fmt.Errorf("failed to marshal markers: %w", err) } return m.querier.UpdateVtxoMarkers(ctx, queries.UpdateVtxoMarkersParams{ - Markers: sql.NullString{String: string(markersJSON), Valid: len(markerIDs) > 0}, + Markers: string(markersJSON), Txid: outpoint.Txid, Vout: int64(outpoint.VOut), }) @@ -421,12 +444,12 @@ func rowToVtxoFromMarkerQuery(row queries.SelectVtxosByMarkerIdRow) domain.Vtxo SpentBy: row.VtxoVw.SpentBy.String, Spent: row.VtxoVw.Spent, Unrolled: row.VtxoVw.Unrolled, - Swept: row.VtxoVw.Swept != 0, + Swept: toBool(row.VtxoVw.Swept), Preconfirmed: row.VtxoVw.Preconfirmed, ExpiresAt: row.VtxoVw.ExpiresAt, CreatedAt: row.VtxoVw.CreatedAt, Depth: uint32(row.VtxoVw.Depth), - MarkerIDs: parseMarkersJSON(row.VtxoVw.Markers.String), + MarkerIDs: parseMarkersJSON(row.VtxoVw.Markers), } } @@ -449,12 +472,12 @@ func rowToVtxoFromDepthRangeQuery(row queries.SelectVtxosByDepthRangeRow) domain SpentBy: row.VtxoVw.SpentBy.String, Spent: row.VtxoVw.Spent, Unrolled: row.VtxoVw.Unrolled, - Swept: row.VtxoVw.Swept != 0, + Swept: toBool(row.VtxoVw.Swept), Preconfirmed: row.VtxoVw.Preconfirmed, ExpiresAt: row.VtxoVw.ExpiresAt, CreatedAt: row.VtxoVw.CreatedAt, Depth: uint32(row.VtxoVw.Depth), - MarkerIDs: parseMarkersJSON(row.VtxoVw.Markers.String), + MarkerIDs: parseMarkersJSON(row.VtxoVw.Markers), } } @@ -477,12 +500,12 @@ func rowToVtxoFromArkTxidQuery(row queries.SelectVtxosByArkTxidRow) domain.Vtxo SpentBy: row.VtxoVw.SpentBy.String, Spent: row.VtxoVw.Spent, Unrolled: row.VtxoVw.Unrolled, - Swept: row.VtxoVw.Swept != 0, + Swept: toBool(row.VtxoVw.Swept), Preconfirmed: row.VtxoVw.Preconfirmed, ExpiresAt: row.VtxoVw.ExpiresAt, CreatedAt: row.VtxoVw.CreatedAt, Depth: uint32(row.VtxoVw.Depth), - MarkerIDs: parseMarkersJSON(row.VtxoVw.Markers.String), + MarkerIDs: parseMarkersJSON(row.VtxoVw.Markers), } } @@ -505,12 +528,12 @@ func rowToVtxoFromChainQuery(row queries.SelectVtxoChainByMarkerRow) domain.Vtxo SpentBy: row.VtxoVw.SpentBy.String, Spent: row.VtxoVw.Spent, Unrolled: row.VtxoVw.Unrolled, - Swept: row.VtxoVw.Swept != 0, + Swept: toBool(row.VtxoVw.Swept), Preconfirmed: row.VtxoVw.Preconfirmed, ExpiresAt: row.VtxoVw.ExpiresAt, CreatedAt: row.VtxoVw.CreatedAt, Depth: uint32(row.VtxoVw.Depth), - MarkerIDs: parseMarkersJSON(row.VtxoVw.Markers.String), + MarkerIDs: parseMarkersJSON(row.VtxoVw.Markers), } } diff --git a/internal/infrastructure/db/sqlite/migration/20260416120000_add_swept_vtxo.down.sql b/internal/infrastructure/db/sqlite/migration/20260416120000_add_swept_vtxo.down.sql new file mode 100644 index 000000000..6c947b853 --- /dev/null +++ b/internal/infrastructure/db/sqlite/migration/20260416120000_add_swept_vtxo.down.sql @@ -0,0 +1,30 @@ +DROP TABLE IF EXISTS swept_vtxo; + +DROP VIEW IF EXISTS intent_with_inputs_vw; +DROP VIEW IF EXISTS vtxo_vw; + +CREATE VIEW vtxo_vw AS +SELECT v.*, + COALESCE(( + SELECT group_concat(commitment_txid, ',') + FROM vtxo_commitment_txid + WHERE vtxo_txid = v.txid AND vtxo_vout = v.vout + ), '') AS commitments, + EXISTS ( + SELECT 1 FROM swept_marker sm + JOIN json_each(v.markers) j ON j.value = sm.marker_id + ) AS swept, + COALESCE(ap.asset_id, '') AS asset_id, + COALESCE(ap.amount, 0) AS asset_amount +FROM vtxo v +LEFT JOIN ( + SELECT DISTINCT txid, vout, asset_id, amount + FROM asset_projection +) AS ap +ON ap.txid = v.txid AND ap.vout = v.vout; + +CREATE VIEW intent_with_inputs_vw AS +SELECT vtxo_vw.*, intent.id, intent.round_id, intent.proof, intent.message, intent.txid AS intent_txid +FROM intent +LEFT OUTER JOIN vtxo_vw +ON intent.id = vtxo_vw.intent_id; diff --git a/internal/infrastructure/db/sqlite/migration/20260416120000_add_swept_vtxo.up.sql b/internal/infrastructure/db/sqlite/migration/20260416120000_add_swept_vtxo.up.sql new file mode 100644 index 000000000..cf4d64bc5 --- /dev/null +++ b/internal/infrastructure/db/sqlite/migration/20260416120000_add_swept_vtxo.up.sql @@ -0,0 +1,82 @@ +-- Per-outpoint sweep tracking for checkpoint sweeps (see Postgres migration). +CREATE TABLE IF NOT EXISTS swept_vtxo ( + txid TEXT NOT NULL, + vout INTEGER NOT NULL, + swept_at INTEGER NOT NULL, + PRIMARY KEY (txid, vout) +); + +-- Rebuild vtxo_vw: swept if marker in swept_marker OR outpoint in swept_vtxo +DROP VIEW IF EXISTS intent_with_inputs_vw; +DROP VIEW IF EXISTS vtxo_vw; + +CREATE VIEW vtxo_vw AS +SELECT v.*, + COALESCE(( + SELECT group_concat(commitment_txid, ',') + FROM vtxo_commitment_txid + WHERE vtxo_txid = v.txid AND vtxo_vout = v.vout + ), '') AS commitments, + ( + EXISTS ( + SELECT 1 FROM swept_marker sm + JOIN json_each(v.markers) j ON j.value = sm.marker_id + ) + OR EXISTS ( + SELECT 1 FROM swept_vtxo sv + WHERE sv.txid = v.txid AND sv.vout = v.vout + ) + ) AS swept, + COALESCE(ap.asset_id, '') AS asset_id, + COALESCE(ap.amount, 0) AS asset_amount +FROM vtxo v +LEFT JOIN ( + SELECT DISTINCT txid, vout, asset_id, amount + FROM asset_projection +) AS ap +ON ap.txid = v.txid AND ap.vout = v.vout; + +CREATE VIEW intent_with_inputs_vw AS +SELECT + v.txid, + v.vout, + v.pubkey, + v.amount, + v.expires_at, + v.created_at, + v.commitment_txid, + v.spent_by, + v.spent, + v.unrolled, + v.preconfirmed, + v.settled_by, + v.ark_txid, + v.intent_id, + v.updated_at, + v.depth, + v.markers, + COALESCE(( + SELECT group_concat(vc.commitment_txid) + FROM vtxo_commitment_txid vc + WHERE vc.vtxo_txid = v.txid AND vc.vtxo_vout = v.vout + ), '') AS commitments, + ( + EXISTS ( + SELECT 1 FROM swept_marker sm + JOIN json_each(v.markers) j ON j.value = sm.marker_id + ) + OR EXISTS ( + SELECT 1 FROM swept_vtxo sv + WHERE sv.txid = v.txid AND sv.vout = v.vout + ) + ) AS swept, + COALESCE(ap.asset_id, '') AS asset_id, + COALESCE(ap.amount, 0) AS asset_amount, + intent.id, + intent.round_id, + intent.proof, + intent.message, + intent.txid AS intent_txid +FROM intent +LEFT OUTER JOIN vtxo v ON intent.id = v.intent_id +LEFT JOIN asset_projection ap ON v.txid = ap.txid AND v.vout = ap.vout; diff --git a/internal/infrastructure/db/sqlite/round_repo.go b/internal/infrastructure/db/sqlite/round_repo.go index dd5ccb486..e35cf8770 100644 --- a/internal/infrastructure/db/sqlite/round_repo.go +++ b/internal/infrastructure/db/sqlite/round_repo.go @@ -710,7 +710,7 @@ func combinedRowToVtxo(row queries.IntentWithInputsVw) domain.Vtxo { SpentBy: row.SpentBy.String, Spent: row.Spent.Bool, Unrolled: row.Unrolled.Bool, - Swept: row.Swept != 0, + Swept: toBool(row.Swept), Preconfirmed: row.Preconfirmed.Bool, ExpiresAt: row.ExpiresAt.Int64, CreatedAt: row.CreatedAt.Int64, diff --git a/internal/infrastructure/db/sqlite/sqlc/queries/models.go b/internal/infrastructure/db/sqlite/sqlc/queries/models.go index 003949df3..d7cfb7279 100644 --- a/internal/infrastructure/db/sqlite/sqlc/queries/models.go +++ b/internal/infrastructure/db/sqlite/sqlc/queries/models.go @@ -79,7 +79,7 @@ type IntentWithInputsVw struct { Depth sql.NullInt64 Markers sql.NullString Commitments interface{} - Swept int64 + Swept interface{} AssetID string AssetAmount string ID sql.NullString @@ -206,6 +206,12 @@ type SweptMarker struct { SweptAt int64 } +type SweptVtxo struct { + Txid string + Vout int64 + SweptAt int64 +} + type Tx struct { Txid string Tx string @@ -232,7 +238,7 @@ type Vtxo struct { IntentID sql.NullString UpdatedAt sql.NullInt64 Depth int64 - Markers sql.NullString + Markers string } type VtxoCommitmentTxid struct { @@ -258,9 +264,9 @@ type VtxoVw struct { IntentID sql.NullString UpdatedAt sql.NullInt64 Depth int64 - Markers sql.NullString + Markers string Commitments interface{} - Swept int64 + Swept interface{} AssetID string AssetAmount string } diff --git a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go index 15adcd901..b63bd0aa1 100644 --- a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go @@ -177,6 +177,22 @@ func (q *Queries) InsertSweptMarker(ctx context.Context, arg InsertSweptMarkerPa return err } +const insertSweptVtxo = `-- name: InsertSweptVtxo :exec +INSERT OR IGNORE INTO swept_vtxo (txid, vout, swept_at) +VALUES (?, ?, ?) +` + +type InsertSweptVtxoParams struct { + Txid string + Vout int64 + SweptAt int64 +} + +func (q *Queries) InsertSweptVtxo(ctx context.Context, arg InsertSweptVtxoParams) error { + _, err := q.db.ExecContext(ctx, insertSweptVtxo, arg.Txid, arg.Vout, arg.SweptAt) + return err +} + const insertVtxoAssetProjection = `-- name: InsertVtxoAssetProjection :exec INSERT INTO asset_projection (asset_id, txid, vout, amount) VALUES (?1, ?2, ?3, ?4) @@ -2459,7 +2475,7 @@ UPDATE vtxo SET markers = ?1 WHERE txid = ?2 AND vout = ?3 ` type UpdateVtxoMarkersParams struct { - Markers sql.NullString + Markers string Txid string Vout int64 } @@ -2866,7 +2882,7 @@ type UpsertVtxoParams struct { ExpiresAt int64 CreatedAt int64 Depth int64 - Markers sql.NullString + Markers string } func (q *Queries) UpsertVtxo(ctx context.Context, arg UpsertVtxoParams) error { diff --git a/internal/infrastructure/db/sqlite/sqlc/query.sql b/internal/infrastructure/db/sqlite/sqlc/query.sql index 204c96fe8..3e926ea32 100644 --- a/internal/infrastructure/db/sqlite/sqlc/query.sql +++ b/internal/infrastructure/db/sqlite/sqlc/query.sql @@ -543,3 +543,7 @@ SELECT control_asset_id FROM asset WHERE id = ?; -- name: SelectAssetExists :one SELECT 1 FROM asset WHERE id = ? LIMIT 1; + +-- name: InsertSweptVtxo :exec +INSERT OR IGNORE INTO swept_vtxo (txid, vout, swept_at) +VALUES (?, ?, ?); diff --git a/internal/infrastructure/db/sqlite/vtxo_repo.go b/internal/infrastructure/db/sqlite/vtxo_repo.go index 434000d6f..848740fa1 100644 --- a/internal/infrastructure/db/sqlite/vtxo_repo.go +++ b/internal/infrastructure/db/sqlite/vtxo_repo.go @@ -51,7 +51,7 @@ func (v *vtxoRepository) AddVtxos(ctx context.Context, vtxos []domain.Vtxo) erro if err != nil { return fmt.Errorf("failed to marshal markers: %w", err) } - markersJSON := sql.NullString{String: string(markersData), Valid: true} + markersJSON := string(markersData) if err := querierWithTx.UpsertVtxo( ctx, queries.UpsertVtxoParams{ @@ -551,12 +551,12 @@ func rowToVtxo(row queries.VtxoVw) domain.Vtxo { SpentBy: row.SpentBy.String, Spent: row.Spent, Unrolled: row.Unrolled, - Swept: row.Swept != 0, + Swept: toBool(row.Swept), Preconfirmed: row.Preconfirmed, ExpiresAt: row.ExpiresAt, CreatedAt: row.CreatedAt, Depth: uint32(row.Depth), - MarkerIDs: parseMarkersJSONFromVtxo(row.Markers.String), + MarkerIDs: parseMarkersJSONFromVtxo(row.Markers), Assets: assets, } } @@ -570,6 +570,21 @@ func rowToAsset(row queries.VtxoVw) domain.AssetDenomination { } } +// toBool converts an interface{} (from a SQLite view expression that sqlc types +// as interface{}) to a Go bool. Handles int64(0/1) from SQLite. +func toBool(v interface{}) bool { + switch val := v.(type) { + case bool: + return val + case int64: + return val != 0 + case int: + return val != 0 + default: + return false + } +} + // parseMarkersJSONFromVtxo parses a JSON array string into a slice of strings for vtxo repo func parseMarkersJSONFromVtxo(markersJSON string) []string { if markersJSON == "" { From 2ff4b78fd8fa9d5713e7ec27ddd567798c92ebb8 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Fri, 17 Apr 2026 11:33:12 -0400 Subject: [PATCH 47/54] fix benchmark metrics, SQL safety, and auth/marker edge cases --- internal/core/application/indexer.go | 1 + internal/core/application/indexer_bench_test.go | 4 ++-- internal/core/application/service.go | 8 ++++++++ internal/infrastructure/db/postgres/marker_repo.go | 3 +++ .../infrastructure/db/postgres/sqlc/queries/query.sql.go | 4 ++-- internal/infrastructure/db/postgres/sqlc/query.sql | 4 ++-- internal/infrastructure/db/service.go | 2 +- .../infrastructure/db/sqlite/sqlc/queries/query.sql.go | 4 ++-- internal/infrastructure/db/sqlite/sqlc/query.sql | 5 +++-- 9 files changed, 24 insertions(+), 11 deletions(-) diff --git a/internal/core/application/indexer.go b/internal/core/application/indexer.go index 9593bda63..775666aa6 100644 --- a/internal/core/application/indexer.go +++ b/internal/core/application/indexer.go @@ -724,6 +724,7 @@ func (i *indexerService) walkVtxoChain( Spends: []string{ptx.UnsignedTx.TxIn[0].PreviousOutPoint.String()}, }) + allOutpoints = append(allOutpoints, Outpoint{Txid: txid, VOut: 0}) chainTx.Spends = append(chainTx.Spends, txid) // populate newNextVtxos with checkpoints inputs diff --git a/internal/core/application/indexer_bench_test.go b/internal/core/application/indexer_bench_test.go index acb9ac8b3..89c07be04 100644 --- a/internal/core/application/indexer_bench_test.go +++ b/internal/core/application/indexer_bench_test.go @@ -617,9 +617,9 @@ func BenchmarkOffchainTxBulkVsSingle(b *testing.B) { vtxoRepo: vtxoRepo, offchainRepo: repo, }} b.ReportAllocs() + repo.reset() b.ResetTimer() for i := 0; i < b.N; i++ { - repo.reset() _, err := svc.GetVtxoChain(ctx, "", start, nil, "") if err != nil { b.Fatal(err) @@ -636,9 +636,9 @@ func BenchmarkOffchainTxBulkVsSingle(b *testing.B) { vtxoRepo: vtxoRepo, offchainRepo: repo, }} b.ReportAllocs() + repo.reset() b.ResetTimer() for i := 0; i < b.N; i++ { - repo.reset() _, err := svc.GetVtxoChain(ctx, "", start, nil, "") if err != nil { b.Fatal(err) diff --git a/internal/core/application/service.go b/internal/core/application/service.go index b18597f58..f285dc655 100644 --- a/internal/core/application/service.go +++ b/internal/core/application/service.go @@ -369,6 +369,14 @@ func (s *service) registerEventHandlers() { return } + if len(spentVtxos) != len(spentVtxoKeys) { + log.Warnf( + "incomplete parent read: got %d of %d spent vtxos for tx %s", + len(spentVtxos), len(spentVtxoKeys), txid, + ) + return + } + // Calculate depth for new vtxos: max(parent depths) + 1 var maxDepth uint32 for _, v := range spentVtxos { diff --git a/internal/infrastructure/db/postgres/marker_repo.go b/internal/infrastructure/db/postgres/marker_repo.go index 0ac7a01f2..7757023fc 100644 --- a/internal/infrastructure/db/postgres/marker_repo.go +++ b/internal/infrastructure/db/postgres/marker_repo.go @@ -256,6 +256,9 @@ func (m *markerRepository) UpdateVtxoMarkers( outpoint domain.Outpoint, markerIDs []string, ) error { + if markerIDs == nil { + markerIDs = []string{} + } markersJSON, err := json.Marshal(markerIDs) if err != nil { return fmt.Errorf("failed to marshal markers: %w", err) diff --git a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go index 309ed699a..5a9ba2bb2 100644 --- a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go @@ -121,7 +121,7 @@ func (q *Queries) ClearScheduledSession(ctx context.Context) error { } const countUnsweptVtxosByMarkerId = `-- name: CountUnsweptVtxosByMarkerId :one -SELECT COUNT(*) FROM vtxo_vw WHERE markers @> jsonb_build_array($1::TEXT) AND swept = false +SELECT COUNT(DISTINCT (txid, vout)) FROM vtxo_vw WHERE markers @> jsonb_build_array($1::TEXT) AND swept = false ` // Count VTXOs whose markers JSONB array contains the given marker_id and are not swept @@ -136,7 +136,7 @@ const getDescendantMarkerIds = `-- name: GetDescendantMarkerIds :many WITH RECURSIVE descendant_markers(id) AS ( -- Base case: the marker being swept SELECT marker.id FROM marker WHERE marker.id = $1 - UNION ALL + UNION -- Recursive case: find markers whose parent_markers jsonb array contains any descendant SELECT m.id FROM marker m INNER JOIN descendant_markers dm ON ( diff --git a/internal/infrastructure/db/postgres/sqlc/query.sql b/internal/infrastructure/db/postgres/sqlc/query.sql index 6d3b4e33b..121f00eca 100644 --- a/internal/infrastructure/db/postgres/sqlc/query.sql +++ b/internal/infrastructure/db/postgres/sqlc/query.sql @@ -477,7 +477,7 @@ SELECT EXISTS(SELECT 1 FROM swept_marker WHERE marker_id = @marker_id) AS is_swe WITH RECURSIVE descendant_markers(id) AS ( -- Base case: the marker being swept SELECT marker.id FROM marker WHERE marker.id = @root_marker_id - UNION ALL + UNION -- Recursive case: find markers whose parent_markers jsonb array contains any descendant SELECT m.id FROM marker m INNER JOIN descendant_markers dm ON ( @@ -496,7 +496,7 @@ SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE markers @> jsonb_build_array(@mark -- name: CountUnsweptVtxosByMarkerId :one -- Count VTXOs whose markers JSONB array contains the given marker_id and are not swept -SELECT COUNT(*) FROM vtxo_vw WHERE markers @> jsonb_build_array(@marker_id::TEXT) AND swept = false; +SELECT COUNT(DISTINCT (txid, vout)) FROM vtxo_vw WHERE markers @> jsonb_build_array(@marker_id::TEXT) AND swept = false; -- Chain traversal queries for GetVtxoChain optimization diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index 9c15c053b..11f05b499 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -669,8 +669,8 @@ func (s *service) updateProjectionsAfterOffchainTxEvents(events []domain.Event) log.WithError(err).Warnf("failed to create dust marker %s", dustMarkerID) } else { createdDustMarkerIDs = append(createdDustMarkerIDs, dustMarkerID) + vtxoMarkerIDs = append(append([]string{}, markerIDs...), dustMarkerID) } - vtxoMarkerIDs = append(append([]string{}, markerIDs...), dustMarkerID) } newVtxos = append(newVtxos, domain.Vtxo{ diff --git a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go index b63bd0aa1..6fcffb229 100644 --- a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go @@ -85,7 +85,7 @@ func (q *Queries) ClearScheduledSession(ctx context.Context) error { } const countUnsweptVtxosByMarkerId = `-- name: CountUnsweptVtxosByMarkerId :one -SELECT COUNT(*) FROM vtxo_vw WHERE markers LIKE '%"' || ?1 || '"%' AND swept = false +SELECT COUNT(DISTINCT txid || ':' || CAST(vout AS TEXT)) FROM vtxo_vw WHERE markers LIKE '%"' || ?1 || '"%' AND swept = false ` // Count VTXOs whose markers JSON array contains the given marker_id and are not swept. @@ -101,7 +101,7 @@ const getDescendantMarkerIds = `-- name: GetDescendantMarkerIds :many WITH RECURSIVE descendant_markers(id) AS ( -- Base case: the marker being swept SELECT marker.id FROM marker WHERE marker.id = ?1 - UNION ALL + UNION -- Recursive case: find markers whose parent_markers JSON array contains any descendant SELECT m.id FROM marker m INNER JOIN descendant_markers dm ON EXISTS ( diff --git a/internal/infrastructure/db/sqlite/sqlc/query.sql b/internal/infrastructure/db/sqlite/sqlc/query.sql index 3e926ea32..45d808f8a 100644 --- a/internal/infrastructure/db/sqlite/sqlc/query.sql +++ b/internal/infrastructure/db/sqlite/sqlc/query.sql @@ -479,7 +479,7 @@ SELECT EXISTS(SELECT 1 FROM swept_marker WHERE marker_id = @marker_id) AS is_swe WITH RECURSIVE descendant_markers(id) AS ( -- Base case: the marker being swept SELECT marker.id FROM marker WHERE marker.id = @root_marker_id - UNION ALL + UNION -- Recursive case: find markers whose parent_markers JSON array contains any descendant SELECT m.id FROM marker m INNER JOIN descendant_markers dm ON EXISTS ( @@ -501,7 +501,8 @@ SELECT sqlc.embed(vtxo_vw) FROM vtxo_vw WHERE markers LIKE '%"' || @marker_id || -- name: CountUnsweptVtxosByMarkerId :one -- Count VTXOs whose markers JSON array contains the given marker_id and are not swept. -- Uses LIKE because sqlc cannot parse json_each with view columns. -SELECT COUNT(*) FROM vtxo_vw WHERE markers LIKE '%"' || @marker_id || '"%' AND swept = false; +-- Uses DISTINCT to avoid double-counting VTXOs with multiple asset projections. +SELECT COUNT(DISTINCT txid || ':' || CAST(vout AS TEXT)) FROM vtxo_vw WHERE markers LIKE '%"' || @marker_id || '"%' AND swept = false; -- Chain traversal queries for GetVtxoChain optimization From 30cc8c65999f3c68586b1decf91cc37bfbad05c8 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Fri, 17 Apr 2026 13:19:25 -0400 Subject: [PATCH 48/54] fix GetAllChildrenVtxos sibling over-reach and swept_vtxo down migration, tests --- internal/core/application/admin.go | 2 +- internal/core/application/indexer.go | 8 +- internal/core/application/indexer_test.go | 74 +++++++++++- internal/core/application/service.go | 11 +- internal/core/application/sweeper.go | 4 +- internal/core/application/sweeper_test.go | 14 +-- internal/core/domain/vtxo_repo.go | 2 +- .../infrastructure/db/badger/vtxo_repo.go | 25 +++- .../infrastructure/db/postgres/marker_repo.go | 52 ++++---- .../20260416120000_add_swept_vtxo.down.sql | 17 +++ .../db/postgres/sqlc/queries/query.sql.go | 24 +++- .../infrastructure/db/postgres/sqlc/query.sql | 15 ++- .../infrastructure/db/postgres/vtxo_repo.go | 16 ++- internal/infrastructure/db/service_test.go | 107 ++++++++++++++++- .../infrastructure/db/sqlite/marker_repo.go | 60 ++++++---- .../db/sqlite/marker_repo_test.go | 83 +++++++++++++ .../20260416120000_add_swept_vtxo.down.sql | 22 ++++ .../db/sqlite/sqlc/queries/query.sql.go | 23 +++- .../infrastructure/db/sqlite/sqlc/query.sql | 13 +- .../infrastructure/db/sqlite/vtxo_repo.go | 16 ++- .../infrastructure/db/swept_vtxo_down_test.go | 112 ++++++++++++++++++ 21 files changed, 607 insertions(+), 93 deletions(-) create mode 100644 internal/infrastructure/db/sqlite/marker_repo_test.go create mode 100644 internal/infrastructure/db/swept_vtxo_down_test.go diff --git a/internal/core/application/admin.go b/internal/core/application/admin.go index 604b12516..5ef0766e6 100644 --- a/internal/core/application/admin.go +++ b/internal/core/application/admin.go @@ -729,7 +729,7 @@ func (a *adminService) saveBatchSweptEvents( } else { seen := make(map[string]struct{}) for _, leafVtxo := range leafVtxos { - children, err := vtxoRepo.GetAllChildrenVtxos(ctx, leafVtxo.Txid) + children, err := vtxoRepo.GetAllChildrenVtxos(ctx, leafVtxo) if err != nil { log.WithError(err).Error("error while getting children vtxos") continue diff --git a/internal/core/application/indexer.go b/internal/core/application/indexer.go index 775666aa6..83c905015 100644 --- a/internal/core/application/indexer.go +++ b/internal/core/application/indexer.go @@ -609,13 +609,19 @@ func (i *indexerService) walkVtxoChain( loadedMarkers := make(map[string]bool) // Eagerly preload VTXOs and offchain txs by walking the marker DAG upward. + // Failures in the marker-driven preload are treated as optimization misses: + // the per-hop walk loop below falls back to Vtxos().GetVtxos + ensureVtxosCached, + // so we log marker-repo errors here and continue instead of aborting. if i.repoManager.Markers() != nil { startVtxos, err := i.repoManager.Vtxos().GetVtxos(ctx, nextVtxos) if err != nil { return nil, nil, "", err } if err := i.preloadByMarkers(ctx, startVtxos, vtxoCache, offchainTxCache); err != nil { - return nil, nil, "", err + log.WithError(err).Warnf( + "marker-driven preload failed for frontier of %d outpoints; "+ + "falling back to per-hop walk", len(nextVtxos), + ) } } diff --git a/internal/core/application/indexer_test.go b/internal/core/application/indexer_test.go index 5407009c8..13c98fe4a 100644 --- a/internal/core/application/indexer_test.go +++ b/internal/core/application/indexer_test.go @@ -119,7 +119,7 @@ func (m *mockVtxoRepoForIndexer) GetSweepableVtxosByCommitmentTxid( func (m *mockVtxoRepoForIndexer) GetAllChildrenVtxos( ctx context.Context, - txid string, + outpoint domain.Outpoint, ) ([]domain.Outpoint, error) { return nil, nil } @@ -1300,6 +1300,78 @@ func TestGetVtxoChain_PreloadReducesDBCalls(t *testing.T) { markerRepo.AssertNumberOfCalls(t, "GetMarkersByIds", markersCount) } +// TestGetVtxoChain_PreloadMarkerErrorFallback verifies that when the marker +// repo errors during preloadByMarkers, GetVtxoChain still returns the correct +// chain via the per-hop GetVtxos + ensureVtxosCached fallback, rather than +// aborting the request entirely. +func TestGetVtxoChain_PreloadMarkerErrorFallback(t *testing.T) { + vtxoRepo, markerRepo, offchainTxRepo, indexer := newChainTestIndexerWithOffchain() + ctx := context.Background() + + txidA := strings.Repeat("a", 64) + txidB := strings.Repeat("b", 64) + txidC := strings.Repeat("c", 64) + + vtxoA := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: txidA, VOut: 0}, + Preconfirmed: true, + ExpiresAt: 1000, + MarkerIDs: []string{"marker-A"}, + } + vtxoB := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: txidB, VOut: 0}, + Preconfirmed: true, + ExpiresAt: 2000, + MarkerIDs: []string{"marker-B"}, + } + vtxoC := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: txidC, VOut: 0}, + Preconfirmed: true, + ExpiresAt: 3000, + MarkerIDs: []string{"marker-C"}, + } + + // Initial preload fetch for the frontier. + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: txidA, VOut: 0}}). + Return([]domain.Vtxo{vtxoA}, nil) + + // Preload's first marker lookup fails — this is the fault we're injecting. + // Per-hop fallback should take over from here. + markerRepo.On("GetVtxoChainByMarkers", ctx, matchIDs("marker-A")). + Return(nil, fmt.Errorf("transient marker repo failure")) + + // ensureVtxosCached fetches B and C on cache miss. The fix lets these run + // even though preload aborted partway through. + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: txidB, VOut: 0}}). + Return([]domain.Vtxo{vtxoB}, nil) + vtxoRepo.On("GetVtxos", ctx, []domain.Outpoint{{Txid: txidC, VOut: 0}}). + Return([]domain.Vtxo{vtxoC}, nil) + + // Marker window loading during the walk — can either succeed empty or + // error; ensureVtxosCached logs and continues either way. + markerRepo.On("GetVtxosByMarker", ctx, mock.Anything). + Return([]domain.Vtxo{}, nil).Maybe() + + // Offchain tx for the preconfirmed chain: A → B → C. + cpA := makeCheckpointPSBT(t, txidB, 0) + cpB := makeCheckpointPSBT(t, txidC, 0) + offchainTxRepo.On("GetOffchainTx", ctx, txidA). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{"cp-a": cpA}}, nil) + offchainTxRepo.On("GetOffchainTx", ctx, txidB). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{"cp-b": cpB}}, nil) + offchainTxRepo.On("GetOffchainTx", ctx, txidC). + Return(&domain.OffchainTx{CheckpointTxs: map[string]string{}}, nil) + + resp, err := indexer.GetVtxoChain(ctx, "", Outpoint{Txid: txidA, VOut: 0}, nil, "") + require.NoError(t, err, "marker preload failure must not abort GetVtxoChain") + require.Equal(t, 5, len(resp.Chain)) // A(ark+cp) + B(ark+cp) + C(ark) + + // The preload GetVtxoChainByMarkers was attempted (and failed). + markerRepo.AssertCalled(t, "GetVtxoChainByMarkers", ctx, matchIDs("marker-A")) + // And the fallback did per-hop GetVtxos for B and C (plus the initial A). + vtxoRepo.AssertNumberOfCalls(t, "GetVtxos", 3) +} + // TestGetVtxoChain_Fanout verifies that a VTXO with 2 checkpoints pointing // to different parents correctly traverses both branches. // diff --git a/internal/core/application/service.go b/internal/core/application/service.go index f285dc655..98fc9eeb0 100644 --- a/internal/core/application/service.go +++ b/internal/core/application/service.go @@ -8,6 +8,7 @@ import ( "math" "runtime" "slices" + "sort" "strings" "sync" "sync/atomic" @@ -370,8 +371,13 @@ func (s *service) registerEventHandlers() { } if len(spentVtxos) != len(spentVtxoKeys) { - log.Warnf( - "incomplete parent read: got %d of %d spent vtxos for tx %s", + // Partial parent read: this means the offchain tx's finalization + // event references spent vtxos that we can no longer resolve from + // the DB. Drop propagation rather than emit a half-populated event; + // log at Error level so this inconsistency is surfaced for investigation. + log.Errorf( + "incomplete parent read: got %d of %d spent vtxos for tx %s; "+ + "dropping TransactionEvent propagation", len(spentVtxos), len(spentVtxoKeys), txid, ) return @@ -1118,6 +1124,7 @@ func (s *service) SubmitOffchainTx( for id := range parentMarkerSet { parentMarkerIDs = append(parentMarkerIDs, id) } + sort.Strings(parentMarkerIDs) change, err := offchainTx.Accept( fullySignedArkTx, signedCheckpointTxsMap, diff --git a/internal/core/application/sweeper.go b/internal/core/application/sweeper.go index 62c1dff28..b02eb8848 100644 --- a/internal/core/application/sweeper.go +++ b/internal/core/application/sweeper.go @@ -696,7 +696,7 @@ func (s *sweeper) createBatchSweepTask(commitmentTxid, vtxoTreeRootTxid string) // get all vtxos related to the leaf swept seen := make(map[string]struct{}) for _, leafVtxo := range leafVtxoKeys { - children, childErr := vtxoRepo.GetAllChildrenVtxos(ctx, leafVtxo.Txid) + children, childErr := vtxoRepo.GetAllChildrenVtxos(ctx, leafVtxo) if childErr != nil { log.WithError(childErr).Error("error while getting children vtxos") continue @@ -764,7 +764,7 @@ func (s *sweeper) createCheckpointSweepTask( // because markers can be shared across independent subtrees when // offchain txs consolidate inputs from different lineages. Sweeping // by marker would over-reach and incorrectly mark unrelated VTXOs. - childrenVtxos, err := s.repoManager.Vtxos().GetAllChildrenVtxos(ctx, vtxo.Txid) + childrenVtxos, err := s.repoManager.Vtxos().GetAllChildrenVtxos(ctx, vtxo) if err != nil { return err } diff --git a/internal/core/application/sweeper_test.go b/internal/core/application/sweeper_test.go index 07fb7241c..909aa693b 100644 --- a/internal/core/application/sweeper_test.go +++ b/internal/core/application/sweeper_test.go @@ -163,9 +163,9 @@ type mockVtxoRepository struct { func (m *mockVtxoRepository) GetAllChildrenVtxos( ctx context.Context, - txid string, + outpoint domain.Outpoint, ) ([]domain.Outpoint, error) { - args := m.Called(ctx, txid) + args := m.Called(ctx, outpoint) if args.Get(0) == nil { return nil, args.Error(1) } @@ -525,7 +525,7 @@ func TestCreateCheckpointSweepTask_SweepsVtxoOutpoints(t *testing.T) { wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). Return("sweeptxid123", nil) - vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). + vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint). Return(childOutpoints, nil) // SweepVtxoOutpoints should be called with the exact child outpoints @@ -562,7 +562,7 @@ func TestCreateCheckpointSweepTask_SweptAtTimestamp(t *testing.T) { wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). Return("sweeptxid_ts", nil) - vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). + vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint). Return(childOutpoints, nil) beforeExec := time.Now().UnixMilli() @@ -599,7 +599,7 @@ func TestCreateCheckpointSweepTask_SweepVtxoOutpointsError(t *testing.T) { wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). Return("sweeptxid_err", nil) - vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). + vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint). Return(childOutpoints, nil) dbError := fmt.Errorf("database connection failed") @@ -629,7 +629,7 @@ func TestCreateCheckpointSweepTask_GetAllChildrenVtxosError(t *testing.T) { wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). Return("sweeptxid_children_err", nil) - vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). + vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint). Return(nil, fmt.Errorf("failed to query children vtxos")) task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) @@ -706,7 +706,7 @@ func TestCreateCheckpointSweepTask_NoChildrenVtxos(t *testing.T) { wallet.On("BroadcastTransaction", mock.Anything, []string{"sweeptx_hex"}). Return("sweeptxid_nc", nil) - vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint.Txid). + vtxoRepo.On("GetAllChildrenVtxos", mock.Anything, vtxoOutpoint). Return([]domain.Outpoint{}, nil) task := s.createCheckpointSweepTask(toSweep, vtxoOutpoint) diff --git a/internal/core/domain/vtxo_repo.go b/internal/core/domain/vtxo_repo.go index 61beb8cca..e25ba627e 100644 --- a/internal/core/domain/vtxo_repo.go +++ b/internal/core/domain/vtxo_repo.go @@ -23,7 +23,7 @@ type VtxoRepository interface { GetSweepableVtxosByCommitmentTxid( ctx context.Context, commitmentTxid string, ) ([]Outpoint, error) - GetAllChildrenVtxos(ctx context.Context, txid string) ([]Outpoint, error) + GetAllChildrenVtxos(ctx context.Context, outpoint Outpoint) ([]Outpoint, error) GetVtxoPubKeysByCommitmentTxid( ctx context.Context, commitmentTxid string, withMinimumAmount uint64, ) ( diff --git a/internal/infrastructure/db/badger/vtxo_repo.go b/internal/infrastructure/db/badger/vtxo_repo.go index c0c5f6d44..88278a66b 100644 --- a/internal/infrastructure/db/badger/vtxo_repo.go +++ b/internal/infrastructure/db/badger/vtxo_repo.go @@ -637,13 +637,32 @@ func (r *VtxoRepository) GetSweepableVtxosByCommitmentTxid( func (r *VtxoRepository) GetAllChildrenVtxos( ctx context.Context, - txid string, + outpoint domain.Outpoint, ) ([]domain.Outpoint, error) { + // Seed with the specific outpoint, not all vouts of the txid, so that + // sibling outputs (which belong to independent lineages) are not included. + seedQuery := badgerhold.Where("Txid").Eq(outpoint.Txid). + And("VOut").Eq(outpoint.VOut) + seedVtxos, err := r.findVtxos(ctx, seedQuery) + if err != nil { + return nil, fmt.Errorf("failed to find seed vtxo %s: %w", outpoint, err) + } + visited := make(map[string]bool) visitedTxids := make(map[string]bool) var outpoints []domain.Outpoint - - queue := []string{txid} + queue := make([]string, 0, len(seedVtxos)) + + for _, vtxo := range seedVtxos { + outpointKey := vtxo.Outpoint.String() + if !visited[outpointKey] { + visited[outpointKey] = true + outpoints = append(outpoints, vtxo.Outpoint) + if vtxo.ArkTxid != "" { + queue = append(queue, vtxo.ArkTxid) + } + } + } for len(queue) > 0 { currentTxid := queue[0] diff --git a/internal/infrastructure/db/postgres/marker_repo.go b/internal/infrastructure/db/postgres/marker_repo.go index 7757023fc..9be837af5 100644 --- a/internal/infrastructure/db/postgres/marker_repo.go +++ b/internal/infrastructure/db/postgres/marker_repo.go @@ -9,6 +9,7 @@ import ( "github.com/arkade-os/arkd/internal/core/domain" "github.com/arkade-os/arkd/internal/infrastructure/db/postgres/sqlc/queries" + log "github.com/sirupsen/logrus" "github.com/sqlc-dev/pqtype" ) @@ -287,29 +288,35 @@ func (m *markerRepository) GetVtxosByMarker( } func (m *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID string) (int64, error) { - // First check if the marker exists (foreign key constraint on swept_marker) - marker, err := m.GetMarker(ctx, markerID) - if err != nil { - return 0, fmt.Errorf("failed to check marker existence: %w", err) - } - if marker == nil { - return 0, nil // Marker doesn't exist, nothing to sweep - } + var count int64 + txBody := func(qtx *queries.Queries) error { + // First check if the marker exists (foreign key constraint on swept_marker) + if _, err := qtx.SelectMarker(ctx, markerID); err != nil { + if err == sql.ErrNoRows { + return nil // Marker doesn't exist, nothing to sweep + } + return fmt.Errorf("failed to check marker existence: %w", err) + } - // Count unswept VTXOs with this marker before inserting to swept_marker - count, err := m.querier.CountUnsweptVtxosByMarkerId(ctx, markerID) - if err != nil { - return 0, fmt.Errorf("failed to count unswept vtxos: %w", err) - } + // Count unswept VTXOs with this marker before inserting to swept_marker + c, err := qtx.CountUnsweptVtxosByMarkerId(ctx, markerID) + if err != nil { + return fmt.Errorf("failed to count unswept vtxos: %w", err) + } - // Insert the marker into swept_marker (sweep state is computed via view) - if err := m.querier.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ - MarkerID: markerID, - SweptAt: time.Now().UnixMilli(), - }); err != nil { - return 0, fmt.Errorf("failed to insert swept marker: %w", err) + // Insert the marker into swept_marker (sweep state is computed via view) + if err := qtx.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ + MarkerID: markerID, + SweptAt: time.Now().UnixMilli(), + }); err != nil { + return fmt.Errorf("failed to insert swept marker: %w", err) + } + count = c + return nil + } + if err := execTx(ctx, m.db, txBody); err != nil { + return 0, err } - return count, nil } @@ -466,13 +473,16 @@ func rowToVtxoFromMarkerQuery(row queries.SelectVtxosByMarkerIdRow) domain.Vtxo } } -// parseMarkersJSONB parses a JSONB array into a slice of strings +// parseMarkersJSONB parses a JSONB array into a slice of strings. +// Logs and returns nil if the JSON is malformed so that corrupt markers are +// surfaced instead of silently treated as empty. func parseMarkersJSONB(markers json.RawMessage) []string { if len(markers) == 0 { return nil } var markerIDs []string if err := json.Unmarshal(markers, &markerIDs); err != nil { + log.WithError(err).Warnf("failed to parse markers JSONB: %q", string(markers)) return nil } return markerIDs diff --git a/internal/infrastructure/db/postgres/migration/20260416120000_add_swept_vtxo.down.sql b/internal/infrastructure/db/postgres/migration/20260416120000_add_swept_vtxo.down.sql index 52e01148e..a23aa8ee7 100644 --- a/internal/infrastructure/db/postgres/migration/20260416120000_add_swept_vtxo.down.sql +++ b/internal/infrastructure/db/postgres/migration/20260416120000_add_swept_vtxo.down.sql @@ -1,3 +1,20 @@ +-- Guard against silently resurrecting swept VTXOs. +-- +-- swept_vtxo holds per-outpoint sweep state for the checkpoint-sweep path. +-- Dropping the table would make vtxo_vw.swept flip back to false for every +-- outpoint tracked only here (marker-based sweeps still survive via +-- swept_marker). When the table has data, fail loudly rather than silently +-- discard it. When the table is empty, the rollback is safe — drop the +-- table and restore the pre-swept_vtxo view shape. +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM swept_vtxo) THEN + RAISE EXCEPTION 'irreversible migration: swept_vtxo contains % entries; rolling back would resurrect swept VTXOs. Truncate swept_vtxo manually if you accept the data loss, then re-run.', + (SELECT count(*) FROM swept_vtxo); + END IF; +END +$$; + DROP TABLE IF EXISTS swept_vtxo; -- Restore views without swept_vtxo check diff --git a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go index 5a9ba2bb2..f15d236dd 100644 --- a/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/postgres/sqlc/queries/query.sql.go @@ -147,7 +147,10 @@ SELECT descendant_markers.id AS marker_id FROM descendant_markers WHERE descendant_markers.id NOT IN (SELECT sm.marker_id FROM swept_marker sm) ` -// Recursively get a marker and all its descendants (markers whose parent_markers contain it) +// Recursively get a marker and all its descendants (markers whose parent_markers contain it). +// Uses UNION (set semantics, not UNION ALL) so rows already produced are filtered, +// which makes this cycle-safe. Do not convert to UNION ALL: cycles in parent_markers +// would cause the recursion to run unbounded. func (q *Queries) GetDescendantMarkerIds(ctx context.Context, rootMarkerID string) ([]string, error) { rows, err := q.db.QueryContext(ctx, getDescendantMarkerIds, rootMarkerID) if err != nil { @@ -2185,12 +2188,12 @@ func (q *Queries) SelectVtxosByMarkerId(ctx context.Context, markerID string) ([ const selectVtxosOutpointsByArkTxidRecursive = `-- name: SelectVtxosOutpointsByArkTxidRecursive :many WITH RECURSIVE descendants_chain AS ( - -- seed + -- seed: only the specific outpoint, not all vouts of the txid SELECT v.txid, v.vout, v.preconfirmed, v.ark_txid, v.spent_by, 0 AS depth, ARRAY[(v.txid||':'||v.vout)]::text[] AS visited FROM vtxo v - WHERE v.txid = $1 + WHERE v.txid = $1 AND v.vout = $2 UNION ALL @@ -2215,14 +2218,23 @@ FROM nodes ORDER BY depth, txid, vout ` +type SelectVtxosOutpointsByArkTxidRecursiveParams struct { + Txid string + Vout int32 +} + type SelectVtxosOutpointsByArkTxidRecursiveRow struct { Txid string Vout int32 } +// Returns the seed outpoint (txid, vout) and all VTXOs descending from it +// via ark_txid links. Scoped to a single outpoint (not the whole txid) so that +// sibling outputs of the seed tx, which belong to independent lineages, are +// not included. // keep one row per node at its MIN depth (layers) -func (q *Queries) SelectVtxosOutpointsByArkTxidRecursive(ctx context.Context, txid string) ([]SelectVtxosOutpointsByArkTxidRecursiveRow, error) { - rows, err := q.db.QueryContext(ctx, selectVtxosOutpointsByArkTxidRecursive, txid) +func (q *Queries) SelectVtxosOutpointsByArkTxidRecursive(ctx context.Context, arg SelectVtxosOutpointsByArkTxidRecursiveParams) ([]SelectVtxosOutpointsByArkTxidRecursiveRow, error) { + rows, err := q.db.QueryContext(ctx, selectVtxosOutpointsByArkTxidRecursive, arg.Txid, arg.Vout) if err != nil { return nil, err } @@ -2724,7 +2736,7 @@ INSERT INTO vtxo ( ) VALUES ( $1, $2, $3, $4, $5, $6, $7, - $8, $9, $10, $11, $12, $13, (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, $14, $15 + $8, $9, $10, $11, $12, $13, (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, $14, $15::jsonb ) ON CONFLICT(txid, vout) DO UPDATE SET pubkey = EXCLUDED.pubkey, amount = EXCLUDED.amount, diff --git a/internal/infrastructure/db/postgres/sqlc/query.sql b/internal/infrastructure/db/postgres/sqlc/query.sql index 121f00eca..fc34639f4 100644 --- a/internal/infrastructure/db/postgres/sqlc/query.sql +++ b/internal/infrastructure/db/postgres/sqlc/query.sql @@ -52,7 +52,7 @@ INSERT INTO vtxo ( ) VALUES ( @txid, @vout, @pubkey, @amount, @commitment_txid, @settled_by, @ark_txid, - @spent_by, @spent, @unrolled, @preconfirmed, @expires_at, @created_at, (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, @depth, @markers + @spent_by, @spent, @unrolled, @preconfirmed, @expires_at, @created_at, (EXTRACT(EPOCH FROM NOW()) * 1000)::BIGINT, @depth, @markers::jsonb ) ON CONFLICT(txid, vout) DO UPDATE SET pubkey = EXCLUDED.pubkey, amount = EXCLUDED.amount, @@ -298,13 +298,17 @@ WHERE v.swept = false OR (',' || COALESCE(v.commitments::text, '') || ',') LIKE '%,' || @commitment_txid || ',%'); -- name: SelectVtxosOutpointsByArkTxidRecursive :many +-- Returns the seed outpoint (txid, vout) and all VTXOs descending from it +-- via ark_txid links. Scoped to a single outpoint (not the whole txid) so that +-- sibling outputs of the seed tx, which belong to independent lineages, are +-- not included. WITH RECURSIVE descendants_chain AS ( - -- seed + -- seed: only the specific outpoint, not all vouts of the txid SELECT v.txid, v.vout, v.preconfirmed, v.ark_txid, v.spent_by, 0 AS depth, ARRAY[(v.txid||':'||v.vout)]::text[] AS visited FROM vtxo v - WHERE v.txid = @txid + WHERE v.txid = @txid AND v.vout = @vout UNION ALL @@ -473,7 +477,10 @@ SELECT * FROM swept_marker WHERE marker_id = ANY(@marker_ids::text[]); SELECT EXISTS(SELECT 1 FROM swept_marker WHERE marker_id = @marker_id) AS is_swept; -- name: GetDescendantMarkerIds :many --- Recursively get a marker and all its descendants (markers whose parent_markers contain it) +-- Recursively get a marker and all its descendants (markers whose parent_markers contain it). +-- Uses UNION (set semantics, not UNION ALL) so rows already produced are filtered, +-- which makes this cycle-safe. Do not convert to UNION ALL: cycles in parent_markers +-- would cause the recursion to run unbounded. WITH RECURSIVE descendant_markers(id) AS ( -- Base case: the marker being swept SELECT marker.id FROM marker WHERE marker.id = @root_marker_id diff --git a/internal/infrastructure/db/postgres/vtxo_repo.go b/internal/infrastructure/db/postgres/vtxo_repo.go index 24b78056e..529eb23d9 100644 --- a/internal/infrastructure/db/postgres/vtxo_repo.go +++ b/internal/infrastructure/db/postgres/vtxo_repo.go @@ -11,6 +11,7 @@ import ( "github.com/arkade-os/arkd/internal/core/domain" "github.com/arkade-os/arkd/internal/infrastructure/db/postgres/sqlc/queries" + log "github.com/sirupsen/logrus" ) type vtxoRepository struct { @@ -405,9 +406,15 @@ func (v *vtxoRepository) GetSweepableVtxosByCommitmentTxid( } func (v *vtxoRepository) GetAllChildrenVtxos( - ctx context.Context, txid string, + ctx context.Context, outpoint domain.Outpoint, ) ([]domain.Outpoint, error) { - res, err := v.querier.SelectVtxosOutpointsByArkTxidRecursive(ctx, txid) + res, err := v.querier.SelectVtxosOutpointsByArkTxidRecursive( + ctx, + queries.SelectVtxosOutpointsByArkTxidRecursiveParams{ + Txid: outpoint.Txid, + Vout: int32(outpoint.VOut), + }, + ) if err != nil { return nil, err } @@ -546,13 +553,16 @@ func rowToAsset(row queries.VtxoVw) domain.AssetDenomination { } } -// parseMarkersJSONBFromVtxo parses a JSONB array into a slice of strings for vtxo repo +// parseMarkersJSONBFromVtxo parses a JSONB array into a slice of strings for vtxo repo. +// Logs and returns nil if the JSON is malformed so that corrupt markers are +// surfaced instead of silently treated as empty. func parseMarkersJSONBFromVtxo(markers json.RawMessage) []string { if len(markers) == 0 { return nil } var markerIDs []string if err := json.Unmarshal(markers, &markerIDs); err != nil { + log.WithError(err).Warnf("failed to parse markers JSONB: %q", string(markers)) return nil } return markerIDs diff --git a/internal/infrastructure/db/service_test.go b/internal/infrastructure/db/service_test.go index 51a0b4678..00a216998 100644 --- a/internal/infrastructure/db/service_test.go +++ b/internal/infrastructure/db/service_test.go @@ -211,6 +211,7 @@ func TestService(t *testing.T) { testSweepableUnrolledExcludesMarkerSwept(t, svc) testSweepVtxoOutpointsNoOverreach(t, svc) testSweepVtxoOutpointsEdgeCases(t, svc) + testGetAllChildrenVtxosSiblingIsolation(t, svc) testConvergentMultiParentMarkerDAG(t, svc) testSweepMarkerWithDescendantsDeepChain(t, svc) testScheduledSessionRepository(t, svc) @@ -933,7 +934,7 @@ func testVtxoRepository(t *testing.T, svc ports.RepoManager) { require.Empty(t, children) // Test recursive query starting from vtxo1 - children, err = svc.Vtxos().GetAllChildrenVtxos(ctx, vtxo1.Txid) + children, err = svc.Vtxos().GetAllChildrenVtxos(ctx, vtxo1.Outpoint) require.NoError(t, err) require.Len(t, children, 4) // Should return all 4 vtxos in the chain @@ -944,17 +945,19 @@ func testVtxoRepository(t *testing.T, svc ports.RepoManager) { require.Equal(t, expectedOutpoints, children) // Test starting from middle of chain (vtxo2) - children, err = svc.Vtxos().GetAllChildrenVtxos(ctx, vtxo2.Txid) + children, err = svc.Vtxos().GetAllChildrenVtxos(ctx, vtxo2.Outpoint) require.NoError(t, err) require.Len(t, children, 3) // Should return vtxo2, vtxo3, vtxo4 // Test starting from end of chain (vtxo4) - children, err = svc.Vtxos().GetAllChildrenVtxos(ctx, vtxo4.Txid) + children, err = svc.Vtxos().GetAllChildrenVtxos(ctx, vtxo4.Outpoint) require.NoError(t, err) require.Len(t, children, 1) // Should return only vtxo4 - // Test with non-existent txid - children, err = svc.Vtxos().GetAllChildrenVtxos(ctx, randomString(32)) + // Test with non-existent outpoint + children, err = svc.Vtxos().GetAllChildrenVtxos( + ctx, domain.Outpoint{Txid: randomString(32), VOut: 0}, + ) require.NoError(t, err) require.Empty(t, children) @@ -5047,6 +5050,100 @@ func testSweepVtxoOutpointsEdgeCases(t *testing.T, svc ports.RepoManager) { }) } +// testGetAllChildrenVtxosSiblingIsolation verifies that GetAllChildrenVtxos, +// when called with a specific (txid, vout) outpoint, returns only that +// outpoint's descendant lineage and does not include sibling outpoints of the +// same txid or their descendants. +// +// Scenario: a parent tx A produces two outputs (A, 0) and (A, 1). Each is +// spent by a different offchain tx (ark_txid X vs Y), each of which has its +// own descendant. Sweeping the checkpoint for (A, 0) must not sweep (A, 1)'s +// lineage, since those funds belong to an independent subtree. +func testGetAllChildrenVtxosSiblingIsolation(t *testing.T, svc ports.RepoManager) { + t.Run("test_get_all_children_vtxos_sibling_isolation", func(t *testing.T) { + ctx := context.Background() + suffix := randomString(16) + + commitmentTxid := randomString(32) + parentTxid := "sibling-parent-" + suffix + arkTxidForVout0 := "sibling-arktx-0-" + suffix + arkTxidForVout1 := "sibling-arktx-1-" + suffix + + // Parent outputs: (parent, 0) spent by arkTxidForVout0, + // (parent, 1) spent by arkTxidForVout1. Same txid, different lineages. + parentVout0 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: parentTxid, VOut: 0}, + PubKey: pubkey, + Amount: 1000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + ArkTxid: arkTxidForVout0, + } + parentVout1 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: parentTxid, VOut: 1}, + PubKey: pubkey2, + Amount: 2000, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + ArkTxid: arkTxidForVout1, + } + + // Descendant of (parent, 0): belongs to the lineage we're sweeping. + descendantOfVout0 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: arkTxidForVout0, VOut: 0}, + PubKey: pubkey, + Amount: 900, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + ArkTxid: "", + } + + // Descendant of (parent, 1): belongs to an independent lineage — + // must NOT be returned when we query (parent, 0). + descendantOfVout1 := domain.Vtxo{ + Outpoint: domain.Outpoint{Txid: arkTxidForVout1, VOut: 0}, + PubKey: pubkey2, + Amount: 1900, + RootCommitmentTxid: commitmentTxid, + CommitmentTxids: []string{commitmentTxid}, + ArkTxid: "", + } + + require.NoError(t, svc.Vtxos().AddVtxos(ctx, []domain.Vtxo{ + parentVout0, parentVout1, descendantOfVout0, descendantOfVout1, + })) + + // Querying (parent, 0) should return (parent, 0) and its descendant + // only — NOT (parent, 1) or its descendant. + got, err := svc.Vtxos().GetAllChildrenVtxos(ctx, parentVout0.Outpoint) + require.NoError(t, err) + gotSet := make(map[domain.Outpoint]bool, len(got)) + for _, op := range got { + gotSet[op] = true + } + require.True(t, gotSet[parentVout0.Outpoint], + "seed outpoint (parent, 0) should be in result") + require.True(t, gotSet[descendantOfVout0.Outpoint], + "descendant of (parent, 0) should be in result") + require.False(t, gotSet[parentVout1.Outpoint], + "sibling (parent, 1) MUST NOT be in result — independent lineage") + require.False(t, gotSet[descendantOfVout1.Outpoint], + "descendant of sibling (parent, 1) MUST NOT be in result") + + // Symmetric check: querying (parent, 1) only returns its own lineage. + got, err = svc.Vtxos().GetAllChildrenVtxos(ctx, parentVout1.Outpoint) + require.NoError(t, err) + gotSet = make(map[domain.Outpoint]bool, len(got)) + for _, op := range got { + gotSet[op] = true + } + require.True(t, gotSet[parentVout1.Outpoint]) + require.True(t, gotSet[descendantOfVout1.Outpoint]) + require.False(t, gotSet[parentVout0.Outpoint]) + require.False(t, gotSet[descendantOfVout0.Outpoint]) + }) +} + // testConvergentMultiParentMarkerDAG builds a diamond-shaped marker DAG where two // independent root→mid branches converge into a single merge marker, then extend // to a leaf. Verifies GetVtxoChainByMarkers returns correct VTXOs per marker set, diff --git a/internal/infrastructure/db/sqlite/marker_repo.go b/internal/infrastructure/db/sqlite/marker_repo.go index f8eb9c963..49f91a449 100644 --- a/internal/infrastructure/db/sqlite/marker_repo.go +++ b/internal/infrastructure/db/sqlite/marker_repo.go @@ -10,6 +10,7 @@ import ( "github.com/arkade-os/arkd/internal/core/domain" "github.com/arkade-os/arkd/internal/infrastructure/db/sqlite/sqlc/queries" + log "github.com/sirupsen/logrus" ) type markerRepository struct { @@ -282,32 +283,38 @@ func (m *markerRepository) GetVtxosByMarker( } func (m *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID string) (int64, error) { - // First check if the marker exists (foreign key constraint on swept_marker) - marker, err := m.GetMarker(ctx, markerID) - if err != nil { - return 0, fmt.Errorf("failed to check marker existence: %w", err) - } - if marker == nil { - return 0, nil // Marker doesn't exist, nothing to sweep - } + var count int64 + txBody := func(qtx *queries.Queries) error { + // First check if the marker exists (foreign key constraint on swept_marker) + if _, err := qtx.SelectMarker(ctx, markerID); err != nil { + if err == sql.ErrNoRows { + return nil // Marker doesn't exist, nothing to sweep + } + return fmt.Errorf("failed to check marker existence: %w", err) + } - // Count unswept VTXOs with this marker before inserting to swept_marker - count, err := m.querier.CountUnsweptVtxosByMarkerId( - ctx, - sql.NullString{String: markerID, Valid: len(markerID) > 0}, - ) - if err != nil { - return 0, fmt.Errorf("failed to count unswept vtxos: %w", err) - } + // Count unswept VTXOs with this marker before inserting to swept_marker + c, err := qtx.CountUnsweptVtxosByMarkerId( + ctx, + sql.NullString{String: markerID, Valid: len(markerID) > 0}, + ) + if err != nil { + return fmt.Errorf("failed to count unswept vtxos: %w", err) + } - // Insert the marker into swept_marker (sweep state is computed via view) - if err := m.querier.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ - MarkerID: markerID, - SweptAt: time.Now().UnixMilli(), - }); err != nil { - return 0, fmt.Errorf("failed to insert swept marker: %w", err) + // Insert the marker into swept_marker (sweep state is computed via view) + if err := qtx.InsertSweptMarker(ctx, queries.InsertSweptMarkerParams{ + MarkerID: markerID, + SweptAt: time.Now().UnixMilli(), + }); err != nil { + return fmt.Errorf("failed to insert swept marker: %w", err) + } + count = c + return nil + } + if err := execTx(ctx, m.db, txBody); err != nil { + return 0, err } - return count, nil } @@ -393,7 +400,7 @@ func (m *markerRepository) GetVtxoChainByMarkers( for _, markerID := range markerIDs { rows, err := m.querier.SelectVtxoChainByMarker( ctx, - sql.NullString{String: markerID, Valid: true}, + sql.NullString{String: markerID, Valid: len(markerID) > 0}, ) if err != nil { return nil, err @@ -537,13 +544,16 @@ func rowToVtxoFromChainQuery(row queries.SelectVtxoChainByMarkerRow) domain.Vtxo } } -// parseMarkersJSON parses a JSON array string into a slice of strings +// parseMarkersJSON parses a JSON array string into a slice of strings. +// Logs and returns nil if the JSON is malformed so that corrupt markers are +// surfaced instead of silently treated as empty. func parseMarkersJSON(markersJSON string) []string { if markersJSON == "" { return nil } var markerIDs []string if err := json.Unmarshal([]byte(markersJSON), &markerIDs); err != nil { + log.WithError(err).Warnf("failed to parse markers JSON: %q", markersJSON) return nil } return markerIDs diff --git a/internal/infrastructure/db/sqlite/marker_repo_test.go b/internal/infrastructure/db/sqlite/marker_repo_test.go new file mode 100644 index 000000000..34f6baf2d --- /dev/null +++ b/internal/infrastructure/db/sqlite/marker_repo_test.go @@ -0,0 +1,83 @@ +package sqlitedb + +import ( + "testing" + + log "github.com/sirupsen/logrus" + "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/require" +) + +// TestParseMarkersJSON_LogsOnMalformed verifies that parseMarkersJSON emits a +// warn log on malformed JSON (instead of silently swallowing the unmarshal +// error and returning nil). Surfacing corrupt markers is important so that +// operators can detect data corruption rather than having it masquerade as +// "no markers present". +func TestParseMarkersJSON_LogsOnMalformed(t *testing.T) { + t.Run("malformed_json_logs_warning", func(t *testing.T) { + hook := test.NewGlobal() + t.Cleanup(func() { + hook.Reset() + log.SetOutput(log.StandardLogger().Out) // restore default + }) + + got := parseMarkersJSON(`not-valid-json`) + require.Nil(t, got, "malformed JSON must still return nil for compatibility") + + entries := hook.AllEntries() + require.NotEmpty(t, entries, "expected a warn log for malformed markers JSON") + + var matched bool + for _, e := range entries { + if e.Level == log.WarnLevel && + e.Message != "" && + containsAll(e.Message, "failed to parse markers JSON") { + matched = true + break + } + } + require.True(t, matched, + "expected a warn entry mentioning 'failed to parse markers JSON', got: %v", + hook.AllEntries()) + }) + + t.Run("empty_input_no_log", func(t *testing.T) { + hook := test.NewGlobal() + t.Cleanup(func() { + hook.Reset() + }) + + got := parseMarkersJSON("") + require.Nil(t, got) + require.Empty(t, hook.AllEntries(), + "empty input is not an error and must not log") + }) + + t.Run("valid_json_no_log", func(t *testing.T) { + hook := test.NewGlobal() + t.Cleanup(func() { + hook.Reset() + }) + + got := parseMarkersJSON(`["m1","m2"]`) + require.Equal(t, []string{"m1", "m2"}, got) + require.Empty(t, hook.AllEntries(), + "valid input must not log") + }) +} + +func containsAll(s string, subs ...string) bool { + for _, sub := range subs { + found := false + for i := 0; i+len(sub) <= len(s); i++ { + if s[i:i+len(sub)] == sub { + found = true + break + } + } + if !found { + return false + } + } + return true +} diff --git a/internal/infrastructure/db/sqlite/migration/20260416120000_add_swept_vtxo.down.sql b/internal/infrastructure/db/sqlite/migration/20260416120000_add_swept_vtxo.down.sql index 6c947b853..db97e34ac 100644 --- a/internal/infrastructure/db/sqlite/migration/20260416120000_add_swept_vtxo.down.sql +++ b/internal/infrastructure/db/sqlite/migration/20260416120000_add_swept_vtxo.down.sql @@ -1,3 +1,25 @@ +-- Guard against silently resurrecting swept VTXOs. +-- +-- swept_vtxo holds per-outpoint sweep state for the checkpoint-sweep path. +-- Dropping the table would make vtxo_vw.swept flip back to false for every +-- outpoint tracked only here (marker-based sweeps still survive via +-- swept_marker). When the table has data, fail loudly rather than silently +-- discard it. When the table is empty, the rollback is safe — drop the +-- table and restore the pre-swept_vtxo view shape. +-- +-- SQLite has no RAISE outside of triggers, so we route through a trigger on +-- a throwaway temp table. The conditional INSERT fires the trigger only when +-- swept_vtxo has at least one row; otherwise it's a no-op and we fall through +-- to the drop + view recreation. +CREATE TEMP TABLE __abort_swept_vtxo_down (x INTEGER); +CREATE TEMP TRIGGER __abort_swept_vtxo_down_trigger BEFORE INSERT ON __abort_swept_vtxo_down +BEGIN + SELECT RAISE(ABORT, 'irreversible migration: swept_vtxo contains entries; rolling back would resurrect swept VTXOs. Truncate swept_vtxo manually if you accept the data loss, then re-run.'); +END; +INSERT INTO __abort_swept_vtxo_down SELECT 1 FROM swept_vtxo LIMIT 1; +DROP TRIGGER __abort_swept_vtxo_down_trigger; +DROP TABLE __abort_swept_vtxo_down; + DROP TABLE IF EXISTS swept_vtxo; DROP VIEW IF EXISTS intent_with_inputs_vw; diff --git a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go index 6fcffb229..e2bea1f9d 100644 --- a/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go +++ b/internal/infrastructure/db/sqlite/sqlc/queries/query.sql.go @@ -90,6 +90,7 @@ SELECT COUNT(DISTINCT txid || ':' || CAST(vout AS TEXT)) FROM vtxo_vw WHERE mark // Count VTXOs whose markers JSON array contains the given marker_id and are not swept. // Uses LIKE because sqlc cannot parse json_each with view columns. +// Uses DISTINCT to avoid double-counting VTXOs with multiple asset projections. func (q *Queries) CountUnsweptVtxosByMarkerId(ctx context.Context, markerID sql.NullString) (int64, error) { row := q.db.QueryRowContext(ctx, countUnsweptVtxosByMarkerId, markerID) var count int64 @@ -113,7 +114,10 @@ WHERE descendant_markers.id NOT IN (SELECT sm.marker_id FROM swept_marker sm) ` // Recursively get a marker and all its descendants (markers whose parent_markers contain it) -// Uses json_each instead of LIKE to avoid false positives with special characters (%, _) +// Uses json_each instead of LIKE to avoid false positives with special characters (%, _). +// Uses UNION (set semantics, not UNION ALL) so rows already produced are filtered, +// which makes this cycle-safe. Do not convert to UNION ALL: cycles in parent_markers +// would cause the recursion to run unbounded. func (q *Queries) GetDescendantMarkerIds(ctx context.Context, rootMarkerID string) ([]string, error) { rows, err := q.db.QueryContext(ctx, getDescendantMarkerIds, rootMarkerID) if err != nil { @@ -2300,12 +2304,12 @@ func (q *Queries) SelectVtxosByMarkerId(ctx context.Context, markerID sql.NullSt const selectVtxosOutpointsByArkTxidRecursive = `-- name: SelectVtxosOutpointsByArkTxidRecursive :many WITH RECURSIVE descendants_chain AS ( - -- seed + -- seed: only the specific outpoint, not all vouts of the txid SELECT v.txid, v.vout, v.preconfirmed, v.ark_txid, v.spent_by, 0 AS depth, v.txid||':'||v.vout AS visited FROM vtxo v - WHERE v.txid = ?1 + WHERE v.txid = ?1 AND v.vout = ?2 UNION ALL @@ -2329,14 +2333,23 @@ FROM nodes ORDER BY depth, txid, vout ` +type SelectVtxosOutpointsByArkTxidRecursiveParams struct { + Txid string + Vout int64 +} + type SelectVtxosOutpointsByArkTxidRecursiveRow struct { Txid string Vout int64 } +// Returns the seed outpoint (txid, vout) and all VTXOs descending from it +// via ark_txid links. Scoped to a single outpoint (not the whole txid) so that +// sibling outputs of the seed tx, which belong to independent lineages, are +// not included. // keep one row per node at its MIN depth (layers) -func (q *Queries) SelectVtxosOutpointsByArkTxidRecursive(ctx context.Context, txid string) ([]SelectVtxosOutpointsByArkTxidRecursiveRow, error) { - rows, err := q.db.QueryContext(ctx, selectVtxosOutpointsByArkTxidRecursive, txid) +func (q *Queries) SelectVtxosOutpointsByArkTxidRecursive(ctx context.Context, arg SelectVtxosOutpointsByArkTxidRecursiveParams) ([]SelectVtxosOutpointsByArkTxidRecursiveRow, error) { + rows, err := q.db.QueryContext(ctx, selectVtxosOutpointsByArkTxidRecursive, arg.Txid, arg.Vout) if err != nil { return nil, err } diff --git a/internal/infrastructure/db/sqlite/sqlc/query.sql b/internal/infrastructure/db/sqlite/sqlc/query.sql index 45d808f8a..ba70effc3 100644 --- a/internal/infrastructure/db/sqlite/sqlc/query.sql +++ b/internal/infrastructure/db/sqlite/sqlc/query.sql @@ -304,13 +304,17 @@ WHERE v.swept = false OR (',' || COALESCE(v.commitments, '') || ',') LIKE '%,' || @commitment_txid || ',%'); -- name: SelectVtxosOutpointsByArkTxidRecursive :many +-- Returns the seed outpoint (txid, vout) and all VTXOs descending from it +-- via ark_txid links. Scoped to a single outpoint (not the whole txid) so that +-- sibling outputs of the seed tx, which belong to independent lineages, are +-- not included. WITH RECURSIVE descendants_chain AS ( - -- seed + -- seed: only the specific outpoint, not all vouts of the txid SELECT v.txid, v.vout, v.preconfirmed, v.ark_txid, v.spent_by, 0 AS depth, v.txid||':'||v.vout AS visited FROM vtxo v - WHERE v.txid = @txid + WHERE v.txid = @txid AND v.vout = @vout UNION ALL @@ -475,7 +479,10 @@ SELECT EXISTS(SELECT 1 FROM swept_marker WHERE marker_id = @marker_id) AS is_swe -- name: GetDescendantMarkerIds :many -- Recursively get a marker and all its descendants (markers whose parent_markers contain it) --- Uses json_each instead of LIKE to avoid false positives with special characters (%, _) +-- Uses json_each instead of LIKE to avoid false positives with special characters (%, _). +-- Uses UNION (set semantics, not UNION ALL) so rows already produced are filtered, +-- which makes this cycle-safe. Do not convert to UNION ALL: cycles in parent_markers +-- would cause the recursion to run unbounded. WITH RECURSIVE descendant_markers(id) AS ( -- Base case: the marker being swept SELECT marker.id FROM marker WHERE marker.id = @root_marker_id diff --git a/internal/infrastructure/db/sqlite/vtxo_repo.go b/internal/infrastructure/db/sqlite/vtxo_repo.go index 848740fa1..6035c3e7c 100644 --- a/internal/infrastructure/db/sqlite/vtxo_repo.go +++ b/internal/infrastructure/db/sqlite/vtxo_repo.go @@ -12,6 +12,7 @@ import ( "github.com/arkade-os/arkd/internal/core/domain" "github.com/arkade-os/arkd/internal/infrastructure/db/sqlite/sqlc/queries" + log "github.com/sirupsen/logrus" ) type vtxoRepository struct { @@ -425,9 +426,15 @@ func (v *vtxoRepository) GetSweepableVtxosByCommitmentTxid( } func (v *vtxoRepository) GetAllChildrenVtxos( - ctx context.Context, txid string, + ctx context.Context, outpoint domain.Outpoint, ) ([]domain.Outpoint, error) { - res, err := v.querier.SelectVtxosOutpointsByArkTxidRecursive(ctx, txid) + res, err := v.querier.SelectVtxosOutpointsByArkTxidRecursive( + ctx, + queries.SelectVtxosOutpointsByArkTxidRecursiveParams{ + Txid: outpoint.Txid, + Vout: int64(outpoint.VOut), + }, + ) if err != nil { return nil, err } @@ -585,13 +592,16 @@ func toBool(v interface{}) bool { } } -// parseMarkersJSONFromVtxo parses a JSON array string into a slice of strings for vtxo repo +// parseMarkersJSONFromVtxo parses a JSON array string into a slice of strings for vtxo repo. +// Logs and returns nil if the JSON is malformed so that corrupt markers are +// surfaced instead of silently treated as empty. func parseMarkersJSONFromVtxo(markersJSON string) []string { if markersJSON == "" { return nil } var markerIDs []string if err := json.Unmarshal([]byte(markersJSON), &markerIDs); err != nil { + log.WithError(err).Warnf("failed to parse markers JSON: %q", markersJSON) return nil } return markerIDs diff --git a/internal/infrastructure/db/swept_vtxo_down_test.go b/internal/infrastructure/db/swept_vtxo_down_test.go new file mode 100644 index 000000000..7de65f70e --- /dev/null +++ b/internal/infrastructure/db/swept_vtxo_down_test.go @@ -0,0 +1,112 @@ +package db_test + +import ( + "database/sql" + "embed" + "strings" + "testing" + + sqlitedb "github.com/arkade-os/arkd/internal/infrastructure/db/sqlite" + "github.com/golang-migrate/migrate/v4" + sqlitemigrate "github.com/golang-migrate/migrate/v4/database/sqlite" + "github.com/golang-migrate/migrate/v4/source/iofs" + "github.com/stretchr/testify/require" +) + +//go:embed sqlite/migration/* +var sweptVtxoTestMigrations embed.FS + +const sweptVtxoMigrationVersion = 20260416120000 + +// TestSweptVtxoDownMigration_Guard verifies that the sqlite down migration +// for 20260416120000_add_swept_vtxo aborts when swept_vtxo has data (to +// prevent silently resurrecting swept VTXOs) but proceeds cleanly when the +// table is empty. +func TestSweptVtxoDownMigration_Guard(t *testing.T) { + t.Run("aborts_when_swept_vtxo_has_data", func(t *testing.T) { + m, db := newSweptVtxoMigrator(t) + t.Cleanup(func() { + // Force the version so cleanup doesn't complain about a dirty + // migration state left by the expected failure. + _ = m.Force(sweptVtxoMigrationVersion) + //nolint:errcheck + db.Close() + }) + + require.NoError(t, m.Migrate(sweptVtxoMigrationVersion)) + + _, err := db.Exec( + `INSERT INTO swept_vtxo (txid, vout, swept_at) VALUES (?, ?, ?)`, + "deadbeef", 0, 1234567890, + ) + require.NoError(t, err, "seed insert must succeed before the guard test") + + // Stepping back one migration should fail: the guard trigger fires + // because swept_vtxo has a row. + err = m.Steps(-1) + require.Error(t, err, "down migration must abort when swept_vtxo is non-empty") + require.True(t, + strings.Contains(err.Error(), "irreversible migration") || + strings.Contains(err.Error(), "swept_vtxo"), + "error should mention the guard: got %v", err, + ) + + // swept_vtxo must still exist and still contain the row — the + // transaction aborted before the DROP ran. + var count int + err = db.QueryRow(`SELECT count(*) FROM swept_vtxo`).Scan(&count) + require.NoError(t, err, + "swept_vtxo should still exist after the aborted down migration") + require.Equal(t, 1, count, + "swept_vtxo data must be preserved when the guard fires") + }) + + t.Run("proceeds_when_swept_vtxo_is_empty", func(t *testing.T) { + m, db := newSweptVtxoMigrator(t) + t.Cleanup(func() { + //nolint:errcheck + db.Close() + }) + + require.NoError(t, m.Migrate(sweptVtxoMigrationVersion)) + + // swept_vtxo exists but is empty — the guard should not fire. + var count int + require.NoError(t, db.QueryRow(`SELECT count(*) FROM swept_vtxo`).Scan(&count)) + require.Equal(t, 0, count) + + require.NoError(t, m.Steps(-1), + "down migration must succeed when swept_vtxo is empty") + + // swept_vtxo should be gone; vtxo_vw should still exist (restored by + // the down migration body that runs past the guard). + err := db.QueryRow(`SELECT count(*) FROM swept_vtxo`).Scan(&count) + require.Error(t, err, "swept_vtxo should have been dropped") + require.Contains(t, err.Error(), "no such table") + + rows, err := db.Query(`SELECT name FROM sqlite_master WHERE type='view' AND name='vtxo_vw'`) + require.NoError(t, err) + defer rows.Close() + require.True(t, rows.Next(), + "vtxo_vw view should have been recreated by the down migration") + }) +} + +// newSweptVtxoMigrator returns a fresh in-memory sqlite DB paired with a +// migrate.Migrate bound to the embedded sqlite migration source. +func newSweptVtxoMigrator(t *testing.T) (*migrate.Migrate, *sql.DB) { + t.Helper() + db, err := sqlitedb.OpenDb(":memory:") + require.NoError(t, err) + + driver, err := sqlitemigrate.WithInstance(db, &sqlitemigrate.Config{}) + require.NoError(t, err) + + source, err := iofs.New(sweptVtxoTestMigrations, "sqlite/migration") + require.NoError(t, err) + + m, err := migrate.NewWithInstance("iofs", source, "arkdb", driver) + require.NoError(t, err) + + return m, db +} From 54924c074782f308e78b13a093bc9f3ca7c71278 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Fri, 17 Apr 2026 14:07:14 -0400 Subject: [PATCH 49/54] fix hardcoded vout=0 when extracting leaf VTXO outpoints for sweep --- internal/core/application/admin.go | 15 +++++++++++---- internal/core/application/sweeper.go | 15 +++++++++++---- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/internal/core/application/admin.go b/internal/core/application/admin.go index 5ef0766e6..707b821e8 100644 --- a/internal/core/application/admin.go +++ b/internal/core/application/admin.go @@ -705,11 +705,18 @@ func (a *adminService) saveBatchSweptEvents( } for _, leaf := range vtxosLeaves { - vtxo := domain.Outpoint{ - Txid: leaf.UnsignedTx.TxID(), - VOut: 0, + // The VTXO is the first non-anchor output; leaf txs can + // carry an anchor at vout 0, so the VTXO is not always at + // vout 0. extractVtxoOutpoint handles that. + vtxo, err := extractVtxoOutpoint(leaf) + if err != nil { + log.WithError(err).Errorf( + "failed to extract vtxo outpoint from leaf %s", + leaf.UnsignedTx.TxID(), + ) + continue } - leafVtxos = append(leafVtxos, vtxo) + leafVtxos = append(leafVtxos, *vtxo) } } } diff --git a/internal/core/application/sweeper.go b/internal/core/application/sweeper.go index b02eb8848..66a2dab19 100644 --- a/internal/core/application/sweeper.go +++ b/internal/core/application/sweeper.go @@ -542,12 +542,19 @@ func (s *sweeper) createBatchSweepTask(commitmentTxid, vtxoTreeRootTxid string) } for _, leaf := range vtxosLeaves { - vtxo := domain.Outpoint{ - Txid: leaf.UnsignedTx.TxID(), - VOut: 0, + // The VTXO is the first non-anchor output; leaf txs can + // carry an anchor at vout 0, so the VTXO is not always + // at vout 0. extractVtxoOutpoint handles that. + vtxo, err := extractVtxoOutpoint(leaf) + if err != nil { + log.WithError(err).Errorf( + "failed to extract vtxo outpoint from leaf %s", + leaf.UnsignedTx.TxID(), + ) + continue } - sweepableVtxos = append(sweepableVtxos, vtxo) + sweepableVtxos = append(sweepableVtxos, *vtxo) } if len(sweepableVtxos) <= 0 { From 2d8c1ba83debc4096fb6a773281727cd652c47a2 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Fri, 17 Apr 2026 15:52:48 -0400 Subject: [PATCH 50/54] restore swept on rollback, fix isActive, document dual sweep path --- internal/core/application/token_cache.go | 10 ++++++++-- .../20260210100000_add_depth_and_markers.down.sql | 12 ++++++++++++ .../migration/20260416120000_add_swept_vtxo.up.sql | 9 +++++++++ .../migration/20260416120000_add_swept_vtxo.up.sql | 9 +++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/internal/core/application/token_cache.go b/internal/core/application/token_cache.go index 49ad34eda..ced1c1dd8 100644 --- a/internal/core/application/token_cache.go +++ b/internal/core/application/token_cache.go @@ -81,7 +81,10 @@ func (c *tokenCache) touch(hash string) { } } -// isActive returns true if the hash has a non-expired cache entry. +// isActive returns true if the hash has any non-expired cache entry. In +// practice touch/add set every outpoint under a hash to the same expiry, so +// any single entry would answer the question; scanning all entries removes +// reliance on that invariant and on Go's non-deterministic map iteration. func (c *tokenCache) isActive(hash string) bool { c.mu.RLock() defer c.mu.RUnlock() @@ -90,8 +93,11 @@ func (c *tokenCache) isActive(hash string) bool { if !ok { return false } + now := time.Now() for _, expiresAt := range outpoints { - return time.Now().Before(expiresAt) + if now.Before(expiresAt) { + return true + } } return false } diff --git a/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.down.sql b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.down.sql index a13d60016..9db38b334 100644 --- a/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.down.sql +++ b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.down.sql @@ -2,6 +2,18 @@ DROP VIEW IF EXISTS intent_with_inputs_vw; DROP VIEW IF EXISTS vtxo_vw; +-- Restore the swept column that the up migration dropped. Backfill from +-- swept_marker (joined via the markers JSON array) before dropping the marker +-- tables, otherwise the rollback silently loses sweep state — VTXOs that +-- were swept via swept_marker would reappear as unswept. +ALTER TABLE vtxo ADD COLUMN swept BOOLEAN NOT NULL DEFAULT false; +UPDATE vtxo v +SET swept = true +WHERE EXISTS ( + SELECT 1 FROM swept_marker sm + WHERE v.markers @> jsonb_build_array(sm.marker_id) +); + -- Drop markers index and column from vtxo DROP INDEX IF EXISTS idx_vtxo_markers; ALTER TABLE vtxo DROP COLUMN IF EXISTS markers; diff --git a/internal/infrastructure/db/postgres/migration/20260416120000_add_swept_vtxo.up.sql b/internal/infrastructure/db/postgres/migration/20260416120000_add_swept_vtxo.up.sql index dd30fe134..d453506f2 100644 --- a/internal/infrastructure/db/postgres/migration/20260416120000_add_swept_vtxo.up.sql +++ b/internal/infrastructure/db/postgres/migration/20260416120000_add_swept_vtxo.up.sql @@ -15,6 +15,15 @@ CREATE TABLE IF NOT EXISTS swept_vtxo ( DROP VIEW IF EXISTS intent_with_inputs_vw; DROP VIEW IF EXISTS vtxo_vw; +-- swept is OR'd across two sources on purpose: +-- * swept_marker — populated by batch/round sweeps. Coarse-grained: a single +-- marker can cover many VTXOs, so marker-based sweeping is efficient for +-- whole-round sweeps but would over-reach if applied to checkpoint sweeps +-- (markers are shared across independent subtrees). +-- * swept_vtxo — populated by checkpoint sweeps. Fine-grained: one row per +-- (txid, vout), so it safely scopes to a single outpoint's lineage. +-- New sweep code paths must pick the right table; maintainers adding a third +-- sweep path should extend this OR rather than re-overloading one of them. CREATE VIEW vtxo_vw AS SELECT v.*, COALESCE(vc.commitments, '') AS commitments, diff --git a/internal/infrastructure/db/sqlite/migration/20260416120000_add_swept_vtxo.up.sql b/internal/infrastructure/db/sqlite/migration/20260416120000_add_swept_vtxo.up.sql index cf4d64bc5..3d7056995 100644 --- a/internal/infrastructure/db/sqlite/migration/20260416120000_add_swept_vtxo.up.sql +++ b/internal/infrastructure/db/sqlite/migration/20260416120000_add_swept_vtxo.up.sql @@ -10,6 +10,15 @@ CREATE TABLE IF NOT EXISTS swept_vtxo ( DROP VIEW IF EXISTS intent_with_inputs_vw; DROP VIEW IF EXISTS vtxo_vw; +-- swept is OR'd across two sources on purpose: +-- * swept_marker — populated by batch/round sweeps. Coarse-grained: a single +-- marker can cover many VTXOs, so marker-based sweeping is efficient for +-- whole-round sweeps but would over-reach if applied to checkpoint sweeps +-- (markers are shared across independent subtrees). +-- * swept_vtxo — populated by checkpoint sweeps. Fine-grained: one row per +-- (txid, vout), so it safely scopes to a single outpoint's lineage. +-- New sweep code paths must pick the right table; maintainers adding a third +-- sweep path should extend this OR rather than re-overloading one of them. CREATE VIEW vtxo_vw AS SELECT v.*, COALESCE(( From e0d0c0eda748eea5c540980cdb27f114db17b0f1 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Mon, 27 Apr 2026 11:39:40 -0400 Subject: [PATCH 51/54] Fix RepoManager interface stubs and test pointer dereference after master merge --- .../core/application/indexer_bench_test.go | 18 +++++++++++------- internal/core/application/indexer_test.go | 10 ++++++---- internal/core/application/sweeper_test.go | 10 ++++++---- internal/core/application/utils_test.go | 6 +++--- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/internal/core/application/indexer_bench_test.go b/internal/core/application/indexer_bench_test.go index 89c07be04..2d51f2fea 100644 --- a/internal/core/application/indexer_bench_test.go +++ b/internal/core/application/indexer_bench_test.go @@ -133,10 +133,12 @@ func (m *benchRepoManager) OffchainTxs() domain.OffchainTxRepository { } return m.offchainRepo } -func (m *benchRepoManager) Convictions() domain.ConvictionRepository { return nil } -func (m *benchRepoManager) Assets() domain.AssetRepository { return nil } -func (m *benchRepoManager) Fees() domain.FeeRepository { return nil } -func (m *benchRepoManager) Close() {} +func (m *benchRepoManager) Convictions() domain.ConvictionRepository { return nil } +func (m *benchRepoManager) Assets() domain.AssetRepository { return nil } +func (m *benchRepoManager) Fees() domain.FeeRepository { return nil } +func (m *benchRepoManager) RegisterBatchUpdateHandler(func(data domain.Round)) {} +func (m *benchRepoManager) RegisterOffchainTxUpdateHandler(func(domain.OffchainTx)) {} +func (m *benchRepoManager) Close() {} // benchTxid returns a deterministic 64-char hex txid for index i. func benchTxid(i int) string { @@ -870,6 +872,8 @@ func (m *wrappedRepoManager) OffchainTxs() domain.OffchainTxRepository { return func (m *wrappedRepoManager) Convictions() domain.ConvictionRepository { panic("Convictions: not wired") } -func (m *wrappedRepoManager) Assets() domain.AssetRepository { panic("Assets: not wired") } -func (m *wrappedRepoManager) Fees() domain.FeeRepository { panic("Fees: not wired") } -func (m *wrappedRepoManager) Close() {} +func (m *wrappedRepoManager) Assets() domain.AssetRepository { panic("Assets: not wired") } +func (m *wrappedRepoManager) Fees() domain.FeeRepository { panic("Fees: not wired") } +func (m *wrappedRepoManager) RegisterBatchUpdateHandler(func(data domain.Round)) {} +func (m *wrappedRepoManager) RegisterOffchainTxUpdateHandler(func(domain.OffchainTx)) {} +func (m *wrappedRepoManager) Close() {} diff --git a/internal/core/application/indexer_test.go b/internal/core/application/indexer_test.go index 13c98fe4a..ea6d7b24e 100644 --- a/internal/core/application/indexer_test.go +++ b/internal/core/application/indexer_test.go @@ -352,10 +352,12 @@ func (m *mockRepoManagerForIndexer) OffchainTxs() domain.OffchainTxRepository { } return m.offchainTxs } -func (m *mockRepoManagerForIndexer) Convictions() domain.ConvictionRepository { return nil } -func (m *mockRepoManagerForIndexer) Assets() domain.AssetRepository { return nil } -func (m *mockRepoManagerForIndexer) Fees() domain.FeeRepository { return nil } -func (m *mockRepoManagerForIndexer) Close() {} +func (m *mockRepoManagerForIndexer) Convictions() domain.ConvictionRepository { return nil } +func (m *mockRepoManagerForIndexer) Assets() domain.AssetRepository { return nil } +func (m *mockRepoManagerForIndexer) Fees() domain.FeeRepository { return nil } +func (m *mockRepoManagerForIndexer) RegisterBatchUpdateHandler(func(data domain.Round)) {} +func (m *mockRepoManagerForIndexer) RegisterOffchainTxUpdateHandler(func(domain.OffchainTx)) {} +func (m *mockRepoManagerForIndexer) Close() {} // newTestIndexer creates a fresh set of mock repos and an indexerService for testing. func newChainTestIndexer() ( diff --git a/internal/core/application/sweeper_test.go b/internal/core/application/sweeper_test.go index 909aa693b..1b0513a0c 100644 --- a/internal/core/application/sweeper_test.go +++ b/internal/core/application/sweeper_test.go @@ -470,10 +470,12 @@ func (m *mockRepoManager) Vtxos() domain.VtxoRepository { retur func (m *mockRepoManager) Markers() domain.MarkerRepository { return m.markers } func (m *mockRepoManager) ScheduledSession() domain.ScheduledSessionRepo { return nil } func (m *mockRepoManager) OffchainTxs() domain.OffchainTxRepository { return nil } -func (m *mockRepoManager) Convictions() domain.ConvictionRepository { return nil } -func (m *mockRepoManager) Assets() domain.AssetRepository { return nil } -func (m *mockRepoManager) Fees() domain.FeeRepository { return nil } -func (m *mockRepoManager) Close() {} +func (m *mockRepoManager) Convictions() domain.ConvictionRepository { return nil } +func (m *mockRepoManager) Assets() domain.AssetRepository { return nil } +func (m *mockRepoManager) Fees() domain.FeeRepository { return nil } +func (m *mockRepoManager) RegisterBatchUpdateHandler(func(data domain.Round)) {} +func (m *mockRepoManager) RegisterOffchainTxUpdateHandler(func(domain.OffchainTx)) {} +func (m *mockRepoManager) Close() {} type mockScheduler struct{} diff --git a/internal/core/application/utils_test.go b/internal/core/application/utils_test.go index 4a46ed167..c9c07dedb 100644 --- a/internal/core/application/utils_test.go +++ b/internal/core/application/utils_test.go @@ -89,7 +89,7 @@ func TestGetNewVtxosFromRound_MarkerIDsAndDepth(t *testing.T) { }, } - vtxos := getNewVtxosFromRound(round) + vtxos := getNewVtxosFromRound(*round) require.Len(t, vtxos, 2) @@ -133,7 +133,7 @@ func TestGetNewVtxosFromRound_EmptyVtxoTree(t *testing.T) { VtxoTree: nil, } - vtxos := getNewVtxosFromRound(round) + vtxos := getNewVtxosFromRound(*round) require.Nil(t, vtxos) } @@ -165,7 +165,7 @@ func TestGetNewVtxosFromRound_SingleOutput(t *testing.T) { }, } - vtxos := getNewVtxosFromRound(round) + vtxos := getNewVtxosFromRound(*round) require.Len(t, vtxos, 1) vtxo := vtxos[0] From 4bf8b902f13241d7095e1cd27a2f6056fd432847 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Tue, 28 Apr 2026 20:22:58 -0400 Subject: [PATCH 52/54] dispatch batch update handler for non-ended rounds --- internal/infrastructure/db/service.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index db65b0e02..567897c81 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -499,6 +499,7 @@ func (s *service) updateProjectionsAfterRoundEvents(events []domain.Event) { log.Debugf("added or updated round %s", round.Id) if !round.IsEnded() { + go s.batchUpdateHandler.dispatch(*round) return } From e8d55da85d9a025dd2e4ef7ebe100bf4bd2d349a Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Wed, 29 Apr 2026 10:34:01 -0400 Subject: [PATCH 53/54] fix marker over-reach in batch sweep, add chain walk bound --- internal/core/application/indexer.go | 23 ++++++- .../infrastructure/db/badger/marker_repo.go | 20 +++--- ...0260210100000_add_depth_and_markers.up.sql | 2 + internal/infrastructure/db/service.go | 61 +++---------------- ...0260210000000_add_depth_and_markers.up.sql | 4 +- 5 files changed, 47 insertions(+), 63 deletions(-) diff --git a/internal/core/application/indexer.go b/internal/core/application/indexer.go index d145dbd41..e456ae584 100644 --- a/internal/core/application/indexer.go +++ b/internal/core/application/indexer.go @@ -37,6 +37,11 @@ const ( maxPageSizeVtxoChain = 100 maxPageSizeVirtualTxs = 100 + // maxVtxoChainWalkSize is a hard upper bound applied when walking the full + // chain before paginating (GetVtxoChainByIntent). Prevents unbounded memory + // growth on pathologically deep chains. + maxVtxoChainWalkSize = 50_000 + defaultAuthTokenTTL = 5 * time.Minute ) @@ -458,10 +463,17 @@ func (i *indexerService) GetVtxoChainByIntent( switch i.txExposure { case exposurePublic: - chain, _, _, err := i.walkVtxoChain(ctx, []domain.Outpoint{outpoint}, math.MaxInt32) + chain, _, _, err := i.walkVtxoChain( + ctx, + []domain.Outpoint{outpoint}, + maxVtxoChainWalkSize+1, + ) if err != nil { return nil, err } + if len(chain) > maxVtxoChainWalkSize { + return nil, fmt.Errorf("chain exceeds maximum size of %d", maxVtxoChainWalkSize) + } txChain, pageResp := paginate(chain, page, maxPageSizeVtxoChain) return &VtxoChainResp{Chain: txChain, Page: pageResp}, nil case exposureWithheld, exposurePrivate: @@ -470,10 +482,17 @@ func (i *indexerService) GetVtxoChainByIntent( } } - chain, allOutpoints, _, err := i.walkVtxoChain(ctx, []domain.Outpoint{outpoint}, math.MaxInt32) + chain, allOutpoints, _, err := i.walkVtxoChain( + ctx, + []domain.Outpoint{outpoint}, + maxVtxoChainWalkSize+1, + ) if err != nil { return nil, err } + if len(chain) > maxVtxoChainWalkSize { + return nil, fmt.Errorf("chain exceeds maximum size of %d", maxVtxoChainWalkSize) + } token, err := i.createAuthToken(allOutpoints) if err != nil { diff --git a/internal/infrastructure/db/badger/marker_repo.go b/internal/infrastructure/db/badger/marker_repo.go index d380ebae3..ea92ba145 100644 --- a/internal/infrastructure/db/badger/marker_repo.go +++ b/internal/infrastructure/db/badger/marker_repo.go @@ -484,20 +484,24 @@ func (r *markerRepository) GetVtxosByMarker( } func (r *markerRepository) SweepVtxosByMarker(ctx context.Context, markerID string) (int64, error) { - // Mark the marker as swept (this also updates vtxo Swept fields) - if err := r.SweepMarker(ctx, markerID, time.Now().Unix()); err != nil { + // Count unswept VTXOs before marking to match Postgres/SQLite behaviour. + var dtos []vtxoDTO + if err := r.vtxoStore.Find(&dtos, + badgerhold.Where("MarkerIDs").Contains(markerID)); err != nil { return 0, err } + var unsweptCount int64 + for _, dto := range dtos { + if !dto.Swept { + unsweptCount++ + } + } - // Count VTXOs affected - var dtos []vtxoDTO - err := r.vtxoStore.Find(&dtos, - badgerhold.Where("MarkerIDs").Contains(markerID)) - if err != nil { + if err := r.SweepMarker(ctx, markerID, time.Now().UnixMilli()); err != nil { return 0, err } - return int64(len(dtos)), nil + return unsweptCount, nil } func (r *markerRepository) CreateRootMarkersForVtxos( diff --git a/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql index 044b06c62..4cac60754 100644 --- a/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql +++ b/internal/infrastructure/db/postgres/migration/20260210100000_add_depth_and_markers.up.sql @@ -50,6 +50,8 @@ ON intent.id = vtxo_vw.intent_id; -- Backfill: Create a marker for every existing VTXO using its outpoint as marker ID -- This ensures every VTXO has at least 1 marker +-- NOTE: this INSERT and the UPDATE below run over all VTXOs and will hold locks. +-- On large production DBs (millions of rows) expect 10-60 seconds; plan a maintenance window. INSERT INTO marker (id, depth, parent_markers) SELECT v.txid || ':' || v.vout, diff --git a/internal/infrastructure/db/service.go b/internal/infrastructure/db/service.go index 567897c81..a61367b80 100644 --- a/internal/infrastructure/db/service.go +++ b/internal/infrastructure/db/service.go @@ -510,10 +510,15 @@ func (s *service) updateProjectionsAfterRoundEvents(events []domain.Event) { event := lastEvent.(domain.BatchSwept) allSweptVtxos := append(event.LeafVtxos, event.PreconfirmedVtxos...) - // marker-based sweeping - sweptCount := s.sweepVtxosWithMarkers(ctx, allSweptVtxos) - if sweptCount > 0 { - log.Debugf("swept %d vtxos using marker-based sweeping", sweptCount) + // Per-outpoint sweeping avoids marker over-reach: markers can be shared + // across independent subtrees when offchain txs consolidate inputs from + // different lineages. Sweeping by marker would incorrectly mark unrelated + // VTXOs as swept (same reason the checkpoint path uses SweepVtxoOutpoints). + sweptAt := time.Now().UnixMilli() + if err := s.markerStore.SweepVtxoOutpoints(ctx, allSweptVtxos, sweptAt); err != nil { + log.WithError(err).Warn("failed to sweep vtxo outpoints for batch") + } else if len(allSweptVtxos) > 0 { + log.Debugf("swept %d vtxo outpoints for batch", len(allSweptVtxos)) } if event.FullySwept { @@ -841,54 +846,6 @@ func getNewVtxosFromRound(round domain.Round, txDecoder ports.TxDecoder) []domai return vtxos } -// sweepVtxosWithMarkers performs marker-based sweeping for VTXOs. -// It groups VTXOs by their marker, sweeps each marker via swept_marker table. -// Returns the total count of VTXOs swept. -func (s *service) sweepVtxosWithMarkers( - ctx context.Context, - vtxoOutpoints []domain.Outpoint, -) int64 { - if len(vtxoOutpoints) == 0 { - return 0 - } - - // Get VTXOs to find their markers - vtxos, err := s.vtxoStore.GetVtxos(ctx, vtxoOutpoints) - if err != nil { - log.WithError(err).Warn("failed to get vtxos for marker-based sweep") - return 0 - } - - // Collect all unique markers from all VTXOs - // Every VTXO is guaranteed to have at least 1 marker after migration - uniqueMarkers := make(map[string]struct{}) - for _, vtxo := range vtxos { - for _, markerID := range vtxo.MarkerIDs { - uniqueMarkers[markerID] = struct{}{} - } - } - - if len(uniqueMarkers) == 0 { - return 0 - } - - // Convert marker set to slice for bulk sweeping - markerIDs := make([]string, 0, len(uniqueMarkers)) - for markerID := range uniqueMarkers { - markerIDs = append(markerIDs, markerID) - } - - sweptAt := time.Now().UnixMilli() - if err := s.markerStore.BulkSweepMarkers(ctx, markerIDs, sweptAt); err != nil { - log.WithError(err).Warn("failed to bulk sweep markers") - return 0 - } - - totalSwept := int64(len(vtxos)) - log.Debugf("bulk swept %d markers affecting %d vtxos", len(markerIDs), totalSwept) - return totalSwept -} - func getAssetsFromTxOuts(txid string, txOuts []ports.TxOut) ( []domain.Asset, map[uint32][]domain.AssetDenomination, error, ) { diff --git a/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql index 1fedbd68b..0e61bf3fa 100644 --- a/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql +++ b/internal/infrastructure/db/sqlite/migration/20260210000000_add_depth_and_markers.up.sql @@ -48,6 +48,9 @@ ON intent.id = vtxo_vw.intent_id; -- Backfill: Create a marker for every existing VTXO using its outpoint as marker ID -- This ensures every VTXO has at least 1 marker +-- NOTE: this INSERT and the UPDATE below run in a single transaction over all VTXOs. +-- On large production DBs (millions of rows) expect a table lock for 10-60 seconds. +-- Plan a maintenance window or apply with a connection timeout if live traffic is present. INSERT INTO marker (id, depth, parent_markers) SELECT v.txid || ':' || v.vout, @@ -108,7 +111,6 @@ ALTER TABLE vtxo_new RENAME TO vtxo; -- Recreate indexes CREATE INDEX IF NOT EXISTS fk_vtxo_intent_id ON vtxo(intent_id); -CREATE INDEX IF NOT EXISTS idx_vtxo_markers ON vtxo(markers); -- Recreate views to compute swept status dynamically CREATE VIEW vtxo_vw AS From 52ff7a2144facb71b385d8f353b30832e0ba4c93 Mon Sep 17 00:00:00 2001 From: Bob Smith <5396652+bitcoin-coder-bob@users.noreply.github.com> Date: Wed, 29 Apr 2026 11:57:40 -0400 Subject: [PATCH 54/54] drop WalletType from vtxo_chain_test, removed from InitArgs in master --- internal/test/e2e/vtxo_chain_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/test/e2e/vtxo_chain_test.go b/internal/test/e2e/vtxo_chain_test.go index 575590588..146a1135a 100644 --- a/internal/test/e2e/vtxo_chain_test.go +++ b/internal/test/e2e/vtxo_chain_test.go @@ -57,7 +57,6 @@ func TestVtxoChain(t *testing.T) { t.Logf("wallet seed: %s", seed) err = client.Init(ctx, arksdk.InitArgs{ - WalletType: arksdk.SingleKeyWallet, ServerUrl: *arkServerUrl, Password: password, Seed: seed,