Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions docs/contrib/read-seq-rollout.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
# Rolling out monotonic read sequences

Read progress is shared by a user across devices. Redis updates, cache fills, and
MongoDB writes must retain the greatest confirmed `read_seq` for each
`(user_id, conversation_id)`. Read-receipt notifications continue to use the
existing asynchronous queue and MQ pipeline.

## MongoDB preflight

The server creates a unique compound index named `user_conversation_unique` on
`seq_user`. The explicit name allows it to coexist with the previous non-unique
index on MongoDB 5.0 and later (the documented deployment uses MongoDB 6.0).
The old index is not automatically dropped.

Before deploying, check for duplicate rows in the intended OpenIM database:

```javascript
db.seq_user.aggregate([
{ $group: {
_id: { user_id: "$user_id", conversation_id: "$conversation_id" },
count: { $sum: 1 }
} },
{ $match: { count: { $gt: 1 } } },
{ $limit: 20 }
])
```

Resolve duplicates through a reviewed data migration before rollout, preserving
the greatest confirmed `read_seq` and reviewing the independent `min_seq` and
`max_seq` bounds. Coordinate the migration with writers. The application does not
delete or merge historical records automatically; index creation fails explicitly
if duplicates remain. Check index build time and permissions for the deployment.
If an equivalent unique index has already been installed manually, reconcile its
name with `user_conversation_unique` before rollout.

## Redis compatibility and limits

Read-sequence keys keep the existing Redis hash `value` field and 30-day write
TTL. Their dedicated monotonic fill path replaces generic RocksCache fills;
updating a value also invalidates an older RocksCache fill owner. Complete the
rollout across all writers, since old processes can still perform unconditional
writes. Other cached fields retain their existing cache behavior.

Atomic maximum updates tolerate concurrent devices, duplicate receipts, and
out-of-order persistence. They do not repair a previously lost persistence event
or guarantee delivery through an in-memory queue. Reliable retry/compensation and
reconciliation of existing Redis/MongoDB differences remain separate work. Do not
mark all messages as read to repair a discrepancy; use confirmed read progress.
3 changes: 3 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ go 1.25.0

require (
firebase.google.com/go/v4 v4.14.1
github.com/alicebob/miniredis/v2 v2.34.0
github.com/dtm-labs/rockscache v0.1.1
github.com/gin-gonic/gin v1.9.1
github.com/go-playground/validator/v10 v10.20.0
Expand Down Expand Up @@ -56,6 +57,7 @@ require (
cloud.google.com/go/longrunning v0.5.5 // indirect
cloud.google.com/go/storage v1.40.0 // indirect
github.com/MicahParks/keyfunc v1.9.0 // indirect
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect
github.com/aws/aws-sdk-go-v2 v1.32.5 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.5.1 // indirect
Expand Down Expand Up @@ -182,6 +184,7 @@ require (
github.com/xdg-go/scram v1.1.2 // indirect
github.com/xdg-go/stringprep v1.0.4 // indirect
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
github.com/yuin/gopher-lua v1.1.1 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.etcd.io/etcd/api/v3 v3.5.13 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.13 // indirect
Expand Down
6 changes: 6 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,10 @@ github.com/IBM/sarama v1.43.0/go.mod h1:zlE6HEbC/SMQ9mhEYaF7nNLYOUyrs0obySKCckWP
github.com/MicahParks/keyfunc v1.9.0 h1:lhKd5xrFHLNOWrDc4Tyb/Q1AJ4LCzQ48GVJyVIID3+o=
github.com/MicahParks/keyfunc v1.9.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw=
github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE=
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
github.com/alicebob/miniredis/v2 v2.34.0 h1:mBFWMaJSNL9RwdGRyEDoAAv8OQc5UlEhLDQggTglU/0=
github.com/alicebob/miniredis/v2 v2.34.0/go.mod h1:kWShP4b58T1CW0Y5dViCd5ztzrDqRWqM3nksiyXk5s8=
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g=
github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
github.com/aws/aws-sdk-go-v2 v1.32.5 h1:U8vdWJuY7ruAkzaOdD7guwJjD06YSKmnKCJs7s3IkIo=
Expand Down Expand Up @@ -478,6 +482,8 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
go.etcd.io/etcd/api/v3 v3.5.13 h1:8WXU2/NBge6AUF1K1gOexB6e07NgsN1hXK0rSTtgSp4=
Expand Down
33 changes: 11 additions & 22 deletions internal/rpc/msg/as_read.go
Original file line number Diff line number Diff line change
Expand Up @@ -96,10 +96,11 @@ func (m *msgServer) SetConversationHasReadSeq(ctx context.Context, req *msg.SetC
if req.HasReadSeq > maxSeq {
return nil, errs.ErrArgs.WrapMsg("hasReadSeq must not be bigger than maxSeq")
}
if err := m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq); err != nil {
hasReadSeq, err := m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq)
if err != nil {
return nil, err
}
m.sendMarkAsReadNotification(ctx, req.ConversationID, constant.SingleChatType, req.UserID, req.UserID, nil, req.HasReadSeq)
m.sendMarkAsReadNotification(ctx, req.ConversationID, constant.SingleChatType, req.UserID, req.UserID, nil, hasReadSeq)
return &msg.SetConversationHasReadSeqResp{}, nil
}

Expand All @@ -122,16 +123,10 @@ func (m *msgServer) MarkMsgsAsRead(ctx context.Context, req *msg.MarkMsgsAsReadR
if err := m.MsgDatabase.MarkSingleChatMsgsAsRead(ctx, req.UserID, req.ConversationID, req.Seqs); err != nil {
return nil, err
}
currentHasReadSeq, err := m.MsgDatabase.GetHasReadSeq(ctx, req.UserID, req.ConversationID)
if err != nil && !errors.Is(err, redis.Nil) {
hasReadSeq, err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, hasReadSeq)
if err != nil {
return nil, err
}
if hasReadSeq > currentHasReadSeq {
err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, hasReadSeq)
if err != nil {
return nil, err
}
}

reqCallback := &cbapi.CallbackSingleMsgReadReq{
ConversationID: conversation.ConversationID,
Expand Down Expand Up @@ -176,23 +171,17 @@ func (m *msgServer) MarkConversationAsRead(ctx context.Context, req *msg.MarkCon
return nil, err
}
}
if req.HasReadSeq > hasReadSeq {
err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq)
if err != nil {
return nil, err
}
hasReadSeq = req.HasReadSeq
hasReadSeq, err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq)
if err != nil {
return nil, err
}
m.sendMarkAsReadNotification(ctx, req.ConversationID, conversation.ConversationType, req.UserID,
m.conversationAndGetRecvID(conversation, req.UserID), seqs, hasReadSeq)
} else if conversation.ConversationType == constant.ReadGroupChatType ||
conversation.ConversationType == constant.NotificationChatType {
if req.HasReadSeq > hasReadSeq {
err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq)
if err != nil {
return nil, err
}
hasReadSeq = req.HasReadSeq
hasReadSeq, err = m.MsgDatabase.SetHasReadSeq(ctx, req.UserID, req.ConversationID, req.HasReadSeq)
if err != nil {
return nil, err
}
m.sendMarkAsReadNotification(ctx, req.ConversationID, constant.SingleChatType, req.UserID,
req.UserID, seqs, hasReadSeq)
Expand Down
107 changes: 107 additions & 0 deletions internal/rpc/msg/as_read_monotonic_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
package msg

import (
"context"
"encoding/json"
"fmt"
"testing"
"time"

"github.com/openimsdk/open-im-server/v3/pkg/common/config"
"github.com/openimsdk/open-im-server/v3/pkg/common/storage/controller"
"github.com/openimsdk/open-im-server/v3/pkg/common/webhook"
"github.com/openimsdk/open-im-server/v3/pkg/notification"
"github.com/openimsdk/open-im-server/v3/pkg/rpccache"
"github.com/openimsdk/open-im-server/v3/pkg/rpcli"
"github.com/openimsdk/protocol/constant"
"github.com/openimsdk/protocol/conversation"
pbmsg "github.com/openimsdk/protocol/msg"
"github.com/openimsdk/protocol/sdkws"
"github.com/openimsdk/tools/mcontext"
"github.com/openimsdk/tools/mq/memamq"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
)

type receiptDatabase struct {
controller.CommonMsgDatabase
calls int
initial int64
}

func (*receiptDatabase) GetMaxSeq(context.Context, string) (int64, error) { return 346, nil }
func (db *receiptDatabase) GetHasReadSeq(context.Context, string, string) (int64, error) {
return db.initial, nil
}
func (db *receiptDatabase) SetHasReadSeq(context.Context, string, string, int64) (int64, error) {
db.calls++
return 346, nil // Another device advanced after the RPC's initial read.
}
func (*receiptDatabase) MarkSingleChatMsgsAsRead(context.Context, string, string, []int64) error {
return nil
}

type receiptConversationClient struct {
conversation.ConversationClient
sessionType int32
}

func (c *receiptConversationClient) GetConversation(_ context.Context, req *conversation.GetConversationReq, _ ...grpc.CallOption) (*conversation.GetConversationResp, error) {
return &conversation.GetConversationResp{Conversation: &conversation.Conversation{
ConversationID: req.ConversationID, OwnerUserID: req.OwnerUserID,
ConversationType: c.sessionType, UserID: "peer", GroupID: "group",
}}, nil
}

func TestReadSeqReceiptsUseEffectiveProgress(t *testing.T) {
for _, method := range []string{"set", "messages", "single conversation", "group conversation"} {
for _, requested := range []int64{200, 346} {
for _, initial := range []int64{100, 346} {
t.Run(fmt.Sprintf("%s/initial=%d/requested=%d", method, initial, requested), func(t *testing.T) {
ctx := mcontext.WithOpUserIDContext(context.Background(), "user")
db := &receiptDatabase{initial: initial}
queue := memamq.NewMemoryQueue(1, 8)
defer queue.Stop()
sent := make(chan *sdkws.MsgData, 1)
notifier := notification.NewNotificationSender(&config.Notification{}, notification.WithQueue(queue),
notification.WithLocalSendMsg(func(_ context.Context, req *pbmsg.SendMsgReq) (*pbmsg.SendMsgResp, error) {
sent <- req.MsgData
return &pbmsg.SendMsgResp{}, nil
}))
sessionType := int32(constant.SingleChatType)
if method == "group conversation" {
sessionType = constant.ReadGroupChatType
}
client := &rpcli.ConversationClient{ConversationClient: &receiptConversationClient{sessionType: sessionType}}
server := &msgServer{MsgDatabase: db, notificationSender: notifier, config: &Config{},
ConversationLocalCache: rpccache.NewConversationLocalCache(client, &config.LocalCache{}, nil),
webhookClient: webhook.NewWebhookClient("", queue),
}
var err error
switch method {
case "set":
_, err = server.SetConversationHasReadSeq(ctx, &pbmsg.SetConversationHasReadSeqReq{UserID: "user", ConversationID: "group", HasReadSeq: requested})
case "messages":
_, err = server.MarkMsgsAsRead(ctx, &pbmsg.MarkMsgsAsReadReq{UserID: "user", ConversationID: "group", Seqs: []int64{requested}})
default:
_, err = server.MarkConversationAsRead(ctx, &pbmsg.MarkConversationAsReadReq{UserID: "user", ConversationID: "group", HasReadSeq: requested})
}
require.NoError(t, err)
require.Equal(t, 1, db.calls, "equal reports must still allow persistence/notification repair")
select {
case message := <-sent:
require.EqualValues(t, constant.HasReadReceipt, message.ContentType)
var elem sdkws.NotificationElem
require.NoError(t, json.Unmarshal(message.Content, &elem))
var tips sdkws.MarkAsReadTips
require.NoError(t, json.Unmarshal([]byte(elem.Detail), &tips))
require.EqualValues(t, 346, tips.HasReadSeq)
require.Equal(t, "group", tips.ConversationID)
case <-time.After(3 * time.Second):
t.Fatal("read receipt was not emitted")
}
})
}
}
}
}
Loading
Loading