From ebe7bbc920c5d8172c59aae0a5fc30cacecae405 Mon Sep 17 00:00:00 2001 From: Sajjad Hassanzadeh Date: Mon, 20 Jul 2026 17:43:43 +0330 Subject: [PATCH] feat: add check replica command to identify broken snapshot chains Add 'longhornctl check replica' which inspects the snapshot chains in the Longhorn replica data directories on each node and reports integrity issues: - snapshots referencing a missing parent (broken chain) - disk files without metadata files, and metadata files without disk files - missing or corrupted volume.meta and disk metadata files - a missing volume head, unexpected extra volume head files, and metadata declaring a mismatching disk name - loops in the snapshot chain The command follows the existing DaemonSet pattern: the remote command deploys privileged pods that run 'longhornctl-local check replica' against the host data directory, then aggregates the per-node JSON results into YAML. Results can be filtered with --volume-name and --name, and include the reachable snapshot chain from the volume head for context. A warning is emitted when the replica is in use, since findings may be transient while the engine is modifying the chain. Longhorn 9102 Signed-off-by: Sajjad Hassanzadeh --- cmd/local/subcmd/check.go | 56 ++++++ cmd/remote/subcmd/check.go | 83 ++++++++ docs/longhornctl_check.md | 3 +- docs/longhornctl_check_replica.md | 68 +++++++ pkg/consts/replica.go | 1 + pkg/local/replica/checker.go | 318 ++++++++++++++++++++++++++++++ pkg/local/replica/checker_test.go | 211 ++++++++++++++++++++ pkg/local/replica/getter.go | 31 ++- pkg/remote/replica/checker.go | 228 +++++++++++++++++++++ pkg/types/replica.go | 16 ++ 10 files changed, 997 insertions(+), 18 deletions(-) create mode 100644 docs/longhornctl_check_replica.md create mode 100644 pkg/local/replica/checker.go create mode 100644 pkg/local/replica/checker_test.go create mode 100644 pkg/remote/replica/checker.go diff --git a/cmd/local/subcmd/check.go b/cmd/local/subcmd/check.go index 68939874..3e98697b 100644 --- a/cmd/local/subcmd/check.go +++ b/cmd/local/subcmd/check.go @@ -9,6 +9,7 @@ import ( "github.com/longhorn/cli/pkg/consts" local "github.com/longhorn/cli/pkg/local/preflight" + localreplica "github.com/longhorn/cli/pkg/local/replica" "github.com/longhorn/cli/pkg/types" "github.com/longhorn/cli/pkg/utils" ) @@ -22,6 +23,61 @@ func NewCmdCheck(globalOpts *types.GlobalCmdOptions) *cobra.Command { utils.SetGlobalOptionsLocal(cmd, globalOpts) cmd.AddCommand(newCmdCheckPreflight(globalOpts)) + cmd.AddCommand(newCmdCheckReplica(globalOpts)) + + return cmd +} + +func newCmdCheckReplica(globalOpts *types.GlobalCmdOptions) *cobra.Command { + var localChecker = localreplica.Checker{} + + cmd := &cobra.Command{ + Use: consts.SubCmdReplica, + Short: "Check Longhorn replica integrity", + Long: `This command checks the integrity of the snapshot chains in the Longhorn replica data directories. +It identifies broken snapshot chains, for example snapshots referencing a missing parent, disk files without metadata files, and metadata files without disk files. +The results are presented by the replica data directory names, not the actual Custom Resource (CR) names. + +By default, this command checks all Longhorn replicas in the data directory. +You can narrow down the results by using the following options: +- --name: Specify the Longhorn replica data directory name to check a specific replica. +- --volume-name: Filter replicas by the volume they belong to.`, + + PreRun: func(cmd *cobra.Command, args []string) { + localChecker.LogLevel = globalOpts.LogLevel + + err := localChecker.Init() + if err != nil { + utils.CheckErr(errors.Wrap(err, "Failed to initialize replica checker")) + } + }, + + Run: func(cmd *cobra.Command, args []string) { + err := localChecker.Run() + if err != nil { + utils.CheckErr(errors.Wrap(err, "Failed to run replica checker")) + } + + logrus.Info("Successfully checked replica integrity") + }, + + PostRun: func(cmd *cobra.Command, args []string) { + err := localChecker.Output() + if err != nil { + utils.CheckErr(errors.Wrap(err, "Failed to output replica checker collection")) + } + + logrus.Info("Successfully output replica checker collection") + }, + } + + utils.SetGlobalOptionsLocal(cmd, globalOpts) + + cmd.Flags().StringVar(&localChecker.CurrentNodeID, consts.CmdOptNodeId, os.Getenv(consts.EnvCurrentNodeID), "Current node ID.") + cmd.Flags().StringVarP(&localChecker.OutputFilePath, consts.CmdOptOutputFile, "o", os.Getenv(consts.EnvOutputFilePath), "Output the result to a file, default to stdout.") + cmd.Flags().StringVar(&localChecker.ReplicaName, consts.CmdOptName, os.Getenv(consts.EnvLonghornReplicaName), "Specify the name of the replica to check.") + cmd.Flags().StringVar(&localChecker.VolumeName, consts.CmdOptLonghornVolumeName, os.Getenv(consts.EnvLonghornVolumeName), "Specify the name of the volume to check its replicas.") + cmd.Flags().StringVar(&localChecker.LonghornDataDirectory, consts.CmdOptLonghornDataDirectory, os.Getenv(consts.EnvLonghornDataDirectory), "Specify the Longhorn data directory. If not provided, the default will be attempted, or it will fall back to the directory of longhorn-disk.cfg.") return cmd } diff --git a/cmd/remote/subcmd/check.go b/cmd/remote/subcmd/check.go index 51235251..aafc9f56 100644 --- a/cmd/remote/subcmd/check.go +++ b/cmd/remote/subcmd/check.go @@ -7,6 +7,7 @@ import ( "github.com/longhorn/cli/pkg/consts" "github.com/longhorn/cli/pkg/remote/preflight" + "github.com/longhorn/cli/pkg/remote/replica" "github.com/longhorn/cli/pkg/types" "github.com/longhorn/cli/pkg/utils" ) @@ -20,6 +21,88 @@ func NewCmdCheck(globalOpts *types.GlobalCmdOptions) *cobra.Command { utils.SetGlobalOptionsRemote(cmd, globalOpts) cmd.AddCommand(newCmdCheckPreflight(globalOpts)) + cmd.AddCommand(newCmdCheckReplica(globalOpts)) + + return cmd +} + +func newCmdCheckReplica(globalOpts *types.GlobalCmdOptions) *cobra.Command { + var replicaChecker = replica.Checker{} + + cmd := &cobra.Command{ + Use: consts.SubCmdReplica, + Short: "Check Longhorn replica integrity", + Long: `This command checks the integrity of the snapshot chains in the Longhorn replica data directories on each node. +It identifies broken snapshot chains, for example snapshots referencing a missing parent, disk files without metadata files, and metadata files without disk files. +The results are presented by the replica data directory names, not the actual Custom Resource (CR) names. + +By default, this command checks all Longhorn replicas in the data directory. +You can narrow down the results by using the following options: +- --name: Specify the Longhorn replica data directory name to check a specific replica. +- --volume-name: Filter replicas by the volume they belong to.`, + Example: `$ longhornctl check replica +INFO[2024-07-16T17:23:47+08:00] Initializing replica checker +INFO[2024-07-16T17:23:47+08:00] Cleaning up replica checker +INFO[2024-07-16T17:23:47+08:00] Running replica checker +INFO[2024-07-16T17:23:51+08:00] Retrieved replica check results: + replicas: + pvc-48a6457d-585e-423b-b530-bbc68a5f948a-0e2603a7: + - node: ip-10-0-2-123 + directory: /var/lib/longhorn/replicas/pvc-48a6457d-585e-423b-b530-bbc68a5f948a-0e2603a7 + volumeName: pvc-48a6457d-585e-423b-b530-bbc68a5f948a + snapshotChain: + - volume-head-001.img + - volume-snap-40b3b028-b3b3-4a35-a806-8bea77f27c00.img + errors: + - 'broken snapshot chain: disk volume-snap-40b3b028-b3b3-4a35-a806-8bea77f27c00.img references parent volume-snap-6f244bbe-2857-46e4-92e2-eb1e16a63ba1.img, but the parent metadata file is missing' +INFO[2024-07-16T17:23:51+08:00] Cleaning up replica checker +INFO[2024-07-16T17:23:51+08:00] Completed replica checker`, + + PreRun: func(cmd *cobra.Command, args []string) { + replicaChecker.Image = globalOpts.Image + replicaChecker.ImageRegistry = globalOpts.ImageRegistry + replicaChecker.ImagePullSecret = globalOpts.ImagePullSecret + replicaChecker.KubeConfigPath = globalOpts.KubeConfigPath + replicaChecker.NodeSelector = globalOpts.NodeSelector + replicaChecker.Tolerations = globalOpts.Tolerations + replicaChecker.Namespace = globalOpts.Namespace + + logrus.Info("Initializing replica checker") + if err := replicaChecker.Init(); err != nil { + utils.CheckErr(errors.Wrap(err, "Failed to initialize replica checker")) + } + + logrus.Info("Cleaning up replica checker") + if err := replicaChecker.Cleanup(); err != nil { + utils.CheckErr(errors.Wrapf(err, "Failed to cleanup replica checker")) + } + }, + + Run: func(cmd *cobra.Command, args []string) { + logrus.Info("Running replica checker") + output, err := replicaChecker.Run() + if err != nil { + utils.CheckErr(errors.Wrap(err, "Failed to run replica checker")) + } + + logrus.Infof("Retrieved replica check results:\n %v", output) + }, + + PostRun: func(cmd *cobra.Command, args []string) { + logrus.Info("Cleaning up replica checker") + if err := replicaChecker.Cleanup(); err != nil { + utils.CheckErr(errors.Wrapf(err, "Failed to cleanup replica checker")) + } + + logrus.Info("Completed replica checker") + }, + } + + utils.SetGlobalOptionsRemote(cmd, globalOpts) + + cmd.Flags().StringVar(&replicaChecker.ReplicaName, consts.CmdOptName, "", "Specify the name of the replica to check.") + cmd.Flags().StringVar(&replicaChecker.VolumeName, consts.CmdOptLonghornVolumeName, "", "Specify the name of the volume to check its replicas.") + cmd.Flags().StringVar(&replicaChecker.LonghornDataDirectory, consts.CmdOptLonghornDataDirectory, "/var/lib/longhorn", "Specify the Longhorn data directory. If not provided, the default will be attempted, or it will fall back to the directory of longhorn-disk.cfg.") return cmd } diff --git a/docs/longhornctl_check.md b/docs/longhornctl_check.md index d481efb5..683b33f5 100644 --- a/docs/longhornctl_check.md +++ b/docs/longhornctl_check.md @@ -25,5 +25,6 @@ Longhorn checking operations * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl check preflight](longhornctl_check_preflight.md) - Run a preflight check for Longhorn +* [longhornctl check replica](longhornctl_check_replica.md) - Check Longhorn replica integrity -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 20-Jul-2026 diff --git a/docs/longhornctl_check_replica.md b/docs/longhornctl_check_replica.md new file mode 100644 index 00000000..4a414bd9 --- /dev/null +++ b/docs/longhornctl_check_replica.md @@ -0,0 +1,68 @@ +## longhornctl check replica + +Check Longhorn replica integrity + +### Synopsis + +This command checks the integrity of the snapshot chains in the Longhorn replica data directories on each node. +It identifies broken snapshot chains, for example snapshots referencing a missing parent, disk files without metadata files, and metadata files without disk files. +The results are presented by the replica data directory names, not the actual Custom Resource (CR) names. + +By default, this command checks all Longhorn replicas in the data directory. +You can narrow down the results by using the following options: +- --name: Specify the Longhorn replica data directory name to check a specific replica. +- --volume-name: Filter replicas by the volume they belong to. + +``` +longhornctl check replica [flags] +``` + +### Examples + +``` +$ longhornctl check replica +INFO[2024-07-16T17:23:47+08:00] Initializing replica checker +INFO[2024-07-16T17:23:47+08:00] Cleaning up replica checker +INFO[2024-07-16T17:23:47+08:00] Running replica checker +INFO[2024-07-16T17:23:51+08:00] Retrieved replica check results: + replicas: + pvc-48a6457d-585e-423b-b530-bbc68a5f948a-0e2603a7: + - node: ip-10-0-2-123 + directory: /var/lib/longhorn/replicas/pvc-48a6457d-585e-423b-b530-bbc68a5f948a-0e2603a7 + volumeName: pvc-48a6457d-585e-423b-b530-bbc68a5f948a + snapshotChain: + - volume-head-001.img + - volume-snap-40b3b028-b3b3-4a35-a806-8bea77f27c00.img + errors: + - 'broken snapshot chain: disk volume-snap-40b3b028-b3b3-4a35-a806-8bea77f27c00.img references parent volume-snap-6f244bbe-2857-46e4-92e2-eb1e16a63ba1.img, but the parent metadata file is missing' +INFO[2024-07-16T17:23:51+08:00] Cleaning up replica checker +INFO[2024-07-16T17:23:51+08:00] Completed replica checker +``` + +### Options + +``` + --data-dir string Specify the Longhorn data directory. If not provided, the default will be attempted, or it will fall back to the directory of longhorn-disk.cfg. (default "/var/lib/longhorn") + -h, --help help for replica + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image-pull-secret string Secret with registry credentials for pulling images + --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. + --kubeconfig string Kubernetes config (kubeconfig) path + -l, --log-level string Log level (default "info") + --name string Specify the name of the replica to check. + --namespace string The namespace to run DaemonSet pods. (default "longhorn-system") + --node-selector string Comma-separated list of key=value pairs to match against node labels, selecting the nodes the DaemonSet will run on (e.g. env=prod,zone=us-west). + --volume-name string Specify the name of the volume to check its replicas. +``` + +### Options inherited from parent commands + +``` + --tolerations string Semicolon-separated list of tolerations for DaemonSet pods (e.g. key=value:NoSchedule;:NoExecute). +``` + +### SEE ALSO + +* [longhornctl check](longhornctl_check.md) - Longhorn checking operations + +###### Auto generated by spf13/cobra on 20-Jul-2026 diff --git a/pkg/consts/replica.go b/pkg/consts/replica.go index d12f11de..a54d1b94 100644 --- a/pkg/consts/replica.go +++ b/pkg/consts/replica.go @@ -1,6 +1,7 @@ package consts const ( + AppNameReplicaChecker = "longhorn-replica-checker" AppNameReplicaExporter = "longhorn-replica-exporter" AppNameReplicaGetter = "longhorn-replica-getter" ) diff --git a/pkg/local/replica/checker.go b/pkg/local/replica/checker.go new file mode 100644 index 00000000..b5862fdd --- /dev/null +++ b/pkg/local/replica/checker.go @@ -0,0 +1,318 @@ +package replica + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + lhmgrutil "github.com/longhorn/longhorn-manager/util" + + commonio "github.com/longhorn/go-common-libs/io" + + "github.com/longhorn/cli/pkg/consts" + remote "github.com/longhorn/cli/pkg/remote/replica" + "github.com/longhorn/cli/pkg/types" + "github.com/longhorn/cli/pkg/utils" + utilslonghorn "github.com/longhorn/cli/pkg/utils/longhorn" +) + +const ( + diskImageSuffix = ".img" + diskMetadataSuffix = ".img.meta" + + volumeHeadPrefix = "volume-head-" + volumeSnapshotPrefix = "volume-snap-" + volumeMetadataFile = "volume.meta" +) + +// diskMetadata mirrors the metadata the Longhorn engine persists next to each +// disk file (volume head or snapshot) in the replica data directory. +type diskMetadata struct { + Name string + Parent string + Removed bool + UserCreated bool + Created string + Labels map[string]string +} + +// Checker provide functions for the replica checker. +type Checker struct { + remote.CheckerCmdOptions + + logger *logrus.Entry + + OutputFilePath string + CurrentNodeID string + + replicasDirectory string + replicaNames []string + + collection types.ReplicaCheckCollection +} + +// Init initializes the Checker. +func (local *Checker) Init() error { + var err error + + if len(local.OutputFilePath) != 0 { + local.logger = logrus.WithField("output", local.OutputFilePath) + } else { + local.logger = logrus.WithField("output", "stdout") + } + + local.LonghornDataDirectory, err = utilslonghorn.GetDataDirectory(local.logger, consts.VolumeMountHostDirectory, local.LonghornDataDirectory) + if err != nil { + return errors.Wrap(err, "failed to get Longhorn data directory") + } + + local.logger = local.logger.WithField("data-dir", local.LonghornDataDirectory) + + local.replicasDirectory = filepath.Join(local.LonghornDataDirectory, "replicas") + + local.collection.Replicas = make(map[string][]*types.ReplicaCheckInfo) + + return nil +} + +// Run checks the snapshot chain integrity of the replicas in the data directory. +func (local *Checker) Run() error { + var err error + + log := local.logger + if local.VolumeName != "" { + log = log.WithField("volume", local.VolumeName) + } + if local.ReplicaName != "" { + log = log.WithField("replica", local.ReplicaName) + } + + local.replicaNames, err = getReplicaNamesInDirectory(log, local.replicasDirectory, local.VolumeName, local.ReplicaName) + if err != nil { + return err + } + + for _, replicaName := range local.replicaNames { + replicaCheckInfo, err := local.checkReplica(replicaName) + if err != nil { + return errors.Wrapf(err, "failed to check replica %s", replicaName) + } + + if replicaCheckInfo == nil { + continue + } + + local.collection.Replicas[replicaName] = append(local.collection.Replicas[replicaName], replicaCheckInfo) + } + + return nil +} + +// Output converts the collection to JSON and output to stdout or the output file. +func (local *Checker) Output() error { + local.logger.Tracef("Outputting replica checker results") + + jsonBytes, err := json.Marshal(local.collection) + if err != nil { + return errors.Wrap(err, "failed to convert replica check collections to JSON") + } + + return utils.HandleResult(jsonBytes, local.OutputFilePath, local.logger) +} + +// checkReplica checks the snapshot chain integrity of a single replica directory. +func (local *Checker) checkReplica(replicaName string) (*types.ReplicaCheckInfo, error) { + log := local.logger + + log.Infof("Checking snapshot chain for replica %s", replicaName) + + replicaCheckInfo := &types.ReplicaCheckInfo{} + replicaCheckInfo.Node = local.CurrentNodeID + replicaCheckInfo.VolumeName = replicaName[:strings.LastIndex(replicaName, "-")] + + replicaDirectory := filepath.Join(local.replicasDirectory, replicaName) + replicaCheckInfo.Directory = strings.TrimPrefix(replicaDirectory, consts.VolumeMountHostDirectory) + + isEmpty, err := commonio.IsDirectoryEmpty(replicaDirectory) + if err != nil { + replicaCheckInfo.Errors = append(replicaCheckInfo.Errors, errors.Wrapf(err, "failed to check if directory %s is empty", replicaCheckInfo.Directory).Error()) + return replicaCheckInfo, nil + } + + if isEmpty { + log.Warnf("Replica directory %s is empty", replicaCheckInfo.Directory) + replicaCheckInfo.Warnings = append(replicaCheckInfo.Warnings, "replica directory is empty") + return replicaCheckInfo, nil + } + + isReplicaInUse, err := isReplicaDirectoryInUse(replicaDirectory) + if err != nil { + replicaCheckInfo.Warnings = append(replicaCheckInfo.Warnings, errors.Wrapf(err, "failed to check if replica %s is in use", replicaName).Error()) + } else if isReplicaInUse { + replicaCheckInfo.Warnings = append(replicaCheckInfo.Warnings, "replica is in use; findings may be transient while the engine is modifying the snapshot chain") + } + + chain, checkErrors, warnings := validateSnapshotChain(replicaDirectory) + replicaCheckInfo.SnapshotChain = chain + replicaCheckInfo.Errors = append(replicaCheckInfo.Errors, checkErrors...) + replicaCheckInfo.Warnings = append(replicaCheckInfo.Warnings, warnings...) + + return replicaCheckInfo, nil +} + +// validateSnapshotChain inspects the disk files in the given replica directory +// and returns the volume head chain (from the head to the root), along with any +// integrity errors and warnings found. +func validateSnapshotChain(replicaDirectory string) (chain []string, checkErrors []string, warnings []string) { + volumeMeta := &lhmgrutil.VolumeMeta{} + content, err := os.ReadFile(filepath.Join(replicaDirectory, volumeMetadataFile)) + if err != nil { + checkErrors = append(checkErrors, fmt.Sprintf("failed to read %s: %v", volumeMetadataFile, err)) + return nil, checkErrors, warnings + } + if err := json.Unmarshal(content, volumeMeta); err != nil { + checkErrors = append(checkErrors, fmt.Sprintf("failed to parse %s: %v", volumeMetadataFile, err)) + return nil, checkErrors, warnings + } + + if volumeMeta.Error != "" { + checkErrors = append(checkErrors, fmt.Sprintf("%s records an error: %s", volumeMetadataFile, volumeMeta.Error)) + } + if volumeMeta.Rebuilding { + warnings = append(warnings, "replica is marked as rebuilding; the snapshot chain may be incomplete until the rebuild finishes") + } + + disks, images, headImages, diskReadErrors := readDiskMetadataFiles(replicaDirectory) + checkErrors = append(checkErrors, diskReadErrors...) + + // Every disk file must have a metadata file, and vice versa. + for _, diskName := range sortedKeys(disks) { + if !images[diskName] { + checkErrors = append(checkErrors, fmt.Sprintf("disk metadata %s%s exists, but disk file %s is missing", diskName, ".meta", diskName)) + } + if metaName := disks[diskName].Name; metaName != "" && metaName != diskName { + checkErrors = append(checkErrors, fmt.Sprintf("disk metadata %s%s declares mismatching disk name %s", diskName, ".meta", metaName)) + } + } + for _, imageName := range sortedKeys(images) { + if _, ok := disks[imageName]; !ok { + checkErrors = append(checkErrors, fmt.Sprintf("disk file %s exists, but its metadata file %s%s is missing", imageName, imageName, ".meta")) + } + } + + // Every parent reference must resolve, including the ones on snapshot tree + // branches that are not part of the volume head chain. + for _, diskName := range sortedKeys(disks) { + parent := disks[diskName].Parent + if parent == "" { + continue + } + if _, ok := disks[parent]; !ok { + checkErrors = append(checkErrors, fmt.Sprintf("broken snapshot chain: disk %s references parent %s, but the parent metadata file is missing", diskName, parent)) + } + } + + head := volumeMeta.Head + if head == "" { + checkErrors = append(checkErrors, fmt.Sprintf("%s does not specify a volume head", volumeMetadataFile)) + return nil, checkErrors, warnings + } + + if _, ok := disks[head]; !ok { + checkErrors = append(checkErrors, fmt.Sprintf("broken snapshot chain: volume head %s declared in %s is missing its metadata file", head, volumeMetadataFile)) + } + if !images[head] { + checkErrors = append(checkErrors, fmt.Sprintf("broken snapshot chain: volume head file %s declared in %s is missing", head, volumeMetadataFile)) + } + + sort.Strings(headImages) + for _, headImage := range headImages { + if headImage != head { + warnings = append(warnings, fmt.Sprintf("found unexpected volume head file %s; the active volume head is %s", headImage, head)) + } + } + + // Walk the chain from the volume head to the root. Dangling parents and + // missing files are already reported above, so the walk only needs to + // detect loops and record the reachable chain. + visited := map[string]bool{} + current := head + for current != "" { + if visited[current] { + checkErrors = append(checkErrors, fmt.Sprintf("snapshot chain contains a loop at disk %s", current)) + break + } + visited[current] = true + + diskMeta, ok := disks[current] + if !ok { + break + } + + chain = append(chain, current) + current = diskMeta.Parent + } + + return chain, checkErrors, warnings +} + +// readDiskMetadataFiles reads the disk files in the given replica directory and +// returns the parsed disk metadata, the set of disk image files, and the volume +// head image files found. +func readDiskMetadataFiles(replicaDirectory string) (disks map[string]*diskMetadata, images map[string]bool, headImages []string, checkErrors []string) { + disks = map[string]*diskMetadata{} + images = map[string]bool{} + + entries, err := os.ReadDir(replicaDirectory) + if err != nil { + checkErrors = append(checkErrors, fmt.Sprintf("failed to list replica directory: %v", err)) + return disks, images, headImages, checkErrors + } + + for _, entry := range entries { + name := entry.Name() + if !strings.HasPrefix(name, volumeHeadPrefix) && !strings.HasPrefix(name, volumeSnapshotPrefix) { + continue + } + + switch { + case strings.HasSuffix(name, diskMetadataSuffix): + content, err := os.ReadFile(filepath.Join(replicaDirectory, name)) + if err != nil { + checkErrors = append(checkErrors, fmt.Sprintf("failed to read disk metadata %s: %v", name, err)) + continue + } + + diskMeta := &diskMetadata{} + if err := json.Unmarshal(content, diskMeta); err != nil { + checkErrors = append(checkErrors, fmt.Sprintf("failed to parse disk metadata %s: %v", name, err)) + continue + } + + disks[strings.TrimSuffix(name, ".meta")] = diskMeta + + case strings.HasSuffix(name, diskImageSuffix): + images[name] = true + if strings.HasPrefix(name, volumeHeadPrefix) { + headImages = append(headImages, name) + } + } + } + + return disks, images, headImages, checkErrors +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} diff --git a/pkg/local/replica/checker_test.go b/pkg/local/replica/checker_test.go new file mode 100644 index 00000000..ae3cba8b --- /dev/null +++ b/pkg/local/replica/checker_test.go @@ -0,0 +1,211 @@ +package replica + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/suite" + + lhmgrutil "github.com/longhorn/longhorn-manager/util" +) + +type CheckerTestSuite struct { + suite.Suite + + replicaDirectory string +} + +func (s *CheckerTestSuite) SetupTest() { + s.replicaDirectory = s.T().TempDir() +} + +func (s *CheckerTestSuite) writeVolumeMeta(head string) { + volumeMeta := &lhmgrutil.VolumeMeta{ + Size: 1073741824, + Head: head, + SectorSize: 512, + } + content, err := json.Marshal(volumeMeta) + s.Require().NoError(err) + s.Require().NoError(os.WriteFile(filepath.Join(s.replicaDirectory, volumeMetadataFile), content, 0600)) +} + +func (s *CheckerTestSuite) writeDiskMeta(diskName, parent string) { + diskMeta := &diskMetadata{ + Name: diskName, + Parent: parent, + } + content, err := json.Marshal(diskMeta) + s.Require().NoError(err) + s.Require().NoError(os.WriteFile(filepath.Join(s.replicaDirectory, diskName+".meta"), content, 0600)) +} + +func (s *CheckerTestSuite) writeDiskImage(diskName string) { + s.Require().NoError(os.WriteFile(filepath.Join(s.replicaDirectory, diskName), []byte{}, 0600)) +} + +func (s *CheckerTestSuite) writeDisk(diskName, parent string) { + s.writeDiskImage(diskName) + s.writeDiskMeta(diskName, parent) +} + +func (s *CheckerTestSuite) TestHealthyChain() { + s.writeVolumeMeta("volume-head-001.img") + s.writeDisk("volume-snap-a.img", "") + s.writeDisk("volume-snap-b.img", "volume-snap-a.img") + s.writeDisk("volume-head-001.img", "volume-snap-b.img") + + chain, checkErrors, warnings := validateSnapshotChain(s.replicaDirectory) + + s.Empty(checkErrors) + s.Empty(warnings) + s.Equal([]string{"volume-head-001.img", "volume-snap-b.img", "volume-snap-a.img"}, chain) +} + +func (s *CheckerTestSuite) TestHealthySnapshotTree() { + // A snapshot tree with a branch (for example after a revert) is not broken: + // both branches share the root, and the head is on one of them. + s.writeVolumeMeta("volume-head-002.img") + s.writeDisk("volume-snap-root.img", "") + s.writeDisk("volume-snap-branch.img", "volume-snap-root.img") + s.writeDisk("volume-head-002.img", "volume-snap-root.img") + + chain, checkErrors, _ := validateSnapshotChain(s.replicaDirectory) + + s.Empty(checkErrors) + s.Equal([]string{"volume-head-002.img", "volume-snap-root.img"}, chain) +} + +func (s *CheckerTestSuite) TestMissingParent() { + s.writeVolumeMeta("volume-head-001.img") + s.writeDisk("volume-snap-b.img", "volume-snap-a.img") + s.writeDisk("volume-head-001.img", "volume-snap-b.img") + + chain, checkErrors, _ := validateSnapshotChain(s.replicaDirectory) + + s.Len(checkErrors, 1) + s.Contains(checkErrors[0], "broken snapshot chain") + s.Contains(checkErrors[0], "volume-snap-a.img") + s.Equal([]string{"volume-head-001.img", "volume-snap-b.img"}, chain) +} + +func (s *CheckerTestSuite) TestMissingDiskFile() { + s.writeVolumeMeta("volume-head-001.img") + s.writeDisk("volume-snap-a.img", "") + s.writeDiskMeta("volume-snap-b.img", "volume-snap-a.img") + s.writeDisk("volume-head-001.img", "volume-snap-b.img") + + _, checkErrors, _ := validateSnapshotChain(s.replicaDirectory) + + s.Len(checkErrors, 1) + s.Contains(checkErrors[0], "disk file volume-snap-b.img is missing") +} + +func (s *CheckerTestSuite) TestMissingMetadataFile() { + s.writeVolumeMeta("volume-head-001.img") + s.writeDisk("volume-snap-a.img", "") + s.writeDiskImage("volume-snap-b.img") + s.writeDisk("volume-head-001.img", "volume-snap-b.img") + + chain, checkErrors, _ := validateSnapshotChain(s.replicaDirectory) + + s.Len(checkErrors, 2) + s.Contains(checkErrors[0], "metadata file volume-snap-b.img.meta is missing") + s.Contains(checkErrors[1], "broken snapshot chain") + // The chain walk stops at the disk with the missing metadata. + s.Equal([]string{"volume-head-001.img"}, chain) +} + +func (s *CheckerTestSuite) TestMissingVolumeHead() { + s.writeVolumeMeta("volume-head-001.img") + s.writeDisk("volume-snap-a.img", "") + + chain, checkErrors, _ := validateSnapshotChain(s.replicaDirectory) + + s.Len(checkErrors, 2) + s.Contains(checkErrors[0], "volume head volume-head-001.img") + s.Contains(checkErrors[1], "volume head file volume-head-001.img") + s.Empty(chain) +} + +func (s *CheckerTestSuite) TestChainLoop() { + s.writeVolumeMeta("volume-head-001.img") + s.writeDisk("volume-snap-a.img", "volume-snap-b.img") + s.writeDisk("volume-snap-b.img", "volume-snap-a.img") + s.writeDisk("volume-head-001.img", "volume-snap-a.img") + + _, checkErrors, _ := validateSnapshotChain(s.replicaDirectory) + + s.Len(checkErrors, 1) + s.Contains(checkErrors[0], "loop") +} + +func (s *CheckerTestSuite) TestExtraVolumeHead() { + s.writeVolumeMeta("volume-head-002.img") + s.writeDisk("volume-snap-a.img", "") + s.writeDisk("volume-head-002.img", "volume-snap-a.img") + s.writeDisk("volume-head-001.img", "volume-snap-a.img") + + _, checkErrors, warnings := validateSnapshotChain(s.replicaDirectory) + + s.Empty(checkErrors) + s.Len(warnings, 1) + s.Contains(warnings[0], "unexpected volume head file volume-head-001.img") +} + +func (s *CheckerTestSuite) TestCorruptedDiskMetadata() { + s.writeVolumeMeta("volume-head-001.img") + s.writeDisk("volume-snap-a.img", "") + s.writeDiskImage("volume-snap-b.img") + s.Require().NoError(os.WriteFile(filepath.Join(s.replicaDirectory, "volume-snap-b.img.meta"), []byte("{invalid json"), 0600)) + s.writeDisk("volume-head-001.img", "volume-snap-b.img") + + _, checkErrors, _ := validateSnapshotChain(s.replicaDirectory) + + s.NotEmpty(checkErrors) + s.Contains(checkErrors[0], "failed to parse disk metadata volume-snap-b.img.meta") +} + +func (s *CheckerTestSuite) TestMetadataNameMismatch() { + s.writeVolumeMeta("volume-head-001.img") + s.writeDiskImage("volume-snap-a.img") + diskMeta := &diskMetadata{Name: "volume-snap-other.img"} + content, err := json.Marshal(diskMeta) + s.Require().NoError(err) + s.Require().NoError(os.WriteFile(filepath.Join(s.replicaDirectory, "volume-snap-a.img.meta"), content, 0600)) + s.writeDisk("volume-head-001.img", "volume-snap-a.img") + + _, checkErrors, _ := validateSnapshotChain(s.replicaDirectory) + + s.Len(checkErrors, 1) + s.Contains(checkErrors[0], "mismatching disk name") +} + +func (s *CheckerTestSuite) TestMissingVolumeMeta() { + s.writeDisk("volume-snap-a.img", "") + + chain, checkErrors, _ := validateSnapshotChain(s.replicaDirectory) + + s.Len(checkErrors, 1) + s.Contains(checkErrors[0], "failed to read volume.meta") + s.Empty(chain) +} + +func (s *CheckerTestSuite) TestIgnoresChecksumAndUnrelatedFiles() { + s.writeVolumeMeta("volume-head-001.img") + s.writeDisk("volume-snap-a.img", "") + s.writeDisk("volume-head-001.img", "volume-snap-a.img") + s.Require().NoError(os.WriteFile(filepath.Join(s.replicaDirectory, "volume-snap-a.img.checksum"), []byte("{}"), 0600)) + s.Require().NoError(os.WriteFile(filepath.Join(s.replicaDirectory, "revision.counter"), []byte{}, 0600)) + + _, checkErrors, warnings := validateSnapshotChain(s.replicaDirectory) + + s.Empty(checkErrors) + s.Empty(warnings) +} + +func TestCheckerTestSuite(t *testing.T) { + suite.Run(t, new(CheckerTestSuite)) +} diff --git a/pkg/local/replica/getter.go b/pkg/local/replica/getter.go index cdd756dd..0390c373 100644 --- a/pkg/local/replica/getter.go +++ b/pkg/local/replica/getter.go @@ -64,7 +64,15 @@ func (local *Getter) Init() error { func (local *Getter) Run() error { var err error - local.replicaNames, err = local.getReplicaNamesInDirectory() + log := local.logger + if local.VolumeName != "" { + log = log.WithField("volume", local.VolumeName) + } + if local.ReplicaName != "" { + log = log.WithField("replica", local.ReplicaName) + } + + local.replicaNames, err = getReplicaNamesInDirectory(log, local.replicasDirectory, local.VolumeName, local.ReplicaName) if err != nil { return err } @@ -99,16 +107,7 @@ func (local *Getter) Output() error { // getReplicaNamesInDirectory returns a list of replica names in the given directory that match the given volume name. // If the volume name is empty, it returns all replica names in the given directory. -func (local *Getter) getReplicaNamesInDirectory() ([]string, error) { - log := local.logger - replicasDirectory := local.replicasDirectory - - if local.VolumeName != "" { - log = log.WithField("volume", local.VolumeName) - } - if local.ReplicaName != "" { - log = log.WithField("replica", local.ReplicaName) - } +func getReplicaNamesInDirectory(log *logrus.Entry, replicasDirectory, volumeNameFilter, replicaNameFilter string) ([]string, error) { log.Infof("Searching for replicas in %s", replicasDirectory) filePaths, err := commonio.FindFiles(replicasDirectory, "", 1) @@ -135,12 +134,12 @@ func (local *Getter) getReplicaNamesInDirectory() ([]string, error) { continue } - if local.ReplicaName != "" && local.ReplicaName != replicaName { + if replicaNameFilter != "" && replicaNameFilter != replicaName { continue } volumeName := replicaName[:strings.LastIndex(replicaName, "-")] - if local.VolumeName != "" && local.VolumeName != volumeName { + if volumeNameFilter != "" && volumeNameFilter != volumeName { continue } @@ -179,7 +178,7 @@ func (local *Getter) getReplicaInfo(replicaName string) (replicaInfo *types.Repl return replicaInfo, nil } - isReplicaInUse, err := local.isReplicaInUse(replicaName) + isReplicaInUse, err := isReplicaDirectoryInUse(replicaDirectory) if err != nil { replicaInfo.Error = errors.Wrapf(err, "failed to check if replica %s is in use", replicaName).Error() return replicaInfo, nil @@ -189,9 +188,7 @@ func (local *Getter) getReplicaInfo(replicaName string) (replicaInfo *types.Repl return replicaInfo, nil } -func (local *Getter) isReplicaInUse(name string) (bool, error) { - replicaDirectory := filepath.Join(local.replicasDirectory, name) - +func isReplicaDirectoryInUse(replicaDirectory string) (bool, error) { // Check if replica path exists if _, err := os.Stat(replicaDirectory); os.IsNotExist(err) { return false, errors.Wrapf(err, "replica directory %s does not exist", replicaDirectory) diff --git a/pkg/remote/replica/checker.go b/pkg/remote/replica/checker.go new file mode 100644 index 00000000..b2b88a86 --- /dev/null +++ b/pkg/remote/replica/checker.go @@ -0,0 +1,228 @@ +package replica + +import ( + "encoding/json" + "path/filepath" + + "gopkg.in/yaml.v3" + + "k8s.io/utils/ptr" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeclient "k8s.io/client-go/kubernetes" + + commonkube "github.com/longhorn/go-common-libs/kubernetes" + + "github.com/longhorn/cli/pkg/consts" + "github.com/longhorn/cli/pkg/types" + "github.com/longhorn/cli/pkg/utils" + + kubeutils "github.com/longhorn/cli/pkg/utils/kubernetes" +) + +// Checker provide functions for the replica checker. +type Checker struct { + CheckerCmdOptions + + kubeClient *kubeclient.Clientset + + appName string // App name of the DaemonSet. +} + +// CheckerCmdOptions holds the options for the command. +type CheckerCmdOptions struct { + types.GlobalCmdOptions + + LonghornDataDirectory string + VolumeName string + ReplicaName string +} + +// Init initializes the Checker. +func (remote *Checker) Init() error { + kubeClient, err := kubeutils.NewKubeClient("", remote.KubeConfigPath) + if err != nil { + return err + } + remote.kubeClient = kubeClient + + remote.appName = consts.AppNameReplicaChecker + + return nil +} + +// Run creates the DaemonSet for the replica checker. It ensures that the +// init container and the output container completes before collecting the +// replica check results and returning them as a YAML string. +func (remote *Checker) Run() (string, error) { + newDaemonSet, err := kubeutils.PrepareDaemonSet(remote.newDaemonSet(), remote.kubeClient, remote.NodeSelector, remote.ImagePullSecret, remote.Tolerations) + if err != nil { + return "", err + } + + daemonSet, err := commonkube.CreateDaemonSet(remote.kubeClient, newDaemonSet) + if err != nil { + return "", err + } + + err = kubeutils.MonitorDaemonSetContainer(remote.kubeClient, daemonSet, consts.ContainerNameInit, kubeutils.WaitForDaemonSetContainersExit, ptr.To(consts.ContainerConditionMaxTolerationMedium)) + if err != nil { + return "", err + } + + err = kubeutils.MonitorDaemonSetContainer(remote.kubeClient, daemonSet, consts.ContainerNameOutput, kubeutils.WaitForDaemonSetContainersExit, ptr.To(consts.ContainerConditionMaxTolerationShort)) + if err != nil { + return "", err + } + + podCollections, err := kubeutils.GetDaemonSetPodCollections(remote.kubeClient, daemonSet, consts.ContainerNameOutput, false, false, nil) + if err != nil { + return "", err + } + + replicaCheckCollections := types.ReplicaCheckCollection{ + Replicas: make(map[string][]*types.ReplicaCheckInfo), + } + for _, collection := range podCollections.Pods { + var resultMap types.ReplicaCheckCollection + if err := json.Unmarshal([]byte(collection.Log), &resultMap); err != nil { + return "", err + } + + for replicaName, replicaCheckInfo := range resultMap.Replicas { + replicaCheckCollections.Replicas[replicaName] = append(replicaCheckCollections.Replicas[replicaName], replicaCheckInfo...) + } + } + + yamlData, err := yaml.Marshal(replicaCheckCollections) + if err != nil { + return "", err + } + + return string(yamlData), nil +} + +// Cleanup deletes the DaemonSet created for the replica checker. +func (remote *Checker) Cleanup() error { + return commonkube.DeleteDaemonSet(remote.kubeClient, remote.Namespace, remote.appName) +} + +// newDaemonSet prepares the DaemonSet for the replica checker. +func (remote *Checker) newDaemonSet() *appsv1.DaemonSet { + outputFilePath := filepath.Join(consts.VolumeMountSharedDirectory, consts.FileNameOutputJSON) + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: remote.appName, + Namespace: remote.Namespace, + Labels: map[string]string{ + "app": remote.appName, + }, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": remote.appName, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": remote.appName, + }, + }, + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + { + Name: consts.ContainerNameInit, + Image: utils.BuildImageName(remote.Image, remote.ImageRegistry), + Command: []string{consts.CmdLonghornctlLocal, consts.SubCmdCheck, consts.SubCmdReplica}, + Env: []corev1.EnvVar{ + { + Name: consts.EnvCurrentNodeID, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + { + Name: consts.EnvLogLevel, + Value: remote.LogLevel, + }, + { + Name: consts.EnvOutputFilePath, + Value: outputFilePath, + }, + { + Name: consts.EnvLonghornVolumeName, + Value: remote.VolumeName, + }, + { + Name: consts.EnvLonghornReplicaName, + Value: remote.ReplicaName, + }, + { + Name: consts.EnvLonghornDataDirectory, + Value: remote.LonghornDataDirectory, + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: consts.VolumeMountHostName, + MountPath: consts.VolumeMountHostDirectory, + ReadOnly: true, + }, + { + Name: consts.VolumeMountSharedName, + MountPath: consts.VolumeMountSharedDirectory, + }, + }, + }, + { + Name: consts.ContainerNameOutput, + Image: utils.BuildImageName(remote.Image, remote.ImageRegistry), + Command: []string{"cat", outputFilePath}, + Env: []corev1.EnvVar{}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: consts.VolumeMountSharedName, + MountPath: consts.VolumeMountSharedDirectory, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: consts.ContainerNamePause, + Image: utils.BuildImageName(consts.ImagePause, remote.ImageRegistry), + }, + }, + Volumes: []corev1.Volume{ + { + Name: consts.VolumeMountHostName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/", + }, + }, + }, + { + Name: consts.VolumeMountSharedName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ + Type: appsv1.RollingUpdateDaemonSetStrategyType, + }, + }, + } +} diff --git a/pkg/types/replica.go b/pkg/types/replica.go index 08021b4a..44691ebe 100644 --- a/pkg/types/replica.go +++ b/pkg/types/replica.go @@ -9,6 +9,22 @@ type ReplicaCollection struct { Replicas map[string][]*ReplicaInfo `json:"replicas" yaml:"replicas"` } +// ReplicaCheckCollection represents a collection of replica check results. +type ReplicaCheckCollection struct { + Replicas map[string][]*ReplicaCheckInfo `json:"replicas" yaml:"replicas"` +} + +// ReplicaCheckInfo holds the snapshot chain integrity check result of a replica. +type ReplicaCheckInfo struct { + Node string `json:"node,omitempty" yaml:"node,omitempty"` + Directory string `json:"directory,omitempty" yaml:"directory,omitempty"` + VolumeName string `json:"volumeName,omitempty" yaml:"volumeName,omitempty"` + SnapshotChain []string `json:"snapshotChain,omitempty" yaml:"snapshotChain,omitempty"` + + Errors []string `json:"errors,omitempty" yaml:"errors,omitempty"` + Warnings []string `json:"warnings,omitempty" yaml:"warnings,omitempty"` +} + // ReplicaInfo holds information about a replica. type ReplicaInfo struct { Node string `json:"node,omitempty" yaml:"node,omitempty"`