diff --git a/cmd/local/subcmd/check.go b/cmd/local/subcmd/check.go index 68939874..3e98697b 100644 --- a/cmd/local/subcmd/check.go +++ b/cmd/local/subcmd/check.go @@ -9,6 +9,7 @@ import ( "github.com/longhorn/cli/pkg/consts" local "github.com/longhorn/cli/pkg/local/preflight" + localreplica "github.com/longhorn/cli/pkg/local/replica" "github.com/longhorn/cli/pkg/types" "github.com/longhorn/cli/pkg/utils" ) @@ -22,6 +23,61 @@ func NewCmdCheck(globalOpts *types.GlobalCmdOptions) *cobra.Command { utils.SetGlobalOptionsLocal(cmd, globalOpts) cmd.AddCommand(newCmdCheckPreflight(globalOpts)) + cmd.AddCommand(newCmdCheckReplica(globalOpts)) + + return cmd +} + +func newCmdCheckReplica(globalOpts *types.GlobalCmdOptions) *cobra.Command { + var localChecker = localreplica.Checker{} + + cmd := &cobra.Command{ + Use: consts.SubCmdReplica, + Short: "Check Longhorn replica integrity", + Long: `This command checks the integrity of the snapshot chains in the Longhorn replica data directories. +It identifies broken snapshot chains, for example snapshots referencing a missing parent, disk files without metadata files, and metadata files without disk files. +The results are presented by the replica data directory names, not the actual Custom Resource (CR) names. + +By default, this command checks all Longhorn replicas in the data directory. +You can narrow down the results by using the following options: +- --name: Specify the Longhorn replica data directory name to check a specific replica. +- --volume-name: Filter replicas by the volume they belong to.`, + + PreRun: func(cmd *cobra.Command, args []string) { + localChecker.LogLevel = globalOpts.LogLevel + + err := localChecker.Init() + if err != nil { + utils.CheckErr(errors.Wrap(err, "Failed to initialize replica checker")) + } + }, + + Run: func(cmd *cobra.Command, args []string) { + err := localChecker.Run() + if err != nil { + utils.CheckErr(errors.Wrap(err, "Failed to run replica checker")) + } + + logrus.Info("Successfully checked replica integrity") + }, + + PostRun: func(cmd *cobra.Command, args []string) { + err := localChecker.Output() + if err != nil { + utils.CheckErr(errors.Wrap(err, "Failed to output replica checker collection")) + } + + logrus.Info("Successfully output replica checker collection") + }, + } + + utils.SetGlobalOptionsLocal(cmd, globalOpts) + + cmd.Flags().StringVar(&localChecker.CurrentNodeID, consts.CmdOptNodeId, os.Getenv(consts.EnvCurrentNodeID), "Current node ID.") + cmd.Flags().StringVarP(&localChecker.OutputFilePath, consts.CmdOptOutputFile, "o", os.Getenv(consts.EnvOutputFilePath), "Output the result to a file, default to stdout.") + cmd.Flags().StringVar(&localChecker.ReplicaName, consts.CmdOptName, os.Getenv(consts.EnvLonghornReplicaName), "Specify the name of the replica to check.") + cmd.Flags().StringVar(&localChecker.VolumeName, consts.CmdOptLonghornVolumeName, os.Getenv(consts.EnvLonghornVolumeName), "Specify the name of the volume to check its replicas.") + cmd.Flags().StringVar(&localChecker.LonghornDataDirectory, consts.CmdOptLonghornDataDirectory, os.Getenv(consts.EnvLonghornDataDirectory), "Specify the Longhorn data directory. If not provided, the default will be attempted, or it will fall back to the directory of longhorn-disk.cfg.") return cmd } diff --git a/cmd/remote/subcmd/check.go b/cmd/remote/subcmd/check.go index 51235251..aafc9f56 100644 --- a/cmd/remote/subcmd/check.go +++ b/cmd/remote/subcmd/check.go @@ -7,6 +7,7 @@ import ( "github.com/longhorn/cli/pkg/consts" "github.com/longhorn/cli/pkg/remote/preflight" + "github.com/longhorn/cli/pkg/remote/replica" "github.com/longhorn/cli/pkg/types" "github.com/longhorn/cli/pkg/utils" ) @@ -20,6 +21,88 @@ func NewCmdCheck(globalOpts *types.GlobalCmdOptions) *cobra.Command { utils.SetGlobalOptionsRemote(cmd, globalOpts) cmd.AddCommand(newCmdCheckPreflight(globalOpts)) + cmd.AddCommand(newCmdCheckReplica(globalOpts)) + + return cmd +} + +func newCmdCheckReplica(globalOpts *types.GlobalCmdOptions) *cobra.Command { + var replicaChecker = replica.Checker{} + + cmd := &cobra.Command{ + Use: consts.SubCmdReplica, + Short: "Check Longhorn replica integrity", + Long: `This command checks the integrity of the snapshot chains in the Longhorn replica data directories on each node. +It identifies broken snapshot chains, for example snapshots referencing a missing parent, disk files without metadata files, and metadata files without disk files. +The results are presented by the replica data directory names, not the actual Custom Resource (CR) names. + +By default, this command checks all Longhorn replicas in the data directory. +You can narrow down the results by using the following options: +- --name: Specify the Longhorn replica data directory name to check a specific replica. +- --volume-name: Filter replicas by the volume they belong to.`, + Example: `$ longhornctl check replica +INFO[2024-07-16T17:23:47+08:00] Initializing replica checker +INFO[2024-07-16T17:23:47+08:00] Cleaning up replica checker +INFO[2024-07-16T17:23:47+08:00] Running replica checker +INFO[2024-07-16T17:23:51+08:00] Retrieved replica check results: + replicas: + pvc-48a6457d-585e-423b-b530-bbc68a5f948a-0e2603a7: + - node: ip-10-0-2-123 + directory: /var/lib/longhorn/replicas/pvc-48a6457d-585e-423b-b530-bbc68a5f948a-0e2603a7 + volumeName: pvc-48a6457d-585e-423b-b530-bbc68a5f948a + snapshotChain: + - volume-head-001.img + - volume-snap-40b3b028-b3b3-4a35-a806-8bea77f27c00.img + errors: + - 'broken snapshot chain: disk volume-snap-40b3b028-b3b3-4a35-a806-8bea77f27c00.img references parent volume-snap-6f244bbe-2857-46e4-92e2-eb1e16a63ba1.img, but the parent metadata file is missing' +INFO[2024-07-16T17:23:51+08:00] Cleaning up replica checker +INFO[2024-07-16T17:23:51+08:00] Completed replica checker`, + + PreRun: func(cmd *cobra.Command, args []string) { + replicaChecker.Image = globalOpts.Image + replicaChecker.ImageRegistry = globalOpts.ImageRegistry + replicaChecker.ImagePullSecret = globalOpts.ImagePullSecret + replicaChecker.KubeConfigPath = globalOpts.KubeConfigPath + replicaChecker.NodeSelector = globalOpts.NodeSelector + replicaChecker.Tolerations = globalOpts.Tolerations + replicaChecker.Namespace = globalOpts.Namespace + + logrus.Info("Initializing replica checker") + if err := replicaChecker.Init(); err != nil { + utils.CheckErr(errors.Wrap(err, "Failed to initialize replica checker")) + } + + logrus.Info("Cleaning up replica checker") + if err := replicaChecker.Cleanup(); err != nil { + utils.CheckErr(errors.Wrapf(err, "Failed to cleanup replica checker")) + } + }, + + Run: func(cmd *cobra.Command, args []string) { + logrus.Info("Running replica checker") + output, err := replicaChecker.Run() + if err != nil { + utils.CheckErr(errors.Wrap(err, "Failed to run replica checker")) + } + + logrus.Infof("Retrieved replica check results:\n %v", output) + }, + + PostRun: func(cmd *cobra.Command, args []string) { + logrus.Info("Cleaning up replica checker") + if err := replicaChecker.Cleanup(); err != nil { + utils.CheckErr(errors.Wrapf(err, "Failed to cleanup replica checker")) + } + + logrus.Info("Completed replica checker") + }, + } + + utils.SetGlobalOptionsRemote(cmd, globalOpts) + + cmd.Flags().StringVar(&replicaChecker.ReplicaName, consts.CmdOptName, "", "Specify the name of the replica to check.") + cmd.Flags().StringVar(&replicaChecker.VolumeName, consts.CmdOptLonghornVolumeName, "", "Specify the name of the volume to check its replicas.") + cmd.Flags().StringVar(&replicaChecker.LonghornDataDirectory, consts.CmdOptLonghornDataDirectory, "/var/lib/longhorn", "Specify the Longhorn data directory. If not provided, the default will be attempted, or it will fall back to the directory of longhorn-disk.cfg.") return cmd } diff --git a/docs/longhornctl_check.md b/docs/longhornctl_check.md index d481efb5..683b33f5 100644 --- a/docs/longhornctl_check.md +++ b/docs/longhornctl_check.md @@ -25,5 +25,6 @@ Longhorn checking operations * [longhornctl](longhornctl.md) - Command-line interface for Longhorn. * [longhornctl check preflight](longhornctl_check_preflight.md) - Run a preflight check for Longhorn +* [longhornctl check replica](longhornctl_check_replica.md) - Check Longhorn replica integrity -###### Auto generated by spf13/cobra on 6-Jul-2026 +###### Auto generated by spf13/cobra on 20-Jul-2026 diff --git a/docs/longhornctl_check_replica.md b/docs/longhornctl_check_replica.md new file mode 100644 index 00000000..4a414bd9 --- /dev/null +++ b/docs/longhornctl_check_replica.md @@ -0,0 +1,68 @@ +## longhornctl check replica + +Check Longhorn replica integrity + +### Synopsis + +This command checks the integrity of the snapshot chains in the Longhorn replica data directories on each node. +It identifies broken snapshot chains, for example snapshots referencing a missing parent, disk files without metadata files, and metadata files without disk files. +The results are presented by the replica data directory names, not the actual Custom Resource (CR) names. + +By default, this command checks all Longhorn replicas in the data directory. +You can narrow down the results by using the following options: +- --name: Specify the Longhorn replica data directory name to check a specific replica. +- --volume-name: Filter replicas by the volume they belong to. + +``` +longhornctl check replica [flags] +``` + +### Examples + +``` +$ longhornctl check replica +INFO[2024-07-16T17:23:47+08:00] Initializing replica checker +INFO[2024-07-16T17:23:47+08:00] Cleaning up replica checker +INFO[2024-07-16T17:23:47+08:00] Running replica checker +INFO[2024-07-16T17:23:51+08:00] Retrieved replica check results: + replicas: + pvc-48a6457d-585e-423b-b530-bbc68a5f948a-0e2603a7: + - node: ip-10-0-2-123 + directory: /var/lib/longhorn/replicas/pvc-48a6457d-585e-423b-b530-bbc68a5f948a-0e2603a7 + volumeName: pvc-48a6457d-585e-423b-b530-bbc68a5f948a + snapshotChain: + - volume-head-001.img + - volume-snap-40b3b028-b3b3-4a35-a806-8bea77f27c00.img + errors: + - 'broken snapshot chain: disk volume-snap-40b3b028-b3b3-4a35-a806-8bea77f27c00.img references parent volume-snap-6f244bbe-2857-46e4-92e2-eb1e16a63ba1.img, but the parent metadata file is missing' +INFO[2024-07-16T17:23:51+08:00] Cleaning up replica checker +INFO[2024-07-16T17:23:51+08:00] Completed replica checker +``` + +### Options + +``` + --data-dir string Specify the Longhorn data directory. If not provided, the default will be attempted, or it will fall back to the directory of longhorn-disk.cfg. (default "/var/lib/longhorn") + -h, --help help for replica + --image string Image containing longhornctl-local (default "longhornio/longhorn-cli:v1.13.0-dev") + --image-pull-secret string Secret with registry credentials for pulling images + --image-registry string Registry to apply to all images (CLI, engine, pause, BCI, etc.), replacing any registry already specified in those images. + --kubeconfig string Kubernetes config (kubeconfig) path + -l, --log-level string Log level (default "info") + --name string Specify the name of the replica to check. + --namespace string The namespace to run DaemonSet pods. (default "longhorn-system") + --node-selector string Comma-separated list of key=value pairs to match against node labels, selecting the nodes the DaemonSet will run on (e.g. env=prod,zone=us-west). + --volume-name string Specify the name of the volume to check its replicas. +``` + +### Options inherited from parent commands + +``` + --tolerations string Semicolon-separated list of tolerations for DaemonSet pods (e.g. key=value:NoSchedule;:NoExecute). +``` + +### SEE ALSO + +* [longhornctl check](longhornctl_check.md) - Longhorn checking operations + +###### Auto generated by spf13/cobra on 20-Jul-2026 diff --git a/pkg/consts/replica.go b/pkg/consts/replica.go index d12f11de..a54d1b94 100644 --- a/pkg/consts/replica.go +++ b/pkg/consts/replica.go @@ -1,6 +1,7 @@ package consts const ( + AppNameReplicaChecker = "longhorn-replica-checker" AppNameReplicaExporter = "longhorn-replica-exporter" AppNameReplicaGetter = "longhorn-replica-getter" ) diff --git a/pkg/local/replica/checker.go b/pkg/local/replica/checker.go new file mode 100644 index 00000000..b5862fdd --- /dev/null +++ b/pkg/local/replica/checker.go @@ -0,0 +1,318 @@ +package replica + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + + lhmgrutil "github.com/longhorn/longhorn-manager/util" + + commonio "github.com/longhorn/go-common-libs/io" + + "github.com/longhorn/cli/pkg/consts" + remote "github.com/longhorn/cli/pkg/remote/replica" + "github.com/longhorn/cli/pkg/types" + "github.com/longhorn/cli/pkg/utils" + utilslonghorn "github.com/longhorn/cli/pkg/utils/longhorn" +) + +const ( + diskImageSuffix = ".img" + diskMetadataSuffix = ".img.meta" + + volumeHeadPrefix = "volume-head-" + volumeSnapshotPrefix = "volume-snap-" + volumeMetadataFile = "volume.meta" +) + +// diskMetadata mirrors the metadata the Longhorn engine persists next to each +// disk file (volume head or snapshot) in the replica data directory. +type diskMetadata struct { + Name string + Parent string + Removed bool + UserCreated bool + Created string + Labels map[string]string +} + +// Checker provide functions for the replica checker. +type Checker struct { + remote.CheckerCmdOptions + + logger *logrus.Entry + + OutputFilePath string + CurrentNodeID string + + replicasDirectory string + replicaNames []string + + collection types.ReplicaCheckCollection +} + +// Init initializes the Checker. +func (local *Checker) Init() error { + var err error + + if len(local.OutputFilePath) != 0 { + local.logger = logrus.WithField("output", local.OutputFilePath) + } else { + local.logger = logrus.WithField("output", "stdout") + } + + local.LonghornDataDirectory, err = utilslonghorn.GetDataDirectory(local.logger, consts.VolumeMountHostDirectory, local.LonghornDataDirectory) + if err != nil { + return errors.Wrap(err, "failed to get Longhorn data directory") + } + + local.logger = local.logger.WithField("data-dir", local.LonghornDataDirectory) + + local.replicasDirectory = filepath.Join(local.LonghornDataDirectory, "replicas") + + local.collection.Replicas = make(map[string][]*types.ReplicaCheckInfo) + + return nil +} + +// Run checks the snapshot chain integrity of the replicas in the data directory. +func (local *Checker) Run() error { + var err error + + log := local.logger + if local.VolumeName != "" { + log = log.WithField("volume", local.VolumeName) + } + if local.ReplicaName != "" { + log = log.WithField("replica", local.ReplicaName) + } + + local.replicaNames, err = getReplicaNamesInDirectory(log, local.replicasDirectory, local.VolumeName, local.ReplicaName) + if err != nil { + return err + } + + for _, replicaName := range local.replicaNames { + replicaCheckInfo, err := local.checkReplica(replicaName) + if err != nil { + return errors.Wrapf(err, "failed to check replica %s", replicaName) + } + + if replicaCheckInfo == nil { + continue + } + + local.collection.Replicas[replicaName] = append(local.collection.Replicas[replicaName], replicaCheckInfo) + } + + return nil +} + +// Output converts the collection to JSON and output to stdout or the output file. +func (local *Checker) Output() error { + local.logger.Tracef("Outputting replica checker results") + + jsonBytes, err := json.Marshal(local.collection) + if err != nil { + return errors.Wrap(err, "failed to convert replica check collections to JSON") + } + + return utils.HandleResult(jsonBytes, local.OutputFilePath, local.logger) +} + +// checkReplica checks the snapshot chain integrity of a single replica directory. +func (local *Checker) checkReplica(replicaName string) (*types.ReplicaCheckInfo, error) { + log := local.logger + + log.Infof("Checking snapshot chain for replica %s", replicaName) + + replicaCheckInfo := &types.ReplicaCheckInfo{} + replicaCheckInfo.Node = local.CurrentNodeID + replicaCheckInfo.VolumeName = replicaName[:strings.LastIndex(replicaName, "-")] + + replicaDirectory := filepath.Join(local.replicasDirectory, replicaName) + replicaCheckInfo.Directory = strings.TrimPrefix(replicaDirectory, consts.VolumeMountHostDirectory) + + isEmpty, err := commonio.IsDirectoryEmpty(replicaDirectory) + if err != nil { + replicaCheckInfo.Errors = append(replicaCheckInfo.Errors, errors.Wrapf(err, "failed to check if directory %s is empty", replicaCheckInfo.Directory).Error()) + return replicaCheckInfo, nil + } + + if isEmpty { + log.Warnf("Replica directory %s is empty", replicaCheckInfo.Directory) + replicaCheckInfo.Warnings = append(replicaCheckInfo.Warnings, "replica directory is empty") + return replicaCheckInfo, nil + } + + isReplicaInUse, err := isReplicaDirectoryInUse(replicaDirectory) + if err != nil { + replicaCheckInfo.Warnings = append(replicaCheckInfo.Warnings, errors.Wrapf(err, "failed to check if replica %s is in use", replicaName).Error()) + } else if isReplicaInUse { + replicaCheckInfo.Warnings = append(replicaCheckInfo.Warnings, "replica is in use; findings may be transient while the engine is modifying the snapshot chain") + } + + chain, checkErrors, warnings := validateSnapshotChain(replicaDirectory) + replicaCheckInfo.SnapshotChain = chain + replicaCheckInfo.Errors = append(replicaCheckInfo.Errors, checkErrors...) + replicaCheckInfo.Warnings = append(replicaCheckInfo.Warnings, warnings...) + + return replicaCheckInfo, nil +} + +// validateSnapshotChain inspects the disk files in the given replica directory +// and returns the volume head chain (from the head to the root), along with any +// integrity errors and warnings found. +func validateSnapshotChain(replicaDirectory string) (chain []string, checkErrors []string, warnings []string) { + volumeMeta := &lhmgrutil.VolumeMeta{} + content, err := os.ReadFile(filepath.Join(replicaDirectory, volumeMetadataFile)) + if err != nil { + checkErrors = append(checkErrors, fmt.Sprintf("failed to read %s: %v", volumeMetadataFile, err)) + return nil, checkErrors, warnings + } + if err := json.Unmarshal(content, volumeMeta); err != nil { + checkErrors = append(checkErrors, fmt.Sprintf("failed to parse %s: %v", volumeMetadataFile, err)) + return nil, checkErrors, warnings + } + + if volumeMeta.Error != "" { + checkErrors = append(checkErrors, fmt.Sprintf("%s records an error: %s", volumeMetadataFile, volumeMeta.Error)) + } + if volumeMeta.Rebuilding { + warnings = append(warnings, "replica is marked as rebuilding; the snapshot chain may be incomplete until the rebuild finishes") + } + + disks, images, headImages, diskReadErrors := readDiskMetadataFiles(replicaDirectory) + checkErrors = append(checkErrors, diskReadErrors...) + + // Every disk file must have a metadata file, and vice versa. + for _, diskName := range sortedKeys(disks) { + if !images[diskName] { + checkErrors = append(checkErrors, fmt.Sprintf("disk metadata %s%s exists, but disk file %s is missing", diskName, ".meta", diskName)) + } + if metaName := disks[diskName].Name; metaName != "" && metaName != diskName { + checkErrors = append(checkErrors, fmt.Sprintf("disk metadata %s%s declares mismatching disk name %s", diskName, ".meta", metaName)) + } + } + for _, imageName := range sortedKeys(images) { + if _, ok := disks[imageName]; !ok { + checkErrors = append(checkErrors, fmt.Sprintf("disk file %s exists, but its metadata file %s%s is missing", imageName, imageName, ".meta")) + } + } + + // Every parent reference must resolve, including the ones on snapshot tree + // branches that are not part of the volume head chain. + for _, diskName := range sortedKeys(disks) { + parent := disks[diskName].Parent + if parent == "" { + continue + } + if _, ok := disks[parent]; !ok { + checkErrors = append(checkErrors, fmt.Sprintf("broken snapshot chain: disk %s references parent %s, but the parent metadata file is missing", diskName, parent)) + } + } + + head := volumeMeta.Head + if head == "" { + checkErrors = append(checkErrors, fmt.Sprintf("%s does not specify a volume head", volumeMetadataFile)) + return nil, checkErrors, warnings + } + + if _, ok := disks[head]; !ok { + checkErrors = append(checkErrors, fmt.Sprintf("broken snapshot chain: volume head %s declared in %s is missing its metadata file", head, volumeMetadataFile)) + } + if !images[head] { + checkErrors = append(checkErrors, fmt.Sprintf("broken snapshot chain: volume head file %s declared in %s is missing", head, volumeMetadataFile)) + } + + sort.Strings(headImages) + for _, headImage := range headImages { + if headImage != head { + warnings = append(warnings, fmt.Sprintf("found unexpected volume head file %s; the active volume head is %s", headImage, head)) + } + } + + // Walk the chain from the volume head to the root. Dangling parents and + // missing files are already reported above, so the walk only needs to + // detect loops and record the reachable chain. + visited := map[string]bool{} + current := head + for current != "" { + if visited[current] { + checkErrors = append(checkErrors, fmt.Sprintf("snapshot chain contains a loop at disk %s", current)) + break + } + visited[current] = true + + diskMeta, ok := disks[current] + if !ok { + break + } + + chain = append(chain, current) + current = diskMeta.Parent + } + + return chain, checkErrors, warnings +} + +// readDiskMetadataFiles reads the disk files in the given replica directory and +// returns the parsed disk metadata, the set of disk image files, and the volume +// head image files found. +func readDiskMetadataFiles(replicaDirectory string) (disks map[string]*diskMetadata, images map[string]bool, headImages []string, checkErrors []string) { + disks = map[string]*diskMetadata{} + images = map[string]bool{} + + entries, err := os.ReadDir(replicaDirectory) + if err != nil { + checkErrors = append(checkErrors, fmt.Sprintf("failed to list replica directory: %v", err)) + return disks, images, headImages, checkErrors + } + + for _, entry := range entries { + name := entry.Name() + if !strings.HasPrefix(name, volumeHeadPrefix) && !strings.HasPrefix(name, volumeSnapshotPrefix) { + continue + } + + switch { + case strings.HasSuffix(name, diskMetadataSuffix): + content, err := os.ReadFile(filepath.Join(replicaDirectory, name)) + if err != nil { + checkErrors = append(checkErrors, fmt.Sprintf("failed to read disk metadata %s: %v", name, err)) + continue + } + + diskMeta := &diskMetadata{} + if err := json.Unmarshal(content, diskMeta); err != nil { + checkErrors = append(checkErrors, fmt.Sprintf("failed to parse disk metadata %s: %v", name, err)) + continue + } + + disks[strings.TrimSuffix(name, ".meta")] = diskMeta + + case strings.HasSuffix(name, diskImageSuffix): + images[name] = true + if strings.HasPrefix(name, volumeHeadPrefix) { + headImages = append(headImages, name) + } + } + } + + return disks, images, headImages, checkErrors +} + +func sortedKeys[V any](m map[string]V) []string { + keys := make([]string, 0, len(m)) + for key := range m { + keys = append(keys, key) + } + sort.Strings(keys) + return keys +} diff --git a/pkg/local/replica/checker_test.go b/pkg/local/replica/checker_test.go new file mode 100644 index 00000000..ae3cba8b --- /dev/null +++ b/pkg/local/replica/checker_test.go @@ -0,0 +1,211 @@ +package replica + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/suite" + + lhmgrutil "github.com/longhorn/longhorn-manager/util" +) + +type CheckerTestSuite struct { + suite.Suite + + replicaDirectory string +} + +func (s *CheckerTestSuite) SetupTest() { + s.replicaDirectory = s.T().TempDir() +} + +func (s *CheckerTestSuite) writeVolumeMeta(head string) { + volumeMeta := &lhmgrutil.VolumeMeta{ + Size: 1073741824, + Head: head, + SectorSize: 512, + } + content, err := json.Marshal(volumeMeta) + s.Require().NoError(err) + s.Require().NoError(os.WriteFile(filepath.Join(s.replicaDirectory, volumeMetadataFile), content, 0600)) +} + +func (s *CheckerTestSuite) writeDiskMeta(diskName, parent string) { + diskMeta := &diskMetadata{ + Name: diskName, + Parent: parent, + } + content, err := json.Marshal(diskMeta) + s.Require().NoError(err) + s.Require().NoError(os.WriteFile(filepath.Join(s.replicaDirectory, diskName+".meta"), content, 0600)) +} + +func (s *CheckerTestSuite) writeDiskImage(diskName string) { + s.Require().NoError(os.WriteFile(filepath.Join(s.replicaDirectory, diskName), []byte{}, 0600)) +} + +func (s *CheckerTestSuite) writeDisk(diskName, parent string) { + s.writeDiskImage(diskName) + s.writeDiskMeta(diskName, parent) +} + +func (s *CheckerTestSuite) TestHealthyChain() { + s.writeVolumeMeta("volume-head-001.img") + s.writeDisk("volume-snap-a.img", "") + s.writeDisk("volume-snap-b.img", "volume-snap-a.img") + s.writeDisk("volume-head-001.img", "volume-snap-b.img") + + chain, checkErrors, warnings := validateSnapshotChain(s.replicaDirectory) + + s.Empty(checkErrors) + s.Empty(warnings) + s.Equal([]string{"volume-head-001.img", "volume-snap-b.img", "volume-snap-a.img"}, chain) +} + +func (s *CheckerTestSuite) TestHealthySnapshotTree() { + // A snapshot tree with a branch (for example after a revert) is not broken: + // both branches share the root, and the head is on one of them. + s.writeVolumeMeta("volume-head-002.img") + s.writeDisk("volume-snap-root.img", "") + s.writeDisk("volume-snap-branch.img", "volume-snap-root.img") + s.writeDisk("volume-head-002.img", "volume-snap-root.img") + + chain, checkErrors, _ := validateSnapshotChain(s.replicaDirectory) + + s.Empty(checkErrors) + s.Equal([]string{"volume-head-002.img", "volume-snap-root.img"}, chain) +} + +func (s *CheckerTestSuite) TestMissingParent() { + s.writeVolumeMeta("volume-head-001.img") + s.writeDisk("volume-snap-b.img", "volume-snap-a.img") + s.writeDisk("volume-head-001.img", "volume-snap-b.img") + + chain, checkErrors, _ := validateSnapshotChain(s.replicaDirectory) + + s.Len(checkErrors, 1) + s.Contains(checkErrors[0], "broken snapshot chain") + s.Contains(checkErrors[0], "volume-snap-a.img") + s.Equal([]string{"volume-head-001.img", "volume-snap-b.img"}, chain) +} + +func (s *CheckerTestSuite) TestMissingDiskFile() { + s.writeVolumeMeta("volume-head-001.img") + s.writeDisk("volume-snap-a.img", "") + s.writeDiskMeta("volume-snap-b.img", "volume-snap-a.img") + s.writeDisk("volume-head-001.img", "volume-snap-b.img") + + _, checkErrors, _ := validateSnapshotChain(s.replicaDirectory) + + s.Len(checkErrors, 1) + s.Contains(checkErrors[0], "disk file volume-snap-b.img is missing") +} + +func (s *CheckerTestSuite) TestMissingMetadataFile() { + s.writeVolumeMeta("volume-head-001.img") + s.writeDisk("volume-snap-a.img", "") + s.writeDiskImage("volume-snap-b.img") + s.writeDisk("volume-head-001.img", "volume-snap-b.img") + + chain, checkErrors, _ := validateSnapshotChain(s.replicaDirectory) + + s.Len(checkErrors, 2) + s.Contains(checkErrors[0], "metadata file volume-snap-b.img.meta is missing") + s.Contains(checkErrors[1], "broken snapshot chain") + // The chain walk stops at the disk with the missing metadata. + s.Equal([]string{"volume-head-001.img"}, chain) +} + +func (s *CheckerTestSuite) TestMissingVolumeHead() { + s.writeVolumeMeta("volume-head-001.img") + s.writeDisk("volume-snap-a.img", "") + + chain, checkErrors, _ := validateSnapshotChain(s.replicaDirectory) + + s.Len(checkErrors, 2) + s.Contains(checkErrors[0], "volume head volume-head-001.img") + s.Contains(checkErrors[1], "volume head file volume-head-001.img") + s.Empty(chain) +} + +func (s *CheckerTestSuite) TestChainLoop() { + s.writeVolumeMeta("volume-head-001.img") + s.writeDisk("volume-snap-a.img", "volume-snap-b.img") + s.writeDisk("volume-snap-b.img", "volume-snap-a.img") + s.writeDisk("volume-head-001.img", "volume-snap-a.img") + + _, checkErrors, _ := validateSnapshotChain(s.replicaDirectory) + + s.Len(checkErrors, 1) + s.Contains(checkErrors[0], "loop") +} + +func (s *CheckerTestSuite) TestExtraVolumeHead() { + s.writeVolumeMeta("volume-head-002.img") + s.writeDisk("volume-snap-a.img", "") + s.writeDisk("volume-head-002.img", "volume-snap-a.img") + s.writeDisk("volume-head-001.img", "volume-snap-a.img") + + _, checkErrors, warnings := validateSnapshotChain(s.replicaDirectory) + + s.Empty(checkErrors) + s.Len(warnings, 1) + s.Contains(warnings[0], "unexpected volume head file volume-head-001.img") +} + +func (s *CheckerTestSuite) TestCorruptedDiskMetadata() { + s.writeVolumeMeta("volume-head-001.img") + s.writeDisk("volume-snap-a.img", "") + s.writeDiskImage("volume-snap-b.img") + s.Require().NoError(os.WriteFile(filepath.Join(s.replicaDirectory, "volume-snap-b.img.meta"), []byte("{invalid json"), 0600)) + s.writeDisk("volume-head-001.img", "volume-snap-b.img") + + _, checkErrors, _ := validateSnapshotChain(s.replicaDirectory) + + s.NotEmpty(checkErrors) + s.Contains(checkErrors[0], "failed to parse disk metadata volume-snap-b.img.meta") +} + +func (s *CheckerTestSuite) TestMetadataNameMismatch() { + s.writeVolumeMeta("volume-head-001.img") + s.writeDiskImage("volume-snap-a.img") + diskMeta := &diskMetadata{Name: "volume-snap-other.img"} + content, err := json.Marshal(diskMeta) + s.Require().NoError(err) + s.Require().NoError(os.WriteFile(filepath.Join(s.replicaDirectory, "volume-snap-a.img.meta"), content, 0600)) + s.writeDisk("volume-head-001.img", "volume-snap-a.img") + + _, checkErrors, _ := validateSnapshotChain(s.replicaDirectory) + + s.Len(checkErrors, 1) + s.Contains(checkErrors[0], "mismatching disk name") +} + +func (s *CheckerTestSuite) TestMissingVolumeMeta() { + s.writeDisk("volume-snap-a.img", "") + + chain, checkErrors, _ := validateSnapshotChain(s.replicaDirectory) + + s.Len(checkErrors, 1) + s.Contains(checkErrors[0], "failed to read volume.meta") + s.Empty(chain) +} + +func (s *CheckerTestSuite) TestIgnoresChecksumAndUnrelatedFiles() { + s.writeVolumeMeta("volume-head-001.img") + s.writeDisk("volume-snap-a.img", "") + s.writeDisk("volume-head-001.img", "volume-snap-a.img") + s.Require().NoError(os.WriteFile(filepath.Join(s.replicaDirectory, "volume-snap-a.img.checksum"), []byte("{}"), 0600)) + s.Require().NoError(os.WriteFile(filepath.Join(s.replicaDirectory, "revision.counter"), []byte{}, 0600)) + + _, checkErrors, warnings := validateSnapshotChain(s.replicaDirectory) + + s.Empty(checkErrors) + s.Empty(warnings) +} + +func TestCheckerTestSuite(t *testing.T) { + suite.Run(t, new(CheckerTestSuite)) +} diff --git a/pkg/local/replica/getter.go b/pkg/local/replica/getter.go index cdd756dd..0390c373 100644 --- a/pkg/local/replica/getter.go +++ b/pkg/local/replica/getter.go @@ -64,7 +64,15 @@ func (local *Getter) Init() error { func (local *Getter) Run() error { var err error - local.replicaNames, err = local.getReplicaNamesInDirectory() + log := local.logger + if local.VolumeName != "" { + log = log.WithField("volume", local.VolumeName) + } + if local.ReplicaName != "" { + log = log.WithField("replica", local.ReplicaName) + } + + local.replicaNames, err = getReplicaNamesInDirectory(log, local.replicasDirectory, local.VolumeName, local.ReplicaName) if err != nil { return err } @@ -99,16 +107,7 @@ func (local *Getter) Output() error { // getReplicaNamesInDirectory returns a list of replica names in the given directory that match the given volume name. // If the volume name is empty, it returns all replica names in the given directory. -func (local *Getter) getReplicaNamesInDirectory() ([]string, error) { - log := local.logger - replicasDirectory := local.replicasDirectory - - if local.VolumeName != "" { - log = log.WithField("volume", local.VolumeName) - } - if local.ReplicaName != "" { - log = log.WithField("replica", local.ReplicaName) - } +func getReplicaNamesInDirectory(log *logrus.Entry, replicasDirectory, volumeNameFilter, replicaNameFilter string) ([]string, error) { log.Infof("Searching for replicas in %s", replicasDirectory) filePaths, err := commonio.FindFiles(replicasDirectory, "", 1) @@ -135,12 +134,12 @@ func (local *Getter) getReplicaNamesInDirectory() ([]string, error) { continue } - if local.ReplicaName != "" && local.ReplicaName != replicaName { + if replicaNameFilter != "" && replicaNameFilter != replicaName { continue } volumeName := replicaName[:strings.LastIndex(replicaName, "-")] - if local.VolumeName != "" && local.VolumeName != volumeName { + if volumeNameFilter != "" && volumeNameFilter != volumeName { continue } @@ -179,7 +178,7 @@ func (local *Getter) getReplicaInfo(replicaName string) (replicaInfo *types.Repl return replicaInfo, nil } - isReplicaInUse, err := local.isReplicaInUse(replicaName) + isReplicaInUse, err := isReplicaDirectoryInUse(replicaDirectory) if err != nil { replicaInfo.Error = errors.Wrapf(err, "failed to check if replica %s is in use", replicaName).Error() return replicaInfo, nil @@ -189,9 +188,7 @@ func (local *Getter) getReplicaInfo(replicaName string) (replicaInfo *types.Repl return replicaInfo, nil } -func (local *Getter) isReplicaInUse(name string) (bool, error) { - replicaDirectory := filepath.Join(local.replicasDirectory, name) - +func isReplicaDirectoryInUse(replicaDirectory string) (bool, error) { // Check if replica path exists if _, err := os.Stat(replicaDirectory); os.IsNotExist(err) { return false, errors.Wrapf(err, "replica directory %s does not exist", replicaDirectory) diff --git a/pkg/remote/replica/checker.go b/pkg/remote/replica/checker.go new file mode 100644 index 00000000..b2b88a86 --- /dev/null +++ b/pkg/remote/replica/checker.go @@ -0,0 +1,228 @@ +package replica + +import ( + "encoding/json" + "path/filepath" + + "gopkg.in/yaml.v3" + + "k8s.io/utils/ptr" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + kubeclient "k8s.io/client-go/kubernetes" + + commonkube "github.com/longhorn/go-common-libs/kubernetes" + + "github.com/longhorn/cli/pkg/consts" + "github.com/longhorn/cli/pkg/types" + "github.com/longhorn/cli/pkg/utils" + + kubeutils "github.com/longhorn/cli/pkg/utils/kubernetes" +) + +// Checker provide functions for the replica checker. +type Checker struct { + CheckerCmdOptions + + kubeClient *kubeclient.Clientset + + appName string // App name of the DaemonSet. +} + +// CheckerCmdOptions holds the options for the command. +type CheckerCmdOptions struct { + types.GlobalCmdOptions + + LonghornDataDirectory string + VolumeName string + ReplicaName string +} + +// Init initializes the Checker. +func (remote *Checker) Init() error { + kubeClient, err := kubeutils.NewKubeClient("", remote.KubeConfigPath) + if err != nil { + return err + } + remote.kubeClient = kubeClient + + remote.appName = consts.AppNameReplicaChecker + + return nil +} + +// Run creates the DaemonSet for the replica checker. It ensures that the +// init container and the output container completes before collecting the +// replica check results and returning them as a YAML string. +func (remote *Checker) Run() (string, error) { + newDaemonSet, err := kubeutils.PrepareDaemonSet(remote.newDaemonSet(), remote.kubeClient, remote.NodeSelector, remote.ImagePullSecret, remote.Tolerations) + if err != nil { + return "", err + } + + daemonSet, err := commonkube.CreateDaemonSet(remote.kubeClient, newDaemonSet) + if err != nil { + return "", err + } + + err = kubeutils.MonitorDaemonSetContainer(remote.kubeClient, daemonSet, consts.ContainerNameInit, kubeutils.WaitForDaemonSetContainersExit, ptr.To(consts.ContainerConditionMaxTolerationMedium)) + if err != nil { + return "", err + } + + err = kubeutils.MonitorDaemonSetContainer(remote.kubeClient, daemonSet, consts.ContainerNameOutput, kubeutils.WaitForDaemonSetContainersExit, ptr.To(consts.ContainerConditionMaxTolerationShort)) + if err != nil { + return "", err + } + + podCollections, err := kubeutils.GetDaemonSetPodCollections(remote.kubeClient, daemonSet, consts.ContainerNameOutput, false, false, nil) + if err != nil { + return "", err + } + + replicaCheckCollections := types.ReplicaCheckCollection{ + Replicas: make(map[string][]*types.ReplicaCheckInfo), + } + for _, collection := range podCollections.Pods { + var resultMap types.ReplicaCheckCollection + if err := json.Unmarshal([]byte(collection.Log), &resultMap); err != nil { + return "", err + } + + for replicaName, replicaCheckInfo := range resultMap.Replicas { + replicaCheckCollections.Replicas[replicaName] = append(replicaCheckCollections.Replicas[replicaName], replicaCheckInfo...) + } + } + + yamlData, err := yaml.Marshal(replicaCheckCollections) + if err != nil { + return "", err + } + + return string(yamlData), nil +} + +// Cleanup deletes the DaemonSet created for the replica checker. +func (remote *Checker) Cleanup() error { + return commonkube.DeleteDaemonSet(remote.kubeClient, remote.Namespace, remote.appName) +} + +// newDaemonSet prepares the DaemonSet for the replica checker. +func (remote *Checker) newDaemonSet() *appsv1.DaemonSet { + outputFilePath := filepath.Join(consts.VolumeMountSharedDirectory, consts.FileNameOutputJSON) + return &appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: remote.appName, + Namespace: remote.Namespace, + Labels: map[string]string{ + "app": remote.appName, + }, + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": remote.appName, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": remote.appName, + }, + }, + Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{ + { + Name: consts.ContainerNameInit, + Image: utils.BuildImageName(remote.Image, remote.ImageRegistry), + Command: []string{consts.CmdLonghornctlLocal, consts.SubCmdCheck, consts.SubCmdReplica}, + Env: []corev1.EnvVar{ + { + Name: consts.EnvCurrentNodeID, + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + { + Name: consts.EnvLogLevel, + Value: remote.LogLevel, + }, + { + Name: consts.EnvOutputFilePath, + Value: outputFilePath, + }, + { + Name: consts.EnvLonghornVolumeName, + Value: remote.VolumeName, + }, + { + Name: consts.EnvLonghornReplicaName, + Value: remote.ReplicaName, + }, + { + Name: consts.EnvLonghornDataDirectory, + Value: remote.LonghornDataDirectory, + }, + }, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: consts.VolumeMountHostName, + MountPath: consts.VolumeMountHostDirectory, + ReadOnly: true, + }, + { + Name: consts.VolumeMountSharedName, + MountPath: consts.VolumeMountSharedDirectory, + }, + }, + }, + { + Name: consts.ContainerNameOutput, + Image: utils.BuildImageName(remote.Image, remote.ImageRegistry), + Command: []string{"cat", outputFilePath}, + Env: []corev1.EnvVar{}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: consts.VolumeMountSharedName, + MountPath: consts.VolumeMountSharedDirectory, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: consts.ContainerNamePause, + Image: utils.BuildImageName(consts.ImagePause, remote.ImageRegistry), + }, + }, + Volumes: []corev1.Volume{ + { + Name: consts.VolumeMountHostName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/", + }, + }, + }, + { + Name: consts.VolumeMountSharedName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + }, + }, + }, + UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ + Type: appsv1.RollingUpdateDaemonSetStrategyType, + }, + }, + } +} diff --git a/pkg/types/replica.go b/pkg/types/replica.go index 08021b4a..44691ebe 100644 --- a/pkg/types/replica.go +++ b/pkg/types/replica.go @@ -9,6 +9,22 @@ type ReplicaCollection struct { Replicas map[string][]*ReplicaInfo `json:"replicas" yaml:"replicas"` } +// ReplicaCheckCollection represents a collection of replica check results. +type ReplicaCheckCollection struct { + Replicas map[string][]*ReplicaCheckInfo `json:"replicas" yaml:"replicas"` +} + +// ReplicaCheckInfo holds the snapshot chain integrity check result of a replica. +type ReplicaCheckInfo struct { + Node string `json:"node,omitempty" yaml:"node,omitempty"` + Directory string `json:"directory,omitempty" yaml:"directory,omitempty"` + VolumeName string `json:"volumeName,omitempty" yaml:"volumeName,omitempty"` + SnapshotChain []string `json:"snapshotChain,omitempty" yaml:"snapshotChain,omitempty"` + + Errors []string `json:"errors,omitempty" yaml:"errors,omitempty"` + Warnings []string `json:"warnings,omitempty" yaml:"warnings,omitempty"` +} + // ReplicaInfo holds information about a replica. type ReplicaInfo struct { Node string `json:"node,omitempty" yaml:"node,omitempty"`