DAOS-18860 control: calc engine mem_size only on tgt count

src/control/server/engine/config.go

@@ -783,13 +783,13 @@ func (c *Config) WithLogSubsystems(subsystems string) *Config {
 	return c
 }
 
-// WithMemSize sets the NVMe memory size for SPDK memory allocation on this instance.
+// WithMemSize sets the NVMe memory size in MiB for SPDK memory allocation on this instance.
 func (c *Config) WithMemSize(memsize int) *Config {
 	c.MemSize = memsize
 	return c
 }
 
-// WithHugepageSize sets the configured hugepage size on this instance.
+// WithHugepageSize sets the configured hugepage size in MiB on this instance.
 func (c *Config) WithHugepageSize(hugepagesz int) *Config {
 	c.HugepageSz = hugepagesz
 	return c

src/control/server/server_utils.go

@@ -494,49 +494,52 @@ func setDaosHelperEnvs(cfg *config.Server, setenv func(k, v string) error) error
 
 // Per-NUMA hugepage allocations have been calculated and requested from kernel in prepBdevStorage
 // so now set mem-size values for engines and verify there are enough free hugepages to satisfy
-// typical DMA buffer requirements.
-func setEngineMemSize(srv *server, ei *EngineInstance, smi *common.SysMemInfo) {
+// typical DMA buffer requirements. Note that mem_size generally follows the 1gib-per-tgt rule
+// whereas hugepage allocations made in prepBdevStorage may be slightly different.
+func setEngineMemSize(srv *server, ei *EngineInstance, smi *common.SysMemInfo) error {
 	ei.RLock()
 	ec := ei.runner.GetConfig()
 	eIdx := ec.Index
 
 	if ec.Storage.Tiers.Bdevs().Len() == 0 {
 		srv.log.Debugf("skipping mem check on engine %d, no bdevs", eIdx)
 		ei.RUnlock()
-		return
+		return nil
 	}
 	ei.RUnlock()
 
-	// Mem-size for each engine to be calculated based on server config total hugepage
-	// requirements. Mem-size should be the same for each engine to avoid performance imbalance
+	// nrPagesRequired is per-engine and to be calculated based on engine config target counts.
+	// Mem-size should be the same for each engine to avoid performance imbalance
 	// and will act as memory cap to stop DMA buffer growing beyond mem-size.
-	nrPagesRequired := srv.cfg.NrHugepages / len(srv.cfg.Engines)
+	nrPagesRequired, err := storage.CalcMinHugepages(smi.HugepageSizeKiB, ec.TargetCount)
+	if err != nil {
+		return errors.Wrapf(err, "calculating hugepage mem_size for engine %d", eIdx)
+	}
 
 	// Global (rather than per-NUMA) meminfo stats used to verify sufficient free hugepages as
 	// engines should be started even if hugemem has to be used across NUMA boundaries.
 	nrPagesFree := smi.HugepagesFree
 
-	// Calculate mem_size per I/O engine (in MB) based on number of pages required per engine.
+	// Calculate mem_size per I/O engine (in MiB) based on number of pages required per engine.
 	pageSizeMiB := smi.HugepageSizeKiB / humanize.KiByte // kib to mib
 	memSizeReqMiB := nrPagesRequired * pageSizeMiB
 	memSizeFreeMiB := nrPagesFree * pageSizeMiB
 
-	// If free hugepage mem is not enough to meet requested number of hugepages, log notice and
-	// set mem_size engine parameter to free value. Otherwise set to requested value.
-	memSizeMiB := memSizeReqMiB
+	// If free hugepage mem is not enough to meet requested number of hugepages, log notice.
 	if memSizeFreeMiB < memSizeReqMiB {
 		srv.log.Noticef("The amount of hugepage memory available for engine %d (%s, %d "+
-			"hugepages) does not meet what is required (%s, %d hugepages)", ei.Index(),
+			"hugepages) does not meet what is required (%s, %d hugepages)", eIdx,
 			humanize.IBytes(uint64(humanize.MiByte*memSizeFreeMiB)), nrPagesFree,
 			humanize.IBytes(uint64(humanize.MiByte*memSizeReqMiB)), nrPagesRequired)
-		memSizeMiB = memSizeFreeMiB
 	}
 
 	// Set hugepage_size (MiB) values based on hugepage info.
-	srv.log.Debugf("Per-engine MemSize:%dMB, HugepageSize:%dMB (meminfo: %s)", memSizeMiB,
-		pageSizeMiB, smi.Summary())
-	ei.setMemSize(memSizeMiB)
+	srv.log.Debugf("Engine %d Per-engine MemSize:%dMiB, HugepageSize:%dMiB (meminfo: %s)",
+		eIdx, memSizeReqMiB, pageSizeMiB, smi.Summary())
+	ei.setMemSize(memSizeReqMiB)
 	ei.setHugepageSz(pageSizeMiB)
+
+	return nil
 }
 
 // Clean SPDK resources, both lockfiles and orphaned hugepages. Orphaned hugepages will be cleaned
@@ -697,7 +700,9 @@ func registerEngineEventCallbacks(srv *server, engine *EngineInstance, allStarte
 		}
 
 		// Update engine memory related config parameters before starting.
-		setEngineMemSize(srv, engine, smi)
+		if err := setEngineMemSize(srv, engine, smi); err != nil {
+			return errors.Wrap(err, "set engine mem_size value")
+		}
 
 		// Check available RAM can satisfy tmpfs size before starting a new engine.
 		if err := checkEngineTmpfsMem(srv, engine, smi); err != nil {