DAOS-18860 control: calc engine mem_size only on tgt count

src/control/server/engine/config.go

@@ -783,13 +783,13 @@ func (c *Config) WithLogSubsystems(subsystems string) *Config {
 	return c
 }
 
-// WithMemSize sets the NVMe memory size for SPDK memory allocation on this instance.
+// WithMemSize sets the NVMe memory size in MiB for SPDK memory allocation on this instance.
 func (c *Config) WithMemSize(memsize int) *Config {
 	c.MemSize = memsize
 	return c
 }
 
-// WithHugepageSize sets the configured hugepage size on this instance.
+// WithHugepageSize sets the configured hugepage size in MiB on this instance.
 func (c *Config) WithHugepageSize(hugepagesz int) *Config {
 	c.HugepageSz = hugepagesz
 	return c

src/control/server/server_utils.go

@@ -494,29 +494,33 @@ func setDaosHelperEnvs(cfg *config.Server, setenv func(k, v string) error) error
 
 // Per-NUMA hugepage allocations have been calculated and requested from kernel in prepBdevStorage
 // so now set mem-size values for engines and verify there are enough free hugepages to satisfy
-// typical DMA buffer requirements.
-func setEngineMemSize(srv *server, ei *EngineInstance, smi *common.SysMemInfo) {
+// typical DMA buffer requirements. Note that mem_size generally follows the 1gib-per-tgt rule
+// whereas hugepage allocations made in prepBdevStorage may be slightly different.
+func setEngineMemSize(srv *server, ei *EngineInstance, smi *common.SysMemInfo) error {
 	ei.RLock()
 	ec := ei.runner.GetConfig()
 	eIdx := ec.Index
 
 	if ec.Storage.Tiers.Bdevs().Len() == 0 {
 		srv.log.Debugf("skipping mem check on engine %d, no bdevs", eIdx)
 		ei.RUnlock()
-		return
+		return nil
 	}
 	ei.RUnlock()
 
-	// Mem-size for each engine to be calculated based on server config total hugepage
-	// requirements. Mem-size should be the same for each engine to avoid performance imbalance
+	// nrPagesRequired is per-engine and to be calculated based on engine config target counts.
+	// Mem-size should be the same for each engine to avoid performance imbalance
 	// and will act as memory cap to stop DMA buffer growing beyond mem-size.
-	nrPagesRequired := srv.cfg.NrHugepages / len(srv.cfg.Engines)
+	nrPagesRequired, err := storage.CalcMinHugepages(smi.HugepageSizeKiB, ec.TargetCount)
+	if err != nil {
+		return errors.Wrapf(err, "calculating hugepage mem_size for engine %d", eIdx)
+	}
 
 	// Global (rather than per-NUMA) meminfo stats used to verify sufficient free hugepages as
 	// engines should be started even if hugemem has to be used across NUMA boundaries.
 	nrPagesFree := smi.HugepagesFree
 
-	// Calculate mem_size per I/O engine (in MB) based on number of pages required per engine.
+	// Calculate mem_size per I/O engine (in MiB) based on number of pages required per engine.
 	pageSizeMiB := smi.HugepageSizeKiB / humanize.KiByte // kib to mib
 	memSizeReqMiB := nrPagesRequired * pageSizeMiB
 	memSizeFreeMiB := nrPagesFree * pageSizeMiB
@@ -526,17 +530,19 @@ func setEngineMemSize(srv *server, ei *EngineInstance, smi *common.SysMemInfo) {
 	memSizeMiB := memSizeReqMiB
 	if memSizeFreeMiB < memSizeReqMiB {
 		srv.log.Noticef("The amount of hugepage memory available for engine %d (%s, %d "+
-			"hugepages) does not meet what is required (%s, %d hugepages)", ei.Index(),
+			"hugepages) does not meet what is required (%s, %d hugepages)", eIdx,
 			humanize.IBytes(uint64(humanize.MiByte*memSizeFreeMiB)), nrPagesFree,
 			humanize.IBytes(uint64(humanize.MiByte*memSizeReqMiB)), nrPagesRequired)
 		memSizeMiB = memSizeFreeMiB
 	}
 
 	// Set hugepage_size (MiB) values based on hugepage info.
-	srv.log.Debugf("Per-engine MemSize:%dMB, HugepageSize:%dMB (meminfo: %s)", memSizeMiB,
-		pageSizeMiB, smi.Summary())
+	srv.log.Debugf("Engine %d Per-engine MemSize:%dMiB, HugepageSize:%dMiB (meminfo: %s)",
+		ei.Index(), memSizeMiB, pageSizeMiB, smi.Summary())
 	ei.setMemSize(memSizeMiB)
 	ei.setHugepageSz(pageSizeMiB)
+
+	return nil
 }
 
 // Clean SPDK resources, both lockfiles and orphaned hugepages. Orphaned hugepages will be cleaned
@@ -697,7 +703,9 @@ func registerEngineEventCallbacks(srv *server, engine *EngineInstance, allStarte
 		}
 
 		// Update engine memory related config parameters before starting.
-		setEngineMemSize(srv, engine, smi)
+		if err := setEngineMemSize(srv, engine, smi); err != nil {
+			return errors.Wrap(err, "set engine mem_size value")
+		}
 
 		// Check available RAM can satisfy tmpfs size before starting a new engine.
 		if err := checkEngineTmpfsMem(srv, engine, smi); err != nil {

src/control/server/server_utils_test.go

@@ -1331,7 +1331,10 @@ func TestServer_prepBdevStorage_setEngineMemSize(t *testing.T) {
 			runner := engine.NewRunner(log, srv.cfg.Engines[0])
 			ei := NewEngineInstance(log, srv.ctlSvc.storage, nil, runner, nil)
 
-			setEngineMemSize(srv, ei, tc.memInfo2)
+			err = setEngineMemSize(srv, ei, tc.memInfo2)
+			if err != nil {
+				t.Fatalf("setEngineMemSize failed: %v", err)
+			}
 
 			test.AssertEqual(t, tc.expMemSize, ei.runner.GetConfig().MemSize,
 				"unexpected memory size")

src/engine/init.c

@@ -62,7 +62,7 @@ const char	       *dss_nvme_conf;
 /** Socket Directory */
 const char	       *dss_socket_dir = "/var/run/daos_server";
 
-/** NVMe mem_size for SPDK memory allocation */
+/** NVMe mem_size for SPDK memory allocation (MiB) */
 unsigned int		dss_nvme_mem_size = DAOS_NVME_MEM_PRIMARY;
 
 /** NVMe hugepage_size for DPDK/SPDK memory allocation */