From 71199fb21333f0c2bd3ceb508334567dec545dde Mon Sep 17 00:00:00 2001 From: tginart Date: Sun, 7 Jun 2020 22:17:58 -0700 Subject: [PATCH 01/18] bugfixes for mixd --- dlrm_s_pytorch.py | 4 ++-- tricks/md_embedding_bag.py | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 1bb09784..1955bb9c 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -179,9 +179,9 @@ def create_emb(self, m, ln): if self.qr_flag and n > self.qr_threshold: EE = QREmbeddingBag(n, m, self.qr_collisions, operation=self.qr_operation, mode="sum", sparse=True) - elif self.md_flag and n > self.md_threshold: - _m = m[i] + elif self.md_flag: base = max(m) + _m = m[i] if n > self.md_threshold else base EE = PrEmbeddingBag(n, _m, base) # use np initialization as below for consistency... W = np.random.uniform( diff --git a/tricks/md_embedding_bag.py b/tricks/md_embedding_bag.py index 53c9f7af..fffc2289 100644 --- a/tricks/md_embedding_bag.py +++ b/tricks/md_embedding_bag.py @@ -34,10 +34,14 @@ def md_solver(n, alpha, d0=None, B=None, round_dim=True, k=None): d = alpha_power_rule(n.type(torch.float) / k, alpha, d0=d0, B=B) if round_dim: d = pow_2_round(d) - return d + undo_sort = [0] * len(indices) + for i, v in enumerate(indices): + undo_sort[v] = i + return d[undo_sort] def alpha_power_rule(n, alpha, d0=None, B=None): + if d0 is not None: lamb = d0 * (n[0].type(torch.float) ** alpha) elif B is not None: From 0341ca3fbec6715c0deaebe36f676e8515735122 Mon Sep 17 00:00:00 2001 From: tginart Date: Sun, 7 Jun 2020 22:19:39 -0700 Subject: [PATCH 02/18] remove whitespace --- tricks/md_embedding_bag.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tricks/md_embedding_bag.py b/tricks/md_embedding_bag.py index fffc2289..7c4071a2 100644 --- a/tricks/md_embedding_bag.py +++ b/tricks/md_embedding_bag.py @@ -41,7 +41,6 @@ def md_solver(n, alpha, d0=None, B=None, round_dim=True, k=None): def alpha_power_rule(n, alpha, d0=None, B=None): - if d0 is not None: lamb = d0 * (n[0].type(torch.float) ** alpha) elif B is not None: From 9349a4d1555448a863bab220bccfee5978d00179 Mon Sep 17 00:00:00 2001 From: tginart Date: Sun, 30 Aug 2020 23:04:16 -0700 Subject: [PATCH 03/18] switch mixd to dense adam --- bench/dlrm_s_criteo_kaggle.sh | 8 +------- dlrm_s_pytorch.py | 11 +++++++---- tricks/md_embedding_bag.py | 4 ++-- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/bench/dlrm_s_criteo_kaggle.sh b/bench/dlrm_s_criteo_kaggle.sh index 867d8c01..8ba3d6e1 100755 --- a/bench/dlrm_s_criteo_kaggle.sh +++ b/bench/dlrm_s_criteo_kaggle.sh @@ -15,18 +15,12 @@ fi #echo $dlrm_extra_option dlrm_pt_bin="python dlrm_s_pytorch.py" -dlrm_c2_bin="python dlrm_s_caffe2.py" echo "run pytorch ..." # WARNING: the following parameters will be set based on the data set # --arch-embedding-size=... (sparse feature sizes) # --arch-mlp-bot=... (the input to the first layer of bottom mlp) -$dlrm_pt_bin --arch-sparse-feature-size=16 --arch-mlp-bot="13-512-256-64-16" --arch-mlp-top="512-256-1" --data-generation=dataset --data-set=kaggle --raw-data-file=./input/train.txt --processed-data-file=./input/kaggleAdDisplayChallenge_processed.npz --loss-function=bce --round-targets=True --learning-rate=0.1 --mini-batch-size=128 --print-freq=1024 --print-time --test-mini-batch-size=16384 --test-num-workers=16 $dlrm_extra_option 2>&1 | tee run_kaggle_pt.log +$dlrm_pt_bin --arch-sparse-feature-size=16 --arch-mlp-bot="13-512-256-64-16" --arch-mlp-top="512-256-1" --data-generation=dataset --data-set=kaggle --raw-data-file=./input/train.txt --processed-data-file=./input/kaggleAdDisplayChallenge_processed.npz --loss-function=bce --round-targets=True --learning-rate=0.1 --mini-batch-size=256 --print-freq=1024 --print-time --test-freq=4096 --test-mini-batch-size=16384 --test-num-workers=7 $dlrm_extra_option 2>&1 | tee run_kaggle_pt.log -echo "run caffe2 ..." -# WARNING: the following parameters will be set based on the data set -# --arch-embedding-size=... (sparse feature sizes) -# --arch-mlp-bot=... (the input to the first layer of bottom mlp) -$dlrm_c2_bin --arch-sparse-feature-size=16 --arch-mlp-bot="13-512-256-64-16" --arch-mlp-top="512-256-1" --data-generation=dataset --data-set=kaggle --raw-data-file=./input/train.txt --processed-data-file=./input/kaggleAdDisplayChallenge_processed.npz --loss-function=bce --round-targets=True --learning-rate=0.1 --mini-batch-size=128 --print-freq=1024 --print-time $dlrm_extra_option 2>&1 | tee run_kaggle_c2.log echo "done" diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 1955bb9c..1014d8c3 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -181,7 +181,9 @@ def create_emb(self, m, ln): operation=self.qr_operation, mode="sum", sparse=True) elif self.md_flag: base = max(m) + print(f"base: {base}") _m = m[i] if n > self.md_threshold else base + print(f"emb size: {_m}") EE = PrEmbeddingBag(n, _m, base) # use np initialization as below for consistency... W = np.random.uniform( @@ -190,7 +192,7 @@ def create_emb(self, m, ln): EE.embs.weight.data = torch.tensor(W, requires_grad=True) else: - EE = nn.EmbeddingBag(n, m, mode="sum", sparse=True) + EE = nn.EmbeddingBag(n, m, mode="sum", sparse=False) # initialize embeddings # nn.init.uniform_(EE.weight, a=-np.sqrt(1 / n), b=np.sqrt(1 / n)) @@ -684,6 +686,7 @@ def parallel_forward(self, dense_x, lS_o, lS_i): d0=m_spa, round_dim=args.md_round_dims ).tolist() + print(m_spa) # test prints (model arch) if args.debug_mode: @@ -790,7 +793,7 @@ def parallel_forward(self, dense_x, lS_o, lS_i): if not args.inference_only: # specify the optimizer algorithm - optimizer = torch.optim.SGD(dlrm.parameters(), lr=args.learning_rate) + optimizer = torch.optim.Adam(dlrm.parameters(), lr=args.learning_rate) lr_scheduler = LRPolicyScheduler(optimizer, args.lr_num_warmup_steps, args.lr_decay_start_step, args.lr_num_decay_steps) @@ -1009,7 +1012,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): print( "Finished {} it {}/{} of epoch {}, {:.2f} ms/it, ".format( str_run_type, j + 1, nbatches, k, gT - ) + ,flush=True) + "loss {:.6f}, accuracy {:3.3f} %".format(gL, gA * 100) ) # Uncomment the line below to print out the total time with overhead @@ -1174,7 +1177,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): print( "Testing at - {}/{} of epoch {},".format(j + 1, nbatches, 0) + " loss {:.6f}, accuracy {:3.3f} %, best {:3.3f} %".format( - gL_test, gA_test * 100, best_gA_test * 100 + gL_test, gA_test * 100, best_gA_test * 100,flush=True ) ) # Uncomment the line below to print out the total time with overhead diff --git a/tricks/md_embedding_bag.py b/tricks/md_embedding_bag.py index 7c4071a2..9935ed4e 100644 --- a/tricks/md_embedding_bag.py +++ b/tricks/md_embedding_bag.py @@ -57,14 +57,14 @@ def alpha_power_rule(n, alpha, d0=None, B=None): def pow_2_round(dims): - return 2 ** torch.round(torch.log2(dims.type(torch.float))) + return (2 ** torch.round(torch.log2(dims.type(torch.float)))).long() class PrEmbeddingBag(nn.Module): def __init__(self, num_embeddings, embedding_dim, base_dim): super(PrEmbeddingBag, self).__init__() self.embs = nn.EmbeddingBag( - num_embeddings, embedding_dim, mode="sum", sparse=True) + num_embeddings, embedding_dim, mode="sum", sparse=False) torch.nn.init.xavier_uniform_(self.embs.weight) if embedding_dim < base_dim: self.proj = nn.Linear(embedding_dim, base_dim, bias=False) From 3539ade3049328790c5b4e11d869bf1418ff0224 Mon Sep 17 00:00:00 2001 From: tginart Date: Mon, 31 Aug 2020 13:20:39 -0700 Subject: [PATCH 04/18] achieves 79.165 with alpha 0.3 and base 1024 --- dlrm_s_pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 1014d8c3..f305546c 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -998,7 +998,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): ) # print time, loss and accuracy - if should_print or should_test: + if True or should_test: gT = 1000.0 * total_time / total_iter if args.print_time else -1 total_time = 0 From 09be5ea96babb5c7d57ce191272cf51bf78ba7b5 Mon Sep 17 00:00:00 2001 From: tginart Date: Mon, 14 Sep 2020 18:17:47 -0700 Subject: [PATCH 05/18] half prec commit --- dlrm_s_pytorch.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index f305546c..a381c222 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -530,6 +530,7 @@ def parallel_forward(self, dense_x, lS_o, lS_i): parser.add_argument("--print-precision", type=int, default=5) parser.add_argument("--numpy-rand-seed", type=int, default=123) parser.add_argument("--sync-dense-params", type=bool, default=True) + parser.add_argument("--use-half-precision", action="store_true", default=False) # inference parser.add_argument("--inference-only", action="store_true", default=False) # onnx @@ -780,11 +781,14 @@ def parallel_forward(self, dense_x, lS_o, lS_i): if dlrm.ndevices > 1: dlrm.emb_l = dlrm.create_emb(m_spa, ln_emb) + if args.use_half_precision: + dlrm.half() + # specify the loss function if args.loss_function == "mse": loss_fn = torch.nn.MSELoss(reduction="mean") elif args.loss_function == "bce": - loss_fn = torch.nn.BCELoss(reduction="mean") + loss_fn = lambda x,y: torch.nn.functional.binary_cross_entropy(x,y,reduction="mean") elif args.loss_function == "wbce": loss_ws = torch.tensor(np.fromstring(args.loss_weights, dtype=float, sep="-")) loss_fn = torch.nn.BCELoss(reduction="none") @@ -921,6 +925,11 @@ def loss_fn_wrap(Z, T, use_gpu, device): previous_iteration_time = None for j, (X, lS_o, lS_i, T) in enumerate(train_ld): + + if args.use_half_precision: + X = X.half() + T = T.half() + if j < skip_upto_batch: continue From dd31c7b07f2a4238847cc05c3aeaa5f14e85a0be Mon Sep 17 00:00:00 2001 From: tginart Date: Mon, 14 Sep 2020 18:44:24 -0700 Subject: [PATCH 06/18] run model in 16bit but compute loss in 32bit --- dlrm_s_pytorch.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index a381c222..bc4b3b71 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -788,7 +788,7 @@ def parallel_forward(self, dense_x, lS_o, lS_i): if args.loss_function == "mse": loss_fn = torch.nn.MSELoss(reduction="mean") elif args.loss_function == "bce": - loss_fn = lambda x,y: torch.nn.functional.binary_cross_entropy(x,y,reduction="mean") + loss_fn = torch.nn.BCELoss() elif args.loss_function == "wbce": loss_ws = torch.tensor(np.fromstring(args.loss_weights, dtype=float, sep="-")) loss_fn = torch.nn.BCELoss(reduction="none") @@ -824,6 +824,8 @@ def dlrm_wrap(X, lS_o, lS_i, use_gpu, device): return dlrm(X, lS_o, lS_i) def loss_fn_wrap(Z, T, use_gpu, device): + if args.use_half_precision: + Z = Z.float() if args.loss_function == "mse" or args.loss_function == "bce": if use_gpu: return loss_fn(Z, T.to(device)) @@ -928,7 +930,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): if args.use_half_precision: X = X.half() - T = T.half() + #T = T.half() if j < skip_upto_batch: continue From dcd68dd5d5bae699844523935143dbc0d0fdac2b Mon Sep 17 00:00:00 2001 From: tginart Date: Mon, 14 Sep 2020 23:34:00 -0700 Subject: [PATCH 07/18] amp version --- dlrm_s_pytorch.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index bc4b3b71..53e101d6 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -93,6 +93,12 @@ from torch.optim.lr_scheduler import _LRScheduler +try: + from apex import amp +except ImportError: + raise ImportError("Please install apex from https://www.github.com/nvidia/apex to run this example.") + + exc = getattr(builtins, "IOError", "FileNotFoundError") class LRPolicyScheduler(_LRScheduler): @@ -781,9 +787,6 @@ def parallel_forward(self, dense_x, lS_o, lS_i): if dlrm.ndevices > 1: dlrm.emb_l = dlrm.create_emb(m_spa, ln_emb) - if args.use_half_precision: - dlrm.half() - # specify the loss function if args.loss_function == "mse": loss_fn = torch.nn.MSELoss(reduction="mean") @@ -801,6 +804,11 @@ def parallel_forward(self, dense_x, lS_o, lS_i): lr_scheduler = LRPolicyScheduler(optimizer, args.lr_num_warmup_steps, args.lr_decay_start_step, args.lr_num_decay_steps) + + if args.use_half_precision: + dlrm, optimizer = amp.initialize(dlrm, optimizer, opt_level='O3') + + ### main loop ### def time_wrap(use_gpu): if use_gpu: @@ -929,7 +937,8 @@ def loss_fn_wrap(Z, T, use_gpu, device): for j, (X, lS_o, lS_i, T) in enumerate(train_ld): if args.use_half_precision: - X = X.half() + pass + #X = X.half() #T = T.half() if j < skip_upto_batch: @@ -981,7 +990,8 @@ def loss_fn_wrap(Z, T, use_gpu, device): # (where we do not accumulate gradients across mini-batches) optimizer.zero_grad() # backward pass - E.backward() + with amp.scale_loss(E, optimizer) as scaled_loss: + scaled_loss.backward() # debug prints (check gradient norm) # for l in mlp.layers: # if hasattr(l, 'weight'): From f27fa0c86ce81f2ace3c93a6bbda9a006dc41baf Mon Sep 17 00:00:00 2001 From: tginart Date: Mon, 14 Sep 2020 23:40:59 -0700 Subject: [PATCH 08/18] amp control with argparse --- dlrm_s_pytorch.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 53e101d6..4a02d95f 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -832,8 +832,6 @@ def dlrm_wrap(X, lS_o, lS_i, use_gpu, device): return dlrm(X, lS_o, lS_i) def loss_fn_wrap(Z, T, use_gpu, device): - if args.use_half_precision: - Z = Z.float() if args.loss_function == "mse" or args.loss_function == "bce": if use_gpu: return loss_fn(Z, T.to(device)) @@ -990,8 +988,11 @@ def loss_fn_wrap(Z, T, use_gpu, device): # (where we do not accumulate gradients across mini-batches) optimizer.zero_grad() # backward pass - with amp.scale_loss(E, optimizer) as scaled_loss: - scaled_loss.backward() + if args.use_half_precision: + with amp.scale_loss(E, optimizer) as scaled_loss: + scaled_loss.backward() + else: + E.backward() # debug prints (check gradient norm) # for l in mlp.layers: # if hasattr(l, 'weight'): From bba11095776b7feb46c81637b50c203e8c3075d4 Mon Sep 17 00:00:00 2001 From: tginart Date: Tue, 15 Sep 2020 17:29:14 -0700 Subject: [PATCH 09/18] add amsgrad --- dlrm_s_pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 4a02d95f..9a26de07 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -800,7 +800,7 @@ def parallel_forward(self, dense_x, lS_o, lS_i): if not args.inference_only: # specify the optimizer algorithm - optimizer = torch.optim.Adam(dlrm.parameters(), lr=args.learning_rate) + optimizer = torch.optim.Adam(dlrm.parameters(), lr=args.learning_rate, amsgrad=True) lr_scheduler = LRPolicyScheduler(optimizer, args.lr_num_warmup_steps, args.lr_decay_start_step, args.lr_num_decay_steps) From 1bea177587b7cc60731acf9d7d054c5112c0b1d5 Mon Sep 17 00:00:00 2001 From: tginart Date: Fri, 18 Sep 2020 03:12:22 -0700 Subject: [PATCH 10/18] remove amp --- dlrm_s_pytorch.py | 80 +++++++++++++++++++++++++++++------------------ 1 file changed, 49 insertions(+), 31 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 9a26de07..0ac0f679 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -93,14 +93,22 @@ from torch.optim.lr_scheduler import _LRScheduler -try: - from apex import amp -except ImportError: - raise ImportError("Please install apex from https://www.github.com/nvidia/apex to run this example.") - exc = getattr(builtins, "IOError", "FileNotFoundError") + +def emb_distrib_heuristic(Rows, Dims, ndevices): + #inputs: 2 parallel lists (Rows, Dims) and an int (ndevices) + #Rows -- list: i-th entry is # of rows in i-th embedding table (int) + #Dims -- list: i-th entry is dim of rows in i-th table (int) + #output: a list of balanced table assignments to device + num_params = torch.tensor(Rows) * torch.tensor(Dims) + num_params = torch.sort(torch.tensor(Rows) * torch.tensor(Dims)) + + + + + class LRPolicyScheduler(_LRScheduler): def __init__(self, optimizer, num_warmup_steps, decay_start_step, num_decay_steps): self.num_warmup_steps = num_warmup_steps @@ -190,6 +198,7 @@ def create_emb(self, m, ln): print(f"base: {base}") _m = m[i] if n > self.md_threshold else base print(f"emb size: {_m}") + print(f"num rows: {n} ") EE = PrEmbeddingBag(n, _m, base) # use np initialization as below for consistency... W = np.random.uniform( @@ -370,6 +379,25 @@ def sequential_forward(self, dense_x, lS_o, lS_i): return z + + def distribute_embs(self, ndevices): + # distribute embeddings (model parallelism) + t_list = [] + for k, emb in enumerate(self.emb_l): + d = torch.device("cuda:" + str(k % ndevices)) + emb.to(d) + t_list.append(emb.to(d)) + self.emb_l = nn.ModuleList(t_list) + + + def distribute_model(self, device_ids, batch_size): + # replicate mlp (data parallelism) + self.bot_l_replicas = replicate(self.bot_l, device_ids) + self.top_l_replicas = replicate(self.top_l, device_ids) + self.parallel_model_batch_size = batch_size + + + def parallel_forward(self, dense_x, lS_o, lS_i): ### prepare model (overwrite) ### # WARNING: # of devices must be >= batch size in parallel_forward call @@ -383,18 +411,11 @@ def parallel_forward(self, dense_x, lS_o, lS_i): if self.parallel_model_is_not_prepared or self.sync_dense_params: # replicate mlp (data parallelism) - self.bot_l_replicas = replicate(self.bot_l, device_ids) - self.top_l_replicas = replicate(self.top_l, device_ids) - self.parallel_model_batch_size = batch_size + self.distribute_model(device_ids, batch_size) if self.parallel_model_is_not_prepared: # distribute embeddings (model parallelism) - t_list = [] - for k, emb in enumerate(self.emb_l): - d = torch.device("cuda:" + str(k % ndevices)) - emb.to(d) - t_list.append(emb.to(d)) - self.emb_l = nn.ModuleList(t_list) + self.distribute_embs(ndevices) self.parallel_model_is_not_prepared = False ### prepare input (overwrite) ### @@ -536,7 +557,6 @@ def parallel_forward(self, dense_x, lS_o, lS_i): parser.add_argument("--print-precision", type=int, default=5) parser.add_argument("--numpy-rand-seed", type=int, default=123) parser.add_argument("--sync-dense-params", type=bool, default=True) - parser.add_argument("--use-half-precision", action="store_true", default=False) # inference parser.add_argument("--inference-only", action="store_true", default=False) # onnx @@ -569,6 +589,7 @@ def parallel_forward(self, dense_x, lS_o, lS_i): parser.add_argument("--lr-num-decay-steps", type=int, default=0) args = parser.parse_args() + if args.mlperf_logging: print('command line args: ', json.dumps(vars(args))) @@ -805,10 +826,6 @@ def parallel_forward(self, dense_x, lS_o, lS_i): args.lr_num_decay_steps) - if args.use_half_precision: - dlrm, optimizer = amp.initialize(dlrm, optimizer, opt_level='O3') - - ### main loop ### def time_wrap(use_gpu): if use_gpu: @@ -925,7 +942,8 @@ def loss_fn_wrap(Z, T, use_gpu, device): with torch.autograd.profiler.profile(args.enable_profiling, use_gpu) as prof: while k < args.nepochs: if k < skip_upto_epoch: - continue + pass + #continue accum_time_begin = time_wrap(use_gpu) @@ -934,13 +952,10 @@ def loss_fn_wrap(Z, T, use_gpu, device): for j, (X, lS_o, lS_i, T) in enumerate(train_ld): - if args.use_half_precision: - pass - #X = X.half() - #T = T.half() if j < skip_upto_batch: - continue + pass + #continue if args.mlperf_logging: current_time = time_wrap(use_gpu) @@ -954,7 +969,8 @@ def loss_fn_wrap(Z, T, use_gpu, device): # early exit if nbatches was set by the user and has been exceeded if nbatches > 0 and j >= nbatches: - break + print('weird condition') + pass ''' # debug prints print("input and targets") @@ -988,11 +1004,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): # (where we do not accumulate gradients across mini-batches) optimizer.zero_grad() # backward pass - if args.use_half_precision: - with amp.scale_loss(E, optimizer) as scaled_loss: - scaled_loss.backward() - else: - E.backward() + E.backward() # debug prints (check gradient norm) # for l in mlp.layers: # if hasattr(l, 'weight'): @@ -1222,6 +1234,8 @@ def loss_fn_wrap(Z, T, use_gpu, device): + " reached, stop training") break + print('in loop') + print('end epoch') k += 1 # nepochs # profiling @@ -1259,3 +1273,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): dlrm_pytorch_onnx = onnx.load("dlrm_s_pytorch.onnx") # check the onnx model onnx.checker.check_model(dlrm_pytorch_onnx) + + + + print('last line in code') From 9bdb7b8e55c1629daae3b670b7724c39a5ff7e27 Mon Sep 17 00:00:00 2001 From: tginart Date: Mon, 28 Sep 2020 21:21:00 -0700 Subject: [PATCH 11/18] add err message for unsupported solvers --- dlrm_s_pytorch.py | 26 ++++++++++++++++++++------ tricks/md_embedding_bag.py | 4 ++-- 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index b0a0fda8..924504ab 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -56,7 +56,7 @@ # miscellaneous import builtins import functools -# import bisect +# import bisec t # import shutil import time import json @@ -194,11 +194,11 @@ def create_emb(self, m, ln): # construct embedding operator if self.qr_flag and n > self.qr_threshold: EE = QREmbeddingBag(n, m, self.qr_collisions, - operation=self.qr_operation, mode="sum", sparse=True) + operation=self.qr_operation, mode="sum", sparse=self.sparse) elif self.md_flag: base = max(m) _m = m[i] if n > self.md_threshold else base - EE = PrEmbeddingBag(n, _m, base) + EE = PrEmbeddingBag(n, _m, base, sparse=self.sparse) # use np initialization as below for consistency... W = np.random.uniform( low=-np.sqrt(1 / n), high=np.sqrt(1 / n), size=(n, _m) @@ -206,7 +206,7 @@ def create_emb(self, m, ln): EE.embs.weight.data = torch.tensor(W, requires_grad=True) else: - EE = nn.EmbeddingBag(n, m, mode="sum", sparse=False) + EE = nn.EmbeddingBag(n, m, mode="sum", sparse=self.sparse) # initialize embeddings # nn.init.uniform_(EE.weight, a=-np.sqrt(1 / n), b=np.sqrt(1 / n)) @@ -243,6 +243,7 @@ def __init__( qr_threshold=200, md_flag=False, md_threshold=200, + sparse=True ): super(DLRM_Net, self).__init__() @@ -263,6 +264,7 @@ def __init__( self.arch_interaction_itself = arch_interaction_itself self.sync_dense_params = sync_dense_params self.loss_threshold = loss_threshold + self.sparse = sparse # create variables for QR embedding if applicable self.qr_flag = qr_flag if self.qr_flag: @@ -581,6 +583,7 @@ def dash_separated_floats(value): parser.add_argument("--num-workers", type=int, default=0) parser.add_argument("--memory-map", action="store_true", default=False) # training + parser.add_argument("--solver", type=str, default="sgd") parser.add_argument("--mini-batch-size", type=int, default=1) parser.add_argument("--nepochs", type=int, default=1) parser.add_argument("--learning-rate", type=float, default=0.01) @@ -799,6 +802,7 @@ def dash_separated_floats(value): print(T.detach().cpu().numpy()) ndevices = min(ngpus, args.mini_batch_size, num_fea - 1) if use_gpu else -1 + ndevices = 1 ### construct the neural network specified above ### # WARNING: to obtain exactly the same initialization for @@ -822,6 +826,7 @@ def dash_separated_floats(value): qr_threshold=args.qr_threshold, md_flag=args.md_flag, md_threshold=args.md_threshold, + sparse=False if args.solver == 'amsgrad' else True ) # test prints if args.debug_mode: @@ -851,7 +856,16 @@ def dash_separated_floats(value): if not args.inference_only: # specify the optimizer algorithm - optimizer = torch.optim.Adam(dlrm.parameters(), lr=args.learning_rate, amsgrad=True) + if args.solver == 'sgd': + optimizer = torch.optim.SGD( + dlrm.parameters(), lr=args.learning_rate) + elif args.solver == 'amsgrad': + optimizer = torch.optim.Adam( + dlrm.parameters(), lr=args.learning_rate, amsgrad=True) + else: + raise ValueError( + f'Solver {args.solver} is not supported. Select sgd or amsgrad') + lr_scheduler = LRPolicyScheduler(optimizer, args.lr_num_warmup_steps, args.lr_decay_start_step, args.lr_num_decay_steps) @@ -1371,4 +1385,4 @@ def loss_fn_wrap(Z, T, use_gpu, device): sess = rt.InferenceSession(dlrm_pytorch_onnx_file, rt.SessionOptions()) prediction = sess.run(output_names=["pred"], input_feed=dict_inputs) print("prediction", prediction) - ''' \ No newline at end of file + ''' diff --git a/tricks/md_embedding_bag.py b/tricks/md_embedding_bag.py index 9935ed4e..ef5ea882 100644 --- a/tricks/md_embedding_bag.py +++ b/tricks/md_embedding_bag.py @@ -61,10 +61,10 @@ def pow_2_round(dims): class PrEmbeddingBag(nn.Module): - def __init__(self, num_embeddings, embedding_dim, base_dim): + def __init__(self, num_embeddings, embedding_dim, base_dim, sparse=False): super(PrEmbeddingBag, self).__init__() self.embs = nn.EmbeddingBag( - num_embeddings, embedding_dim, mode="sum", sparse=False) + num_embeddings, embedding_dim, mode="sum", sparse=sparse) torch.nn.init.xavier_uniform_(self.embs.weight) if embedding_dim < base_dim: self.proj = nn.Linear(embedding_dim, base_dim, bias=False) From a1b5067311dbd0f7b63c2132d05ea471dd62e5b2 Mon Sep 17 00:00:00 2001 From: tginart Date: Tue, 29 Sep 2020 00:02:54 -0700 Subject: [PATCH 12/18] add print num embs --- dlrm_s_pytorch.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 924504ab..caab3d9c 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -56,7 +56,7 @@ # miscellaneous import builtins import functools -# import bisec t +# import bisect # import shutil import time import json @@ -597,6 +597,7 @@ def dash_separated_floats(value): # gpu parser.add_argument("--use-gpu", action="store_true", default=False) # debugging and profiling + parser.add_argument("--print-num-emb-params", action="store_true", default=False) parser.add_argument("--print-freq", type=int, default=1) parser.add_argument("--test-freq", type=int, default=-1) parser.add_argument("--test-mini-batch-size", type=int, default=-1) @@ -749,6 +750,13 @@ def dash_separated_floats(value): ).tolist() print(m_spa) + if args.print_num_emb_params: + num_params = int(sum(torch.tensor(ln_emb) * m_spa)) + if isinstance(m_spa, list): + num_params += int(sum(torch.tensor(m_spa)*max(m_spa))) + print(f"Num of params in embedding layer {num_params}") + + # test prints (model arch) if args.debug_mode: print("model arch:") From 3c9ffbef8bf401e2e80a97aa169985d1dbe7fc5a Mon Sep 17 00:00:00 2001 From: tginart Date: Tue, 29 Sep 2020 22:18:47 -0700 Subject: [PATCH 13/18] DLRM --- dlrm_s_pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 769c83af..70d6731c 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -756,7 +756,7 @@ def dash_separated_floats(value): print(m_spa) if args.print_num_emb_params: - num_params = int(sum(torch.tensor(ln_emb) * m_spa)) + num_params = int(sum(torch.tensor(ln_emb) * torch.tensor(m_spa))) if isinstance(m_spa, list): num_params += int(sum(torch.tensor(m_spa)*max(m_spa))) print(f"Num of params in embedding layer {num_params}") From b3a824746b00be943d4662ceb0f9c511a00bdb4d Mon Sep 17 00:00:00 2001 From: tginart Date: Thu, 1 Oct 2020 19:29:07 -0700 Subject: [PATCH 14/18] distrib --- dlrm_s_pytorch.py | 55 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 70d6731c..18de9194 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -97,18 +97,6 @@ exc = getattr(builtins, "IOError", "FileNotFoundError") -def emb_distrib_heuristic(Rows, Dims, ndevices): - #inputs: 2 parallel lists (Rows, Dims) and an int (ndevices) - #Rows -- list: i-th entry is # of rows in i-th embedding table (int) - #Dims -- list: i-th entry is dim of rows in i-th table (int) - #output: a list of balanced table assignments to device - num_params = torch.tensor(Rows) * torch.tensor(Dims) - num_params = torch.sort(torch.tensor(Rows) * torch.tensor(Dims)) - - - - - class LRPolicyScheduler(_LRScheduler): def __init__(self, optimizer, num_warmup_steps, decay_start_step, num_decay_steps): self.num_warmup_steps = num_warmup_steps @@ -243,6 +231,7 @@ def __init__( qr_threshold=200, md_flag=False, md_threshold=200, + emb_assignments=None, sparse=True ): super(DLRM_Net, self).__init__() @@ -257,6 +246,7 @@ def __init__( # save arguments self.ndevices = ndevices + self.emb_assignments = emb_assignments self.output_d = 0 self.parallel_model_batch_size = -1 self.parallel_model_is_not_prepared = True @@ -387,7 +377,8 @@ def distribute_embs(self, ndevices): # distribute embeddings (model parallelism) t_list = [] for k, emb in enumerate(self.emb_l): - d = torch.device("cuda:" + str(k % ndevices)) + d = torch.device( + "cuda:" + str(self.emb_assignments[k])) emb.to(d) t_list.append(emb.to(d)) self.emb_l = nn.ModuleList(t_list) @@ -400,7 +391,6 @@ def distribute_model(self, device_ids, batch_size): self.parallel_model_batch_size = batch_size - def parallel_forward(self, dense_x, lS_o, lS_i): ### prepare model (overwrite) ### # WARNING: # of devices must be >= batch size in parallel_forward call @@ -557,6 +547,7 @@ def dash_separated_floats(value): parser.add_argument("--qr-threshold", type=int, default=200) parser.add_argument("--qr-operation", type=str, default="mult") parser.add_argument("--qr-collisions", type=int, default=4) + parser.add_argument("--use-emb-distrib-heuristic", action='store_true', default=False) # activations and loss parser.add_argument("--activation-function", type=str, default="relu") parser.add_argument("--loss-function", type=str, default="mse") # or bce or wbce @@ -758,10 +749,12 @@ def dash_separated_floats(value): if args.print_num_emb_params: num_params = int(sum(torch.tensor(ln_emb) * torch.tensor(m_spa))) if isinstance(m_spa, list): - num_params += int(sum(torch.tensor(m_spa)*max(m_spa))) + _m_spa = torch.tensor(m_spa) + has_proj = _m_spa < max(_m_spa) + num_params += int(torch.sum(has_proj*_m_spa)*max(m_spa)) print(f"Num of params in embedding layer {num_params}") - + # test prints (model arch) if args.debug_mode: print("model arch:") @@ -815,7 +808,34 @@ def dash_separated_floats(value): print(T.detach().cpu().numpy()) ndevices = min(ngpus, args.mini_batch_size, num_fea - 1) if use_gpu else -1 - ndevices = 1 + #ndevices = 1 + + if args.use_emb_distrib_heuristic: + + def emb_distrib_heuristic(Rows, Dims, ndevices): + #inputs: 2 parallel lists (Rows, Dims) and an int (ndevices) + #Rows--list: i-th entry is # of rows in i-th emb table (int) + #Dims--list: i-th entry is dim of rows in i-th table (int) + #output: a list of balanced table assignments to device + num_params = torch.tensor(Rows) * torch.tensor(Dims) + cur_load = torch.zeros(ndevices) + assignments = [0]*len(Rows) + val, idx = torch.sort(num_params, descending=True) + for i,v in enumerate(val): + a = torch.argmin(cur_load) + assignments[idx[i]] = int(a) + cur_load[a] += v + return assignments + + _m_spa = m_spa if isinstance( + m_spa, list) else [m_spa]*len(ln_emb) + + assignments = emb_distrib_heuristic(ln_emb,_m_spa,ndevices) + + else: + assignments = [k % ndevices for k in range(len(ln_emb))] + print(assignments) + ### construct the neural network specified above ### # WARNING: to obtain exactly the same initialization for @@ -839,6 +859,7 @@ def dash_separated_floats(value): qr_threshold=args.qr_threshold, md_flag=args.md_flag, md_threshold=args.md_threshold, + emb_assignments=assignments, sparse=False if args.solver == 'amsgrad' else True ) # test prints From a88c23a4a8f88fd334896cbb990e6d2f79030555 Mon Sep 17 00:00:00 2001 From: tginart Date: Thu, 8 Oct 2020 20:04:48 -0700 Subject: [PATCH 15/18] running correctly with emb assign heuristic --- dlrm_s_pytorch.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 18de9194..7266398b 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -422,7 +422,8 @@ def parallel_forward(self, dense_x, lS_o, lS_i): t_list = [] i_list = [] for k, _ in enumerate(self.emb_l): - d = torch.device("cuda:" + str(k % ndevices)) + dev_id = self.emb_assignments + d = torch.device("cuda:" + str(self.emb_assignments[k])) t_list.append(lS_o[k].to(d)) i_list.append(lS_i[k].to(d)) lS_o = t_list From 1990e22f62fcb761b22b3d656060acdf50b88b91 Mon Sep 17 00:00:00 2001 From: tginart Date: Fri, 9 Oct 2020 14:16:02 -0700 Subject: [PATCH 16/18] remove println --- dlrm_s_pytorch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 7266398b..13aeca0a 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -1313,7 +1313,6 @@ def loss_fn_wrap(Z, T, use_gpu, device): + " reached, stop training") break - print('in loop') print('end epoch') k += 1 # nepochs From d1304b923e61506b962a0a63c0c91325294a72e6 Mon Sep 17 00:00:00 2001 From: tginart Date: Sun, 18 Oct 2020 18:02:28 -0700 Subject: [PATCH 17/18] ready for PR --- bench/dlrm_s_criteo_kaggle.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) mode change 100755 => 100644 bench/dlrm_s_criteo_kaggle.sh diff --git a/bench/dlrm_s_criteo_kaggle.sh b/bench/dlrm_s_criteo_kaggle.sh old mode 100755 new mode 100644 index 8ba3d6e1..867d8c01 --- a/bench/dlrm_s_criteo_kaggle.sh +++ b/bench/dlrm_s_criteo_kaggle.sh @@ -15,12 +15,18 @@ fi #echo $dlrm_extra_option dlrm_pt_bin="python dlrm_s_pytorch.py" +dlrm_c2_bin="python dlrm_s_caffe2.py" echo "run pytorch ..." # WARNING: the following parameters will be set based on the data set # --arch-embedding-size=... (sparse feature sizes) # --arch-mlp-bot=... (the input to the first layer of bottom mlp) -$dlrm_pt_bin --arch-sparse-feature-size=16 --arch-mlp-bot="13-512-256-64-16" --arch-mlp-top="512-256-1" --data-generation=dataset --data-set=kaggle --raw-data-file=./input/train.txt --processed-data-file=./input/kaggleAdDisplayChallenge_processed.npz --loss-function=bce --round-targets=True --learning-rate=0.1 --mini-batch-size=256 --print-freq=1024 --print-time --test-freq=4096 --test-mini-batch-size=16384 --test-num-workers=7 $dlrm_extra_option 2>&1 | tee run_kaggle_pt.log +$dlrm_pt_bin --arch-sparse-feature-size=16 --arch-mlp-bot="13-512-256-64-16" --arch-mlp-top="512-256-1" --data-generation=dataset --data-set=kaggle --raw-data-file=./input/train.txt --processed-data-file=./input/kaggleAdDisplayChallenge_processed.npz --loss-function=bce --round-targets=True --learning-rate=0.1 --mini-batch-size=128 --print-freq=1024 --print-time --test-mini-batch-size=16384 --test-num-workers=16 $dlrm_extra_option 2>&1 | tee run_kaggle_pt.log +echo "run caffe2 ..." +# WARNING: the following parameters will be set based on the data set +# --arch-embedding-size=... (sparse feature sizes) +# --arch-mlp-bot=... (the input to the first layer of bottom mlp) +$dlrm_c2_bin --arch-sparse-feature-size=16 --arch-mlp-bot="13-512-256-64-16" --arch-mlp-top="512-256-1" --data-generation=dataset --data-set=kaggle --raw-data-file=./input/train.txt --processed-data-file=./input/kaggleAdDisplayChallenge_processed.npz --loss-function=bce --round-targets=True --learning-rate=0.1 --mini-batch-size=128 --print-freq=1024 --print-time $dlrm_extra_option 2>&1 | tee run_kaggle_c2.log echo "done" From 89477036f1ec2d5a531180b961950527332178a7 Mon Sep 17 00:00:00 2001 From: tginart Date: Sun, 18 Oct 2020 18:14:53 -0700 Subject: [PATCH 18/18] ready for PR --- dlrm_s_pytorch.py | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index e18824d1..f255aeb1 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -790,7 +790,7 @@ def dash_separated_floats(value): print("data (inputs and targets):") for j, (X, lS_o, lS_i, T) in enumerate(train_ld): - # early exit if nbatches was set by the user and has been exceeded + # early exit if nbatches was set by the user and has been exceeded if nbatches > 0 and j >= nbatches: break @@ -809,7 +809,6 @@ def dash_separated_floats(value): print(T.detach().cpu().numpy()) ndevices = min(ngpus, args.mini_batch_size, num_fea - 1) if use_gpu else -1 - #ndevices = 1 if args.use_emb_distrib_heuristic: @@ -882,7 +881,7 @@ def emb_distrib_heuristic(Rows, Dims, ndevices): if args.loss_function == "mse": loss_fn = torch.nn.MSELoss(reduction="mean") elif args.loss_function == "bce": - loss_fn = torch.nn.BCELoss() + loss_fn = torch.nn.BCELoss(reduction="mean") elif args.loss_function == "wbce": loss_ws = torch.tensor(np.fromstring(args.loss_weights, dtype=float, sep="-")) loss_fn = torch.nn.BCELoss(reduction="none") @@ -1021,8 +1020,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): with torch.autograd.profiler.profile(args.enable_profiling, use_gpu) as prof: while k < args.nepochs: if k < skip_upto_epoch: - pass - #continue + continue accum_time_begin = time_wrap(use_gpu) @@ -1048,8 +1046,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): # early exit if nbatches was set by the user and has been exceeded if nbatches > 0 and j >= nbatches: - print('weird condition') - pass + break ''' # debug prints print("input and targets") @@ -1111,7 +1108,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): ) # print time, loss and accuracy - if True or should_test: + if should_print or should_test: gT = 1000.0 * total_time / total_iter if args.print_time else -1 total_time = 0 @@ -1124,8 +1121,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): str_run_type = "inference" if args.inference_only else "training" print( "Finished {} it {}/{} of epoch {}, {:.2f} ms/it, ".format( - str_run_type, j + 1, nbatches, k, gT - ,flush=True) + str_run_type, j + 1, nbatches, k, gT) + "loss {:.6f}, accuracy {:3.3f} %".format(gL, gA * 100) ) # Uncomment the line below to print out the total time with overhead @@ -1290,7 +1286,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): print( "Testing at - {}/{} of epoch {},".format(j + 1, nbatches, 0) + " loss {:.6f}, accuracy {:3.3f} %, best {:3.3f} %".format( - gL_test, gA_test * 100, best_gA_test * 100,flush=True + gL_test, gA_test * 100, best_gA_test * 100 ) ) # Uncomment the line below to print out the total time with overhead @@ -1313,7 +1309,6 @@ def loss_fn_wrap(Z, T, use_gpu, device): + " reached, stop training") break - print('end epoch') k += 1 # nepochs # profiling