From bdba86d8df19dbe5322ce35106a5edc559ff00cc Mon Sep 17 00:00:00 2001 From: mnaumovfb Date: Sun, 26 Jan 2020 12:16:17 -0800 Subject: [PATCH 01/57] Adding support for binary loader proposed in pull request https://github.com/facebookresearch/dlrm/pull/49 Also, adding an option for early stopping based on AUC. Finally, adding exact model config for mlperf in run_and_time.sh script. Summary: Adding support for binary loader proposed in pull request https://github.com/facebookresearch/dlrm/pull/49 Also, adding an option for early stopping based on AUC. Finally, adding exact model config for mlperf in run_and_time.sh script. Test Plan: Reviewers: Subscribers: Tasks: Tags: --- bench/run_and_time.sh | 19 ++++ data_loader_terabyte.py | 195 +++++++++++++++++++++++++++++++++++++--- dlrm_data_pytorch.py | 151 ++++++++++++++++++++++++++----- dlrm_s_pytorch.py | 38 ++++++-- 4 files changed, 362 insertions(+), 41 deletions(-) create mode 100755 bench/run_and_time.sh diff --git a/bench/run_and_time.sh b/bench/run_and_time.sh new file mode 100755 index 00000000..492f3d80 --- /dev/null +++ b/bench/run_and_time.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +# +#WARNING: must have compiled PyTorch and caffe2 + +#check if extra argument is passed to the test +if [[ $# == 1 ]]; then + dlrm_extra_option=$1 +else + dlrm_extra_option="" +fi +#echo $dlrm_extra_option + +python dlrm_s_pytorch.py --arch-sparse-feature-size=128 --arch-mlp-bot="13-512-256-128" --arch-mlp-top="1024-1024-512-256-1" --max-ind-range=40000000 --data-generation=dataset --data-set=terabyte --raw-data-file=./input/day --processed-data-file=./input/terabyte_processed.npz --loss-function=bce --round-targets=True --learning-rate=1.0 --mini-batch-size=2048 --print-freq=2048 --print-time --test-freq=102400 --test-mini-batch-size=16384 --test-num-workers=16 --memory-map --mlperf-logging --mlperf-auc-threshold=0.8025 --mlperf-bin-file --mlperf-bin-shuffle $dlrm_extra_option 2>&1 | tee run_terabyte_mlperf_pt.log + +echo "done" diff --git a/data_loader_terabyte.py b/data_loader_terabyte.py index 22657cea..42ff8a49 100644 --- a/data_loader_terabyte.py +++ b/data_loader_terabyte.py @@ -8,9 +8,12 @@ import os import numpy as np +from torch.utils.data import Dataset import torch import time import math +from tqdm import tqdm +import argparse class DataLoader: @@ -24,7 +27,7 @@ def __init__( data_directory, days, batch_size, - split = "train", + split="train", drop_last_batch=False ): self.data_filename = data_filename @@ -56,6 +59,23 @@ def __len__(self): return math.ceil(self.length / self.batch_size) +def _transform_features(x_int_batch, x_cat_batch, y_batch, flag_input_torch_tensor=False): + if flag_input_torch_tensor: + x_int_batch = torch.log(x_int_batch.clone().detach().type(torch.float) + 1) + x_cat_batch = x_cat_batch.clone().detach().type(torch.long) + y_batch = y_batch.clone().detach().type(torch.float32).view(-1, 1) + else: + x_int_batch = torch.log(torch.tensor(x_int_batch, dtype=torch.float) + 1) + x_cat_batch = torch.tensor(x_cat_batch, dtype=torch.long) + y_batch = torch.tensor(y_batch, dtype=torch.float32).view(-1, 1) + + batch_size = x_cat_batch.shape[0] + feature_count = x_cat_batch.shape[1] + lS_o = torch.arange(batch_size).reshape(1, -1).repeat(feature_count, 1) + + return x_int_batch, lS_o, x_cat_batch.t(), y_batch.view(-1, 1) + + def _batch_generator(data_filename, data_directory, days, batch_size, split, drop_last): previous_file = None for day in days: @@ -103,7 +123,6 @@ def _batch_generator(data_filename, data_directory, days, batch_size, split, dro y_batch = np.concatenate([previous_file['y'], y_batch], axis=0) previous_file = None - if x_int_batch.shape[0] != batch_size: raise ValueError('should not happen') @@ -137,18 +156,6 @@ def _batch_generator(data_filename, data_directory, days, batch_size, split, dro previous_file['y']) -def _transform_features(x_int_batch, x_cat_batch, y_batch): - x_int_batch = torch.log(torch.tensor(x_int_batch, dtype=torch.float) + 1) - x_cat_batch = torch.tensor(x_cat_batch, dtype=torch.long) - y_batch = torch.tensor(y_batch, dtype=torch.float32).view(-1, 1) - - batch_size = x_cat_batch.shape[0] - feature_count = x_cat_batch.shape[1] - lS_o = torch.arange(batch_size).reshape(1, -1).repeat(feature_count, 1) - - return x_int_batch, lS_o, x_cat_batch.t(), y_batch.view(-1, 1) - - def _test(): generator = _batch_generator( data_filename='day', @@ -169,5 +176,165 @@ def _test(): ) +class CriteoBinDataset(Dataset): + """Binary version of criteo dataset.""" + + def __init__(self, data_file, counts_file, batch_size=1, bytes_per_feature=4): + # dataset + self.tar_fea = 1 # single target + self.den_fea = 13 # 13 dense features + self.spa_fea = 26 # 26 sparse features + self.tad_fea = self.tar_fea + self.den_fea + self.tot_fea = self.tad_fea + self.spa_fea + + self.batch_size = batch_size + self.bytes_per_entry = (bytes_per_feature * self.tot_fea * batch_size) + + self.num_entries = math.ceil(os.path.getsize(data_file) / self.bytes_per_entry) + + print('data file:', data_file, 'number of batches:', self.num_entries) + self.file = open(data_file, 'rb') + + with np.load(counts_file) as data: + self.counts = data["counts"] + + # hardcoded for now + self.m_den = 13 + + def __len__(self): + return self.num_entries + + def __getitem__(self, idx): + self.file.seek(idx * self.bytes_per_entry, 0) + raw_data = self.file.read(self.bytes_per_entry) + array = np.frombuffer(raw_data, dtype=np.float32) + tensor = torch.from_numpy(array).view((-1, self.tot_fea)) + + return _transform_features(x_int_batch=tensor[:, 1:14], + x_cat_batch=tensor[:, 14:], + y_batch=tensor[:, 0], + flag_input_torch_tensor=True) + + +def numpy_to_binary(input_files, output_file_path, split='train'): + """Convert the data to a binary format to be read with CriteoBinDataset.""" + + with open(output_file_path, 'wb') as output_file: + if split == 'train': + for input_file in input_files: + print('Processing file: ', input_file) + + np_data = np.load(input_file) + np_data = np.concatenate([np_data['y'].reshape(-1, 1), + np_data['X_int'], + np_data['X_cat']], axis=1) + np_data = np_data.astype(np.float32) + + output_file.write(np_data.tobytes()) + else: + assert len(input_files) == 1 + np_data = np.load(input_files[0]) + np_data = np.concatenate([np_data['y'].reshape(-1, 1), + np_data['X_int'], + np_data['X_cat']], axis=1) + np_data = np_data.astype(np.float32) + + samples_in_file = np_data.shape[0] + midpoint = int(np.ceil(samples_in_file / 2.)) + if split == "test": + begin = 0 + end = midpoint + elif split == "val": + begin = midpoint + end = samples_in_file + else: + raise ValueError('Unknown split value: ', split) + + output_file.write(np_data[begin:end].tobytes()) + + +def _preprocess(args): + train_files = ['{}_{}_reordered.npz'.format(args.input_data_prefix, day) for + day in range(0, 23)] + + test_valid_file = args.input_data_prefix + '_23_reordered.npz' + + os.makedirs(args.output_directory, exist_ok=True) + for split in ['train', 'val', 'test']: + print('Running preprocessing for split =', split) + + output_file = os.path.join(args.output_directory, + '{}_data.bin'.format(split)) + + input_files = train_files if split == 'train' else [test_valid_file] + numpy_to_binary(input_files=input_files, + output_file_path=output_file, + split=split) + + +def _test_bin(): + parser = argparse.ArgumentParser() + parser.add_argument('--output_directory', required=True) + parser.add_argument('--input_data_prefix', required=True) + parser.add_argument('--split', choices=['train', 'test', 'val'], + required=True) + args = parser.parse_args() + + # _preprocess(args) + + binary_data_file = os.path.join(args.output_directory, + '{}_data.bin'.format(args.split)) + + counts_file = os.path.join(args.output_directory, 'day_fea_count.npz') + dataset_binary = CriteoBinDataset(data_file=binary_data_file, + counts_file=counts_file, + batch_size=2048,) + from dlrm_data_pytorch import CriteoDataset, collate_wrapper_criteo + + binary_loader = torch.utils.data.DataLoader( + dataset_binary, + batch_size=None, + shuffle=False, + num_workers=0, + collate_fn=None, + pin_memory=False, + drop_last=False, + ) + + original_dataset = CriteoDataset( + dataset='terabyte', + max_ind_range=10 * 1000 * 1000, + sub_sample_rate=1, + randomize=True, + split=args.split, + raw_path=args.input_data_prefix, + pro_data='dummy_string', + memory_map=True + ) + + original_loader = torch.utils.data.DataLoader( + original_dataset, + batch_size=2048, + shuffle=False, + num_workers=0, + collate_fn=collate_wrapper_criteo, + pin_memory=False, + drop_last=False, + ) + + assert len(dataset_binary) == len(original_loader) + for i, (old_batch, new_batch) in tqdm(enumerate(zip(original_loader, + binary_loader)), + total=len(dataset_binary)): + + for j in range(len(new_batch)): + if not np.array_equal(old_batch[j], new_batch[j]): + raise ValueError('FAILED: Datasets not equal') + if i > len(dataset_binary): + break + print('PASSED') + + if __name__ == '__main__': _test() + _test_bin diff --git a/dlrm_data_pytorch.py b/dlrm_data_pytorch.py index 942850f5..cdd80124 100644 --- a/dlrm_data_pytorch.py +++ b/dlrm_data_pytorch.py @@ -32,7 +32,7 @@ # pytorch import torch -from torch.utils.data import Dataset +from torch.utils.data import Dataset, RandomSampler import data_loader_terabyte @@ -333,9 +333,8 @@ def collate_wrapper_criteo(list_of_tuples): return X_int, torch.stack(lS_o), torch.stack(lS_i), T -def make_criteo_data_and_loaders(args): - - train_data = CriteoDataset( +def ensure_dataset_preprocessed(args, d_path): + _ = CriteoDataset( args.data_set, args.max_ind_range, args.data_sub_sample_rate, @@ -346,7 +345,7 @@ def make_criteo_data_and_loaders(args): args.memory_map ) - test_data = CriteoDataset( + _ = CriteoDataset( args.data_set, args.max_ind_range, args.data_sub_sample_rate, @@ -357,27 +356,139 @@ def make_criteo_data_and_loaders(args): args.memory_map ) + for split in ['train', 'val', 'test']: + print('Running preprocessing for split =', split) + + train_files = ['{}_{}_reordered.npz'.format(args.raw_data_file, day) + for + day in range(0, 23)] + + test_valid_file = args.raw_data_file + '_23_reordered.npz' + + output_file = d_path + '_{}.bin'.format(split) + + input_files = train_files if split == 'train' else [test_valid_file] + data_loader_terabyte.numpy_to_binary(input_files=input_files, + output_file_path=output_file, + split=split) + + +def make_criteo_data_and_loaders(args): + if args.mlperf_logging and args.memory_map and args.data_set == "terabyte": # more efficient for larger batches data_directory = path.dirname(args.raw_data_file) - data_filename = args.raw_data_file.split("/")[-1] - train_loader = data_loader_terabyte.DataLoader( - data_directory=data_directory, - data_filename=data_filename, - days=list(range(23)), - batch_size=args.mini_batch_size, - split="train" + if args.mlperf_bin_loader: + lstr = args.processed_data_file.split("/") + d_path = "/".join(lstr[0:-1]) + "/" + lstr[-1].split(".")[0] + train_file = d_path + "_train.bin" + test_file = d_path + "_test.bin" + # val_file = d_path + "_val.bin" + counts_file = args.raw_data_file + '_fea_count.npz' + + if any(not path.exists(p) for p in [train_file, + test_file, + counts_file]): + ensure_dataset_preprocessed(args, d_path) + + train_data = data_loader_terabyte.CriteoBinDataset( + data_file=train_file, + counts_file=counts_file, + batch_size=args.mini_batch_size + ) + + train_loader = torch.utils.data.DataLoader( + train_data, + batch_size=None, + batch_sampler=None, + shuffle=False, + num_workers=0, + collate_fn=None, + pin_memory=False, + drop_last=False, + sampler=RandomSampler(train_data) if args.mlperf_bin_shuffle else None + ) + + test_data = data_loader_terabyte.CriteoBinDataset( + data_file=test_file, + counts_file=counts_file, + batch_size=args.test_mini_batch_size + ) + + test_loader = torch.utils.data.DataLoader( + test_data, + batch_size=None, + batch_sampler=None, + shuffle=False, + num_workers=0, + collate_fn=None, + pin_memory=False, + drop_last=False, + ) + else: + data_filename = args.raw_data_file.split("/")[-1] + + train_data = CriteoDataset( + args.data_set, + args.max_ind_range, + args.data_sub_sample_rate, + args.data_randomize, + "train", + args.raw_data_file, + args.processed_data_file, + args.memory_map + ) + + test_data = CriteoDataset( + args.data_set, + args.max_ind_range, + args.data_sub_sample_rate, + args.data_randomize, + "test", + args.raw_data_file, + args.processed_data_file, + args.memory_map + ) + + train_loader = data_loader_terabyte.DataLoader( + data_directory=data_directory, + data_filename=data_filename, + days=list(range(23)), + batch_size=args.mini_batch_size, + split="train" + ) + + test_loader = data_loader_terabyte.DataLoader( + data_directory=data_directory, + data_filename=data_filename, + days=[23], + batch_size=args.test_mini_batch_size, + split="test" + ) + else: + train_data = CriteoDataset( + args.data_set, + args.max_ind_range, + args.data_sub_sample_rate, + args.data_randomize, + "train", + args.raw_data_file, + args.processed_data_file, + args.memory_map ) - test_loader = data_loader_terabyte.DataLoader( - data_directory=data_directory, - data_filename=data_filename, - days=[23], - batch_size=args.test_mini_batch_size, - split="test" + test_data = CriteoDataset( + args.data_set, + args.max_ind_range, + args.data_sub_sample_rate, + args.data_randomize, + "test", + args.raw_data_file, + args.processed_data_file, + args.memory_map ) - else: + train_loader = torch.utils.data.DataLoader( train_data, batch_size=args.mini_batch_size, @@ -387,6 +498,7 @@ def make_criteo_data_and_loaders(args): pin_memory=False, drop_last=False, # True ) + test_loader = torch.utils.data.DataLoader( test_data, batch_size=args.test_mini_batch_size, @@ -538,7 +650,6 @@ def make_random_data_and_loader(args, ln_emb, m_den): return train_data, train_loader - def generate_random_data( m_den, ln_emb, diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 78ff9b9c..69bdeb55 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -511,7 +511,12 @@ def parallel_forward(self, dense_x, lS_o, lS_i): parser.add_argument("--load-model", type=str, default="") # mlperf logging (disables other output and stops early) parser.add_argument("--mlperf-logging", action="store_true", default=False) - parser.add_argument("--mlperf-threshold", type=float, default=0.0) # 0.789 # 0.8107 + # stop at target accuracy Kaggle 0.789, Terabyte (sub-sampled=0.875) 0.8107 + parser.add_argument("--mlperf-acc-threshold", type=float, default=0.0) + # stop at target AUC Terabyte (no subsampling) 0.8025 + parser.add_argument("--mlperf-auc-threshold", type=float, default=0.0) + parser.add_argument("--mlperf-bin-loader", action='store_true', default=False) + parser.add_argument("--mlperf-bin-shuffle", action='store_true', default=False) args = parser.parse_args() if args.mlperf_logging: @@ -789,6 +794,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): # training or inference best_gA_test = 0 + best_auc_test = 0 total_time = 0 total_loss = 0 total_accu = 0 @@ -1072,6 +1078,10 @@ def loss_fn_wrap(Z, T, use_gpu, device): ) if args.mlperf_logging: + is_best = validation_results['roc_auc'] > best_auc_test + if is_best: + best_auc_test = validation_results['roc_auc'] + print( "Testing at - {}/{} of epoch {},".format(j + 1, nbatches, k) + " loss {:.6f}, recall {:.4f}, precision {:.4f},".format( @@ -1079,12 +1089,15 @@ def loss_fn_wrap(Z, T, use_gpu, device): validation_results['recall'], validation_results['precision'] ) - + " f1 {:.4f}, ap {:.4f}, roc_auc {:.4f},".format( + + " f1 {:.4f}, ap {:.4f},".format( validation_results['f1'], validation_results['ap'], - validation_results['roc_auc'] ) - + " accuracy {:3.3f} %, best {:3.3f} %".format( + + " auc {:.4f}, best auc {:.4f},".format( + validation_results['roc_auc'], + best_auc_test + ) + + " accuracy {:3.3f} %, best accuracy {:3.3f} %".format( validation_results['accuracy'] * 100, best_gA_test * 100 ) @@ -1100,9 +1113,20 @@ def loss_fn_wrap(Z, T, use_gpu, device): # print("Total test time for this group: {}" \ # .format(time_wrap(use_gpu) - accum_test_time_begin)) - if ((args.mlperf_threshold > 0) and (best_gA_test > args.mlperf_threshold)): - print("MLperf testing accuracy threshold " - + str(args.mlperf_threshold) + " reached, stop training") + if (args.mlperf_logging + and (args.mlperf_acc_threshold > 0) + and (best_gA_test > args.mlperf_acc_threshold)): + print("MLPerf testing accuracy threshold " + + str(args.mlperf_acc_threshold) + + " reached, stop training") + break + + if (args.mlperf_logging + and (args.mlperf_auc_threshold > 0) + and (best_auc_test > args.mlperf_auc_threshold)): + print("MLPerf testing auc threshold " + + str(args.mlperf_auc_threshold) + + " reached, stop training") break k += 1 # nepochs From 58c28067f0edc05b7c29ea23a56195f7cd6278b2 Mon Sep 17 00:00:00 2001 From: mnaumovfb Date: Wed, 5 Feb 2020 13:47:42 -0800 Subject: [PATCH 02/57] Adding support for testing and mlperf flags to caffe2 version. --- dlrm_s_caffe2.py | 210 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 201 insertions(+), 9 deletions(-) diff --git a/dlrm_s_caffe2.py b/dlrm_s_caffe2.py index e5a4811f..c8b77321 100644 --- a/dlrm_s_caffe2.py +++ b/dlrm_s_caffe2.py @@ -58,12 +58,14 @@ # others import operator import time +import copy # data generation import dlrm_data_caffe2 as dc # numpy import numpy as np +import sklearn.metrics # onnx # The onnx import causes deprecation warnings every time workers @@ -474,6 +476,7 @@ def __init__( sigmoid_top=-1, save_onnx=False, model=None, + test_net=None, tag=None, ndevices=-1, forward_ops=True, @@ -493,11 +496,13 @@ def __init__( workspace.GlobalInit(global_init_opt) self.set_tags() self.model = model_helper.ModelHelper(name="DLRM", init_params=True) + self.test_net = None else: # WARNING: assume that workspace and tags have been initialized elsewhere self.set_tags(tag[0], tag[1], tag[2], tag[3], tag[4], tag[5], tag[6], tag[7], tag[8], tag[9]) self.model = model + self.test_net = test_net # save arguments self.m_spa = m_spa @@ -609,8 +614,10 @@ def create_model(self, X, S_lengths, S_indices, T): # We could use direct calls to self.model functions above to avoid it workspace.RunNetOnce(self.model.param_init_net) workspace.CreateNet(self.model.net) + if self.test_net is not None: + workspace.CreateNet(self.test_net) - def run(self, X, S_lengths, S_indices, T, enable_prof=False): + def run(self, X, S_lengths, S_indices, T, test_net=False, enable_prof=False): # feed input data to blobs # dense features self.FeedBlobWrapper(self.tdin, X, split=True) @@ -632,10 +639,13 @@ def run(self, X, S_lengths, S_indices, T, enable_prof=False): if T is not None: self.FeedBlobWrapper(self.ttar, T, split=True) # execute compute graph - if enable_prof: - workspace.C.benchmark_net(self.model.net.Name(), 0, 1, True) + if test_net: + workspace.RunNet(self.test_net) else: - workspace.RunNet(self.model.net) + if enable_prof: + workspace.C.benchmark_net(self.model.net.Name(), 0, 1, True) + else: + workspace.RunNet(self.model.net) # debug prints # print("intermediate") # print(self.FetchBlobWrapper(self.bot_l[-1])) @@ -790,6 +800,71 @@ def print_activations(self): print(self.FetchBlobWrapper(l)) +def define_metrics(): + metrics = { + 'loss': lambda y_true, y_score: + sklearn.metrics.log_loss( + y_true=y_true, + y_pred=y_score, + labels=[0,1]), + 'recall': lambda y_true, y_score: + sklearn.metrics.recall_score( + y_true=y_true, + y_pred=np.round(y_score) + ), + 'precision': lambda y_true, y_score: + sklearn.metrics.precision_score( + y_true=y_true, + y_pred=np.round(y_score) + ), + 'f1': lambda y_true, y_score: + sklearn.metrics.f1_score( + y_true=y_true, + y_pred=np.round(y_score) + ), + 'ap': sklearn.metrics.average_precision_score, + 'roc_auc': sklearn.metrics.roc_auc_score, + 'accuracy': lambda y_true, y_score: + sklearn.metrics.accuracy_score( + y_true=y_true, + y_pred=np.round(y_score) + ), + # 'pre_curve' : sklearn.metrics.precision_recall_curve, + # 'roc_curve' : sklearn.metrics.roc_curve, + } + return metrics + + +def calculate_metrics(targets, scores): + scores = np.concatenate(scores, axis=0) + targets = np.concatenate(targets, axis=0) + + metrics = define_metrics() + + # print("Compute time for validation metric : ", end="") + # first_it = True + validation_results = {} + for metric_name, metric_function in metrics.items(): + # if first_it: + # first_it = False + # else: + # print(", ", end="") + # metric_compute_start = time_wrap(False) + try: + validation_results[metric_name] = metric_function( + targets, + scores + ) + except Exception as error : + validation_results[metric_name] = -1 + print("{} in calculating {}".format(error, metric_name)) + # metric_compute_end = time_wrap(False) + # met_time = metric_compute_end - metric_compute_start + # print("{} {:.4f}".format(metric_name, 1000 * (met_time)), + # end="") + # print(" ms") + return validation_results + if __name__ == "__main__": ### import packages ### import sys @@ -843,10 +918,17 @@ def print_activations(self): parser.add_argument("--use-gpu", action="store_true", default=False) # debugging and profiling parser.add_argument("--print-freq", type=int, default=1) + parser.add_argument("--test-freq", type=int, default=-1) parser.add_argument("--print-time", action="store_true", default=False) parser.add_argument("--debug-mode", action="store_true", default=False) parser.add_argument("--enable-profiling", action="store_true", default=False) parser.add_argument("--plot-compute-graph", action="store_true", default=False) + # mlperf logging (disables other output and stops early) + parser.add_argument("--mlperf-logging", action="store_true", default=False) + # stop at target accuracy Kaggle 0.789, Terabyte (sub-sampled=0.875) 0.8107 + parser.add_argument("--mlperf-acc-threshold", type=float, default=0.0) + # stop at target AUC Terabyte (no subsampling) 0.8025 + parser.add_argument("--mlperf-auc-threshold", type=float, default=0.0) args = parser.parse_args() ### some basic setup ### @@ -1002,6 +1084,9 @@ def print_activations(self): sys.exit("ERROR: --loss-function=" + args.loss_function + " is not supported") + # define test net (as train net without gradients) + dlrm.test_net = core.Net(copy.deepcopy(dlrm.model.net.Proto())) + # specify the optimizer algorithm dlrm.sgd_optimizer( args.learning_rate, sync_dense_params=args.sync_dense_params @@ -1010,7 +1095,8 @@ def print_activations(self): dlrm.create(lX[0], lS_l[0], lS_i[0], lT[0]) ### main loop ### - print("time/loss/accuracy (if enabled):") + best_gA_test = 0 + best_auc_test = 0 total_time = 0 total_loss = 0 total_accu = 0 @@ -1018,6 +1104,7 @@ def print_activations(self): total_samp = 0 k = 0 + print("time/loss/accuracy (if enabled):") while k < args.nepochs: j = 0 while j < nbatches: @@ -1029,7 +1116,6 @@ def print_activations(self): print(lS_i[j]) print(lT[j].astype(np.float32)) ''' - # forward and backward pass, where the latter runs only # when gradients and loss have been added to the net time1 = time.time() @@ -1054,8 +1140,13 @@ def print_activations(self): total_samp += mbs # print time, loss and accuracy - print_tl = ((j + 1) % args.print_freq == 0) or (j + 1 == nbatches) - if print_tl: + should_print = ((j + 1) % args.print_freq == 0) or (j + 1 == nbatches) + should_test = ( + (args.test_freq > 0) + and (args.data_generation == "dataset") + and (((j + 1) % args.test_freq == 0) or (j + 1 == nbatches)) + ) + if should_print or should_test: gT = 1000. * total_time / total_iter if args.print_time else -1 total_time = 0 @@ -1078,7 +1169,108 @@ def print_activations(self): # print(Z) # print(T) - j += 1 # nbatches + # testing + if should_test and not args.inference_only: + # don't measure training iter time in a test iteration + if args.mlperf_logging: + previous_iteration_time = None + + test_accu = 0 + test_loss = 0 + test_samp = 0 + + if args.mlperf_logging: + scores = [] + targets = [] + + for i in range(nbatches_test): + # early exit if nbatches was set by the user and was exceeded + if nbatches > 0 and i >= nbatches: + break + + # forward pass + dlrm.run(lX_test[i], lS_l_test[i], lS_i_test[i], lT_test[i], test_net=True) + Z_test = dlrm.get_output() + T_test = lT_test[i] + + if args.mlperf_logging: + scores.append(Z_test) + targets.append(T_test) + else: + # compte loss and accuracy + L_test = dlrm.get_loss() + mbs_test = T_test.shape[0] # = mini_batch_size except last + A_test = np.sum((np.round(Z_test, 0) == T_test).astype(np.uint8)) + test_accu += A_test + test_loss += L_test * mbs_test + test_samp += mbs_test + + # compute metrics (after test loop has finished) + if args.mlperf_logging: + validation_results = calculate_metrics(targets, scores) + gA_test = validation_results['accuracy'] + gL_test = validation_results['loss'] + else: + gA_test = test_accu / test_samp + gL_test = test_loss / test_samp + + # print metrics + is_best = gA_test > best_gA_test + if is_best: + best_gA_test = gA_test + + if args.mlperf_logging: + is_best = validation_results['roc_auc'] > best_auc_test + if is_best: + best_auc_test = validation_results['roc_auc'] + + print( + "Testing at - {}/{} of epoch {},".format(j + 1, nbatches, k) + + " loss {:.6f}, recall {:.4f}, precision {:.4f},".format( + validation_results['loss'], + validation_results['recall'], + validation_results['precision'] + ) + + " f1 {:.4f}, ap {:.4f},".format( + validation_results['f1'], + validation_results['ap'], + ) + + " auc {:.4f}, best auc {:.4f},".format( + validation_results['roc_auc'], + best_auc_test + ) + + " accuracy {:3.3f} %, best accuracy {:3.3f} %".format( + validation_results['accuracy'] * 100, + best_gA_test * 100 + ) + ) + else: + print( + "Testing at - {}/{} of epoch {},".format(j + 1, nbatches, 0) + + " loss {:.6f}, accuracy {:3.3f} %, best {:3.3f} %".format( + gL_test, gA_test * 100, best_gA_test * 100 + ) + ) + + # check thresholds + if (args.mlperf_logging + and (args.mlperf_acc_threshold > 0) + and (best_gA_test > args.mlperf_acc_threshold)): + print("MLPerf testing accuracy threshold " + + str(args.mlperf_acc_threshold) + + " reached, stop training") + break + + if (args.mlperf_logging + and (args.mlperf_auc_threshold > 0) + and (best_auc_test > args.mlperf_auc_threshold)): + print("MLPerf testing auc threshold " + + str(args.mlperf_auc_threshold) + + " reached, stop training") + break + + + j += 1 # nbatches k += 1 # nepochs # test prints From 17686580738dcb00e631319edb459b19ebf28d50 Mon Sep 17 00:00:00 2001 From: mnaumovfb Date: Sun, 9 Feb 2020 15:07:27 -0800 Subject: [PATCH 03/57] Fix latent bug in caffe2 version when --max-ind-range is used --- dlrm_s_caffe2.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dlrm_s_caffe2.py b/dlrm_s_caffe2.py index c8b77321..47b27d61 100644 --- a/dlrm_s_caffe2.py +++ b/dlrm_s_caffe2.py @@ -957,7 +957,10 @@ def calculate_metrics(targets, scores): ) # enforce maximum limit on number of vectors per embedding if args.max_ind_range > 0: - ln_emb = np.array(list(map(lambda x: x % args.max_ind_range, ln_emb))) + ln_emb = np.array(list(map( + lambda x: x if x < args.max_ind_range else args.max_ind_range, + ln_emb + ))) ln_bot[0] = m_den else: # input and target at random From 11fcf0152dab31fe9e71615af4407538a15906de Mon Sep 17 00:00:00 2001 From: Tomasz Grel Date: Sat, 15 Feb 2020 01:59:43 +0100 Subject: [PATCH 04/57] Tgrel/minor mlperf fixes (#54) * Fix command-line flag typo * Remove the end-of-epoch evaluation in MLPerf mode to avoid two evals close to each other --- bench/run_and_time.sh | 2 +- dlrm_s_pytorch.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bench/run_and_time.sh b/bench/run_and_time.sh index 492f3d80..e241d801 100755 --- a/bench/run_and_time.sh +++ b/bench/run_and_time.sh @@ -14,6 +14,6 @@ else fi #echo $dlrm_extra_option -python dlrm_s_pytorch.py --arch-sparse-feature-size=128 --arch-mlp-bot="13-512-256-128" --arch-mlp-top="1024-1024-512-256-1" --max-ind-range=40000000 --data-generation=dataset --data-set=terabyte --raw-data-file=./input/day --processed-data-file=./input/terabyte_processed.npz --loss-function=bce --round-targets=True --learning-rate=1.0 --mini-batch-size=2048 --print-freq=2048 --print-time --test-freq=102400 --test-mini-batch-size=16384 --test-num-workers=16 --memory-map --mlperf-logging --mlperf-auc-threshold=0.8025 --mlperf-bin-file --mlperf-bin-shuffle $dlrm_extra_option 2>&1 | tee run_terabyte_mlperf_pt.log +python dlrm_s_pytorch.py --arch-sparse-feature-size=128 --arch-mlp-bot="13-512-256-128" --arch-mlp-top="1024-1024-512-256-1" --max-ind-range=40000000 --data-generation=dataset --data-set=terabyte --raw-data-file=./input/day --processed-data-file=./input/terabyte_processed.npz --loss-function=bce --round-targets=True --learning-rate=1.0 --mini-batch-size=2048 --print-freq=2048 --print-time --test-freq=102400 --test-mini-batch-size=16384 --test-num-workers=16 --memory-map --mlperf-logging --mlperf-auc-threshold=0.8025 --mlperf-bin-loader --mlperf-bin-shuffle $dlrm_extra_option 2>&1 | tee run_terabyte_mlperf_pt.log echo "done" diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 69bdeb55..1c4e239a 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -923,7 +923,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): should_test = ( (args.test_freq > 0) and (args.data_generation == "dataset") - and (((j + 1) % args.test_freq == 0) or (j + 1 == nbatches)) + and (((j + 1) % args.test_freq == 0) or (j + 1 == nbatches and not args.mlperf_logging)) ) # print time, loss and accuracy From eb3094ce1f04c839d7df7065ec0f014fcf305406 Mon Sep 17 00:00:00 2001 From: mnaumovfb <36135179+mnaumovfb@users.noreply.github.com> Date: Fri, 14 Feb 2020 17:27:14 -0800 Subject: [PATCH 05/57] Update README.md --- README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/README.md b/README.md index 6ca5adc8..68cd4095 100644 --- a/README.md +++ b/README.md @@ -315,6 +315,23 @@ Benchmarking *NOTE: Benchmarking scripts accept extra arguments which will be passed along to the model, such as --num-batches=100 to limit the number of data samples* +4) The code supports [MLPerf benchmark](https://mlperf.org) training parameters + + --mlperf-logging that keeps track of multiple metrics, including area under the curve (AUC) + + --mlperf-acc-threshold that allows early stopping based on accuracy metric + + --mlperf-auc-threshold that allows early stopping based on AUC metric + + --mlperf-bin-loader that enables preprocessing of data into a single binary file + + --mlperf-bin-shuffle that controls whether a random shuffle of mini-batches is performed + + The MLPerf training model is completely specified and can be run using the following script + ``` + ./bench/run_and_time.sh [--use-gpu] + ``` + Model checkpoint saving/loading ------------------------------- During training, the model can be saved using --save-model= From bda0921aceb2001945c8b2a8a824c05aeeb20e83 Mon Sep 17 00:00:00 2001 From: mnaumovfb <36135179+mnaumovfb@users.noreply.github.com> Date: Fri, 14 Feb 2020 17:29:47 -0800 Subject: [PATCH 06/57] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 68cd4095..ad165aa0 100644 --- a/README.md +++ b/README.md @@ -328,9 +328,9 @@ Benchmarking --mlperf-bin-shuffle that controls whether a random shuffle of mini-batches is performed The MLPerf training model is completely specified and can be run using the following script - ``` - ./bench/run_and_time.sh [--use-gpu] - ``` + ``` + ./bench/run_and_time.sh [--use-gpu] + ``` Model checkpoint saving/loading ------------------------------- From 73ac38ae19d6f65d0dc50ac8701651f14c4b757c Mon Sep 17 00:00:00 2001 From: mnaumovfb Date: Sat, 15 Feb 2020 15:23:58 -0800 Subject: [PATCH 07/57] adding back end of epoch check for now. --- dlrm_s_pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 1c4e239a..69bdeb55 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -923,7 +923,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): should_test = ( (args.test_freq > 0) and (args.data_generation == "dataset") - and (((j + 1) % args.test_freq == 0) or (j + 1 == nbatches and not args.mlperf_logging)) + and (((j + 1) % args.test_freq == 0) or (j + 1 == nbatches)) ) # print time, loss and accuracy From 0e8818e4b1fefed48933cc66b06e5bc4c9f1e3dc Mon Sep 17 00:00:00 2001 From: mnaumovfb Date: Wed, 19 Feb 2020 19:07:37 -0800 Subject: [PATCH 08/57] Adding flexibility in saving/loading model to/from different devices. Also, enforcing single copy of embeddings across devices on multiple GPUs. --- dlrm_s_pytorch.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 69bdeb55..03de7682 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -338,10 +338,13 @@ def parallel_forward(self, dense_x, lS_o, lS_i): if self.parallel_model_batch_size != batch_size: self.parallel_model_is_not_prepared = True - if self.sync_dense_params or self.parallel_model_is_not_prepared: + if self.parallel_model_is_not_prepared or self.sync_dense_params: # replicate mlp (data parallelism) self.bot_l_replicas = replicate(self.bot_l, device_ids) self.top_l_replicas = replicate(self.top_l, device_ids) + self.parallel_model_batch_size = batch_size + + if self.parallel_model_is_not_prepared: # distribute embeddings (model parallelism) t_list = [] for k, emb in enumerate(self.emb_l): @@ -349,7 +352,6 @@ def parallel_forward(self, dense_x, lS_o, lS_i): emb.to(d) t_list.append(emb.to(d)) self.emb_l = nn.ModuleList(t_list) - self.parallel_model_batch_size = batch_size self.parallel_model_is_not_prepared = False ### prepare input (overwrite) ### @@ -805,7 +807,23 @@ def loss_fn_wrap(Z, T, use_gpu, device): # Load model is specified if not (args.load_model == ""): print("Loading saved model {}".format(args.load_model)) - ld_model = torch.load(args.load_model) + if use_gpu: + if dlrm.ndevices > 1: + # NOTE: when targeting inference on multiple GPUs, + # load the model as is on CPU or GPU, with the move + # to multiple GPUs to be done in parallel_forward + ld_model = torch.load(args.load_model) + else: + # NOTE: when targeting inference on single GPU, + # note that the call to .to(device) has already happened + ld_model = torch.load( + args.load_model, + map_location=torch.device('cuda') + # map_location=lambda storage, loc: storage.cuda(0) + ) + else: + # when targeting inference on CPU + ld_model = torch.load(args.load_model, map_location=torch.device('cpu')) dlrm.load_state_dict(ld_model["state_dict"]) ld_j = ld_model["iter"] ld_k = ld_model["epoch"] From 916c0d643386b7ae7dab899f4a0f4be177d730b6 Mon Sep 17 00:00:00 2001 From: mnaumovfb Date: Thu, 20 Feb 2020 11:41:08 -0800 Subject: [PATCH 09/57] Adjusting the use of --max-ind-range post processing as discussed in https://github.com/facebookresearch/dlrm/issues/53 --- data_loader_terabyte.py | 37 ++++++++++++++++++++++++++++--------- dlrm_data_caffe2.py | 9 ++++++--- dlrm_data_pytorch.py | 13 ++++++++++--- 3 files changed, 44 insertions(+), 15 deletions(-) diff --git a/data_loader_terabyte.py b/data_loader_terabyte.py index 42ff8a49..f420c705 100644 --- a/data_loader_terabyte.py +++ b/data_loader_terabyte.py @@ -27,6 +27,7 @@ def __init__( data_directory, days, batch_size, + max_ind_range=-1, split="train", drop_last_batch=False ): @@ -34,6 +35,7 @@ def __init__( self.data_directory = data_directory self.days = days self.batch_size = batch_size + self.max_ind_range = max_ind_range total_file = os.path.join( data_directory, @@ -49,8 +51,12 @@ def __init__( self.drop_last_batch = drop_last_batch def __iter__(self): - return iter(_batch_generator(self.data_filename, self.data_directory, self.days, - self.batch_size, self.split, self.drop_last_batch)) + return iter( + _batch_generator( + self.data_filename, self.data_directory, self.days, + self.batch_size, self.split, self.drop_last_batch, self.max_ind_range + ) + ) def __len__(self): if self.drop_last_batch: @@ -59,7 +65,12 @@ def __len__(self): return math.ceil(self.length / self.batch_size) -def _transform_features(x_int_batch, x_cat_batch, y_batch, flag_input_torch_tensor=False): +def _transform_features( + x_int_batch, x_cat_batch, y_batch, max_ind_range, flag_input_torch_tensor=False +): + if max_ind_range > 0: + x_cat_batch = x_cat_batch % max_ind_range + if flag_input_torch_tensor: x_int_batch = torch.log(x_int_batch.clone().detach().type(torch.float) + 1) x_cat_batch = x_cat_batch.clone().detach().type(torch.long) @@ -76,7 +87,9 @@ def _transform_features(x_int_batch, x_cat_batch, y_batch, flag_input_torch_tens return x_int_batch, lS_o, x_cat_batch.t(), y_batch.view(-1, 1) -def _batch_generator(data_filename, data_directory, days, batch_size, split, drop_last): +def _batch_generator( + data_filename, data_directory, days, batch_size, split, drop_last, max_ind_range +): previous_file = None for day in days: filepath = os.path.join( @@ -126,7 +139,7 @@ def _batch_generator(data_filename, data_directory, days, batch_size, split, dro if x_int_batch.shape[0] != batch_size: raise ValueError('should not happen') - yield _transform_features(x_int_batch, x_cat_batch, y_batch) + yield _transform_features(x_int_batch, x_cat_batch, y_batch, max_ind_range) batch_start_idx += missing_samples if batch_start_idx != samples_in_file: @@ -151,9 +164,12 @@ def _batch_generator(data_filename, data_directory, days, batch_size, split, dro } if not drop_last: - yield _transform_features(previous_file['x_int'], - previous_file['x_cat'], - previous_file['y']) + yield _transform_features( + previous_file['x_int'], + previous_file['x_cat'], + previous_file['y'], + max_ind_range + ) def _test(): @@ -179,7 +195,8 @@ def _test(): class CriteoBinDataset(Dataset): """Binary version of criteo dataset.""" - def __init__(self, data_file, counts_file, batch_size=1, bytes_per_feature=4): + def __init__(self, data_file, counts_file, + batch_size=1, max_ind_range=-1, bytes_per_feature=4): # dataset self.tar_fea = 1 # single target self.den_fea = 13 # 13 dense features @@ -188,6 +205,7 @@ def __init__(self, data_file, counts_file, batch_size=1, bytes_per_feature=4): self.tot_fea = self.tad_fea + self.spa_fea self.batch_size = batch_size + self.max_ind_range = max_ind_range self.bytes_per_entry = (bytes_per_feature * self.tot_fea * batch_size) self.num_entries = math.ceil(os.path.getsize(data_file) / self.bytes_per_entry) @@ -213,6 +231,7 @@ def __getitem__(self, idx): return _transform_features(x_int_batch=tensor[:, 1:14], x_cat_batch=tensor[:, 14:], y_batch=tensor[:, 0], + max_ind_range=self.max_ind_range, flag_input_torch_tensor=True) diff --git a/dlrm_data_caffe2.py b/dlrm_data_caffe2.py index a4322198..12d8d9f1 100644 --- a/dlrm_data_caffe2.py +++ b/dlrm_data_caffe2.py @@ -95,7 +95,10 @@ def read_dataset( print("Sparse features = %d, Dense features = %d" % (n_emb, m_den)) # adjust parameters - def assemble_samples(X_cat, X_int, y, print_message): + def assemble_samples(X_cat, X_int, y, max_ind_range, print_message): + if max_ind_range > 0: + X_cat = X_cat % max_ind_range + nsamples = len(y) data_size = nsamples # using floor is equivalent to dropping last mini-batch (drop_last = True) @@ -155,12 +158,12 @@ def assemble_samples(X_cat, X_int, y, print_message): # adjust training data (nbatches, lX, lS_lengths, lS_indices, lT) = assemble_samples( - X_cat_train, X_int_train, y_train, "Training data" + X_cat_train, X_int_train, y_train, max_ind_range, "Training data" ) # adjust testing data (nbatches_t, lX_t, lS_lengths_t, lS_indices_t, lT_t) = assemble_samples( - X_cat_test, X_int_test, y_test, "Testing data" + X_cat_test, X_int_test, y_test, max_ind_range, "Testing data" ) #end if memory_map diff --git a/dlrm_data_pytorch.py b/dlrm_data_pytorch.py index cdd80124..6cbe382a 100644 --- a/dlrm_data_pytorch.py +++ b/dlrm_data_pytorch.py @@ -289,7 +289,10 @@ def __getitem__(self, index): else: i = index - return self.X_int[i], self.X_cat[i], self.y[i] + if self.max_ind_range > 0: + return self.X_int[i], self.X_cat[i] % self.max_ind_range, self.y[i] + else: + return self.X_int[i], self.X_cat[i], self.y[i] def _default_preprocess(self, X_int, X_cat, y): X_int = torch.log(torch.tensor(X_int, dtype=torch.float) + 1) @@ -395,7 +398,8 @@ def make_criteo_data_and_loaders(args): train_data = data_loader_terabyte.CriteoBinDataset( data_file=train_file, counts_file=counts_file, - batch_size=args.mini_batch_size + batch_size=args.mini_batch_size, + max_ind_range=args.max_ind_range ) train_loader = torch.utils.data.DataLoader( @@ -413,7 +417,8 @@ def make_criteo_data_and_loaders(args): test_data = data_loader_terabyte.CriteoBinDataset( data_file=test_file, counts_file=counts_file, - batch_size=args.test_mini_batch_size + batch_size=args.test_mini_batch_size, + max_ind_range=args.max_ind_range ) test_loader = torch.utils.data.DataLoader( @@ -456,6 +461,7 @@ def make_criteo_data_and_loaders(args): data_filename=data_filename, days=list(range(23)), batch_size=args.mini_batch_size, + max_ind_range=args.max_ind_range, split="train" ) @@ -464,6 +470,7 @@ def make_criteo_data_and_loaders(args): data_filename=data_filename, days=[23], batch_size=args.test_mini_batch_size, + max_ind_range=args.max_ind_range, split="test" ) else: From 819ef5fbdd92226f08472a1dad12b1aae122db83 Mon Sep 17 00:00:00 2001 From: Tomasz Grel Date: Thu, 27 Feb 2020 16:30:30 +0100 Subject: [PATCH 10/57] Switch the binary dataloader to int32 datatype (#60) --- data_loader_terabyte.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/data_loader_terabyte.py b/data_loader_terabyte.py index f420c705..d1a38efa 100644 --- a/data_loader_terabyte.py +++ b/data_loader_terabyte.py @@ -225,7 +225,7 @@ def __len__(self): def __getitem__(self, idx): self.file.seek(idx * self.bytes_per_entry, 0) raw_data = self.file.read(self.bytes_per_entry) - array = np.frombuffer(raw_data, dtype=np.float32) + array = np.frombuffer(raw_data, dtype=np.int32) tensor = torch.from_numpy(array).view((-1, self.tot_fea)) return _transform_features(x_int_batch=tensor[:, 1:14], @@ -238,6 +238,9 @@ def __getitem__(self, idx): def numpy_to_binary(input_files, output_file_path, split='train'): """Convert the data to a binary format to be read with CriteoBinDataset.""" + # WARNING - both categorical and numerical data must fit into int32 for + # the following code to work correctly + with open(output_file_path, 'wb') as output_file: if split == 'train': for input_file in input_files: @@ -247,7 +250,7 @@ def numpy_to_binary(input_files, output_file_path, split='train'): np_data = np.concatenate([np_data['y'].reshape(-1, 1), np_data['X_int'], np_data['X_cat']], axis=1) - np_data = np_data.astype(np.float32) + np_data = np_data.astype(np.int32) output_file.write(np_data.tobytes()) else: @@ -256,7 +259,7 @@ def numpy_to_binary(input_files, output_file_path, split='train'): np_data = np.concatenate([np_data['y'].reshape(-1, 1), np_data['X_int'], np_data['X_cat']], axis=1) - np_data = np_data.astype(np.float32) + np_data = np_data.astype(np.int32) samples_in_file = np_data.shape[0] midpoint = int(np.ceil(samples_in_file / 2.)) From fde972397dbc42fc2354333a75ca55e78a263c9c Mon Sep 17 00:00:00 2001 From: mnaumovfb Date: Wed, 4 Mar 2020 10:32:53 -0800 Subject: [PATCH 11/57] Adjusting restart from saved model during training. Need to skip early batches when enumerate is used. Summary: Adjusting restart from saved model during training. Need to skip early batches when enumerate is used. Test Plan: Reviewers: Subscribers: Tasks: Tags: --- dlrm_s_pytorch.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 03de7682..9bf762b7 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -797,6 +797,8 @@ def loss_fn_wrap(Z, T, use_gpu, device): # training or inference best_gA_test = 0 best_auc_test = 0 + skip_upto_epoch = 0 + skip_upto_batch = 0 total_time = 0 total_loss = 0 total_accu = 0 @@ -841,8 +843,8 @@ def loss_fn_wrap(Z, T, use_gpu, device): best_gA_test = ld_gA_test total_loss = ld_total_loss total_accu = ld_total_accu - k = ld_k # epochs - j = ld_j # batches + skip_upto_epoch = ld_k # epochs + skip_upto_batch = ld_j # batches else: args.print_freq = ld_nbatches args.test_freq = 0 @@ -866,12 +868,18 @@ def loss_fn_wrap(Z, T, use_gpu, device): print("time/loss/accuracy (if enabled):") with torch.autograd.profiler.profile(args.enable_profiling, use_gpu) as prof: while k < args.nepochs: + if k < skip_upto_epoch: + continue + accum_time_begin = time_wrap(use_gpu) if args.mlperf_logging: previous_iteration_time = None for j, (X, lS_o, lS_i, T) in enumerate(train_ld): + if j < skip_upto_batch: + continue + if args.mlperf_logging: current_time = time_wrap(use_gpu) if previous_iteration_time: From 9074b5e0f8fcfc7ab81b9312a15efe54010a6d01 Mon Sep 17 00:00:00 2001 From: mnaumovfb <36135179+mnaumovfb@users.noreply.github.com> Date: Wed, 25 Mar 2020 14:01:31 -0700 Subject: [PATCH 12/57] Update README.md --- README.md | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index ad165aa0..4f7165a5 100644 --- a/README.md +++ b/README.md @@ -287,50 +287,55 @@ Benchmarking ``` 2) The code supports interface with the [Criteo Kaggle Display Advertising Challenge Dataset](https://labs.criteo.com/2014/02/kaggle-display-advertising-challenge-dataset/). - Please do the following to prepare the dataset for use with DLRM code: + - Please do the following to prepare the dataset for use with DLRM code: - First, specify the raw data file (train.txt) as downloaded with --raw-data-file= - This is then pre-processed (categorize, concat across days...) to allow using with dlrm code - The processed data is stored as *.npz file in /input/*.npz - The processed file (*.npz) can be used for subsequent runs with --processed-data-file= - + - The model can be trained using the following script ``` ./bench/dlrm_s_criteo_kaggle.sh ``` - + 3) The code supports interface with the [Criteo Terabyte Dataset](https://labs.criteo.com/2013/12/download-terabyte-click-logs/). - Please do the following to prepare the dataset for use with DLRM code: + - Please do the following to prepare the dataset for use with DLRM code: - First, download the raw data files day_0.gz, ...,day_23.gz and unzip them - Specify the location of the unzipped text files day_0, ...,day_23, using --raw-data-file= (the day number will be appended automatically) - These are then pre-processed (categorize, concat across days...) to allow using with dlrm code - The processed data is stored as *.npz file in /input/*.npz - The processed file (*.npz) can be used for subsequent runs with --processed-data-file= - + - The model can be trained using the following script ``` - ./bench/dlrm_s_criteo_terabyte.sh ["--memory-map --data-sub-sample-rate=0.875"] + ./bench/dlrm_s_criteo_terabyte.sh ["--memory-map --data-sub-sample-rate=0.875"] ``` + - Corresponding pre-trained model is available under [CC-BY-NC license](https://creativecommons.org/licenses/by-nc/2.0/) and can be downloaded here + [dlrm_subsample0.875_maxindrange10M_pretrained.pt](https://github.com/facebookresearch/dlrm) *NOTE: Benchmarking scripts accept extra arguments which will be passed along to the model, such as --num-batches=100 to limit the number of data samples* -4) The code supports [MLPerf benchmark](https://mlperf.org) training parameters - - --mlperf-logging that keeps track of multiple metrics, including area under the curve (AUC) +4) The code supports interface with [MLPerf benchmark](https://mlperf.org). + - Please refer to the following training parameters + ``` + --mlperf-logging that keeps track of multiple metrics, including area under the curve (AUC) - --mlperf-acc-threshold that allows early stopping based on accuracy metric + --mlperf-acc-threshold that allows early stopping based on accuracy metric - --mlperf-auc-threshold that allows early stopping based on AUC metric + --mlperf-auc-threshold that allows early stopping based on AUC metric - --mlperf-bin-loader that enables preprocessing of data into a single binary file + --mlperf-bin-loader that enables preprocessing of data into a single binary file - --mlperf-bin-shuffle that controls whether a random shuffle of mini-batches is performed - - The MLPerf training model is completely specified and can be run using the following script + --mlperf-bin-shuffle that controls whether a random shuffle of mini-batches is performed + ``` + - The MLPerf training model is completely specified and can be trained using the following script ``` - ./bench/run_and_time.sh [--use-gpu] + ./bench/run_and_time.sh [--use-gpu] ``` + - Corresponding pre-trained model is available under [CC-BY-NC license](https://creativecommons.org/licenses/by-nc/2.0/) and can be downloaded here + [dlrm_subsample0.0_maxindrange40M_pretrained.pt](https://github.com/facebookresearch/dlrm) Model checkpoint saving/loading ------------------------------- From 6f7711dd8698e718419b03604a55cdf70399a97f Mon Sep 17 00:00:00 2001 From: mnaumovfb <36135179+mnaumovfb@users.noreply.github.com> Date: Wed, 25 Mar 2020 16:52:13 -0700 Subject: [PATCH 13/57] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4f7165a5..258ffa0a 100644 --- a/README.md +++ b/README.md @@ -294,7 +294,7 @@ Benchmarking - The processed file (*.npz) can be used for subsequent runs with --processed-data-file= - The model can be trained using the following script ``` - ./bench/dlrm_s_criteo_kaggle.sh + ./bench/dlrm_s_criteo_kaggle.sh [--test-freq=1024] ``` @@ -308,7 +308,7 @@ Benchmarking - The processed file (*.npz) can be used for subsequent runs with --processed-data-file= - The model can be trained using the following script ``` - ./bench/dlrm_s_criteo_terabyte.sh ["--memory-map --data-sub-sample-rate=0.875"] + ./bench/dlrm_s_criteo_terabyte.sh ["--test-freq=10240 --memory-map --data-sub-sample-rate=0.875"] ``` - Corresponding pre-trained model is available under [CC-BY-NC license](https://creativecommons.org/licenses/by-nc/2.0/) and can be downloaded here [dlrm_subsample0.875_maxindrange10M_pretrained.pt](https://github.com/facebookresearch/dlrm) From 66fe6d82beab95111258be57c71ff92bdbf9e3e7 Mon Sep 17 00:00:00 2001 From: mnaumovfb <36135179+mnaumovfb@users.noreply.github.com> Date: Thu, 26 Mar 2020 17:42:10 -0700 Subject: [PATCH 14/57] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 258ffa0a..8cc3c108 100644 --- a/README.md +++ b/README.md @@ -311,7 +311,7 @@ Benchmarking ./bench/dlrm_s_criteo_terabyte.sh ["--test-freq=10240 --memory-map --data-sub-sample-rate=0.875"] ``` - Corresponding pre-trained model is available under [CC-BY-NC license](https://creativecommons.org/licenses/by-nc/2.0/) and can be downloaded here - [dlrm_subsample0.875_maxindrange10M_pretrained.pt](https://github.com/facebookresearch/dlrm) + [dlrm_emb64_subsample0.875_maxindrange10M_pretrained.pt](https://dlrm.s3-us-west-1.amazonaws.com/models/tb0875_10M.pt) @@ -335,7 +335,7 @@ Benchmarking ./bench/run_and_time.sh [--use-gpu] ``` - Corresponding pre-trained model is available under [CC-BY-NC license](https://creativecommons.org/licenses/by-nc/2.0/) and can be downloaded here - [dlrm_subsample0.0_maxindrange40M_pretrained.pt](https://github.com/facebookresearch/dlrm) + [dlrm_emb128_subsample0.0_maxindrange40M_pretrained.pt](https://dlrm.s3-us-west-1.amazonaws.com/models/tb00_40M.pt) Model checkpoint saving/loading ------------------------------- From 7f2129ea9d84c80727c27d1fdc004574a85e20bd Mon Sep 17 00:00:00 2001 From: dkorchevgithub <63178227+dkorchevgithub@users.noreply.github.com> Date: Sun, 10 May 2020 17:59:07 -0700 Subject: [PATCH 15/57] added visualization of DLRM embeddings (#72) * adding script to visualize embedding tables * updated embedding visualization --- tools/visualize.py | 197 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 tools/visualize.py diff --git a/tools/visualize.py b/tools/visualize.py new file mode 100644 index 00000000..8d154328 --- /dev/null +++ b/tools/visualize.py @@ -0,0 +1,197 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +# +# +# This script performs the visualization of the embedding tables created in +# DLRM during the training procedure. We use two popular techniques for +# visualization: umap (https://umap-learn.readthedocs.io/en/latest/) +# and tsne (https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html). +# These links also provide instructions on how to install these packages +# in different environments. +# +# Warning: the size of the data to be visualized depends on the RAM on your machine. +# +# +# A sample run of the code, with a kaggle model is shown below +# $ python ./tools/visualize.py –dataset=kaggle --load-model=./input/dlrm_kaggle.pytorch +# +# +# The following command line arguments are available to the user: +# +# --load-model - DLRM model file +# --dataset - one of ["kaggle", "terabyte"] +# --max-ind-range - max index range used during the traning +# --output-dir - output directory where output plots will be written, default will be on of these: ["kaggle_vis", "terabyte_vis"] +# --max-umap-size - max number of points to visualize using UMAP, default=50000 +# --use-tsne - use T-SNE +# --max-tsne-size - max number of points to visualize using T-SNE, default=1000) +# + +import sys, os +import argparse +import numpy as np +import umap +import json +import torch +import matplotlib.pyplot as plt + +from sklearn import manifold + +import dlrm_data_pytorch as dp +from dlrm_s_pytorch import DLRM_Net + + +def visualize_embeddings_umap(emb_l, + output_dir = "", + max_size = 500000): + + for k in range(0, len(emb_l)): + + E = dlrm.emb_l[k].weight.detach().cpu() + print("umap", E.shape) + + if E.shape[0] < 20: + print("Skipping small embedding") + continue + +# reducer = umap.UMAP(random_state=42, n_neighbors=25, min_dist=0.1) + reducer = umap.UMAP(random_state=42) + Y = reducer.fit_transform(E[:max_size,:]) + + plt.figure(figsize=(8,8)) + if Y.shape[0] > 2000: + size = 1 + else: + size = 5 + plt.scatter(-Y[:,0], -Y[:,1], s=size) + + n_vis = min(max_size, E.shape[0]) + plt.title("UMAP: categorical var. "+str(k)+" ("+str(n_vis)+" of "+str(E.shape[0])+")") + plt.savefig(output_dir+"/cat-"+str(k)+"-"+str(n_vis)+"-of-"+str(E.shape[0])+"-umap.png") + plt.close() + +def visualize_embeddings_tsne(emb_l, + output_dir = "", + max_size = 10000): + + for k in range(0, len(emb_l)): + + E = dlrm.emb_l[k].weight.detach().cpu() + print("tsne", E.shape) + + if E.shape[0] < 20: + print("Skipping small embedding") + continue + + tsne = manifold.TSNE(init='pca', random_state=0, method='exact') + + Y = tsne.fit_transform(E[:max_size,:]) + + plt.figure(figsize=(8,8)) + plt.scatter(-Y[:,0], -Y[:,1]) + + n_vis = min(max_size, E.shape[0]) + plt.title("TSNE: categorical var. "+str(k)+" ("+str(n_vis)+" of "+str(E.shape[0])+")") + plt.savefig(output_dir+"/cat-"+str(k)+"-"+str(n_vis)+"-of-"+str(E.shape[0])+"-tsne.png") + plt.close() + + +if __name__ == "__main__": + + output_dir = "" + + ### parse arguments ### + parser = argparse.ArgumentParser( + description="Exploratory DLRM analysis" + ) + + parser.add_argument("--load-model", type=str, default="") + parser.add_argument("--dataset", choices=["kaggle", "terabyte"], help="dataset") +# parser.add_argument("--dataset-path", required=True, help="path to the dataset") + parser.add_argument("--max-ind-range", type=int, default=-1) +# parser.add_argument("--mlperf-bin-loader", action='store_true', default=False) + parser.add_argument("--output-dir", type=str, default="") + # umap related + parser.add_argument("--max-umap-size", type=int, default=50000) + # tsne related + parser.add_argument("--use-tsne", action='store_true', default=False) + parser.add_argument("--max-tsne-size", type=int, default=1000) + + args = parser.parse_args() + + print('command line args: ', json.dumps(vars(args))) + + if output_dir == "": + output_dir = args.dataset+"_vis" + print('output_dir:', output_dir) + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + if args.dataset == "kaggle": + # 1. Criteo Kaggle Display Advertisement Challenge Dataset (see ./bench/dlrm_s_criteo_kaggle.sh) + m_spa=16 + ln_emb=np.array([1460,583,10131227,2202608,305,24,12517,633,3,93145,5683,8351593,3194,27,14992,5461306,10,5652,2173,4,7046547,18,15,286181,105,142572]) + ln_bot=np.array([13,512,256,64,16]) + ln_top=np.array([367,512,256,1]) + + elif args.dataset == "terabyte": + + if args.max_ind_range == 10000000: + # 2. Criteo Terabyte (see ./bench/dlrm_s_criteo_terabyte.sh [--sub-sample=0.875] --max-in-range=10000000) + m_spa=64 + ln_emb=np.array([9980333,36084,17217,7378,20134,3,7112,1442,61, 9758201,1333352,313829,10,2208,11156,122,4,970,14, 9994222, 7267859, 9946608,415421,12420,101, 36]) + ln_bot=np.array([13,512,256,64]) + ln_top=np.array([415,512,512,256,1]) + elif args.max_ind_range == 40000000: + # 3. Criteo Terabyte MLPerf training (see ./bench/run_and_time.sh --max-in-range=40000000) + m_spa=128 + ln_emb=np.array([39884406,39043,17289,7420,20263,3,7120,1543,63,38532951,2953546,403346,10,2208,11938,155,4,976,14,39979771,25641295,39664984,585935,12972,108,36]) + ln_bot=np.array([13,512,256,128]) + ln_top=np.array([479,1024,1024,512,256,1]) + else: + raise ValueError("only --max-in-range 10M or 40M is supported") + else: + raise ValueError("only kaggle|terabyte dataset options are supported") + + dlrm = DLRM_Net( + m_spa, + ln_emb, + ln_bot, + ln_top, + arch_interaction_op="dot", + arch_interaction_itself=False, + sigmoid_bot=-1, + sigmoid_top=ln_top.size - 2, + sync_dense_params=True, + loss_threshold=0.0, + ndevices=-1, + qr_flag=False, + qr_operation=None, + qr_collisions=None, + qr_threshold=None, + md_flag=False, + md_threshold=None, + ) + + # Load model is specified + if not (args.load_model == ""): + print("Loading saved model {}".format(args.load_model)) + + ld_model = torch.load(args.load_model, map_location=torch.device('cpu')) + dlrm.load_state_dict(ld_model["state_dict"]) + + print("Model loaded", args.load_model) + #print(dlrm) + + visualize_embeddings_umap(emb_l = dlrm.emb_l, + output_dir = output_dir, + max_size = args.max_umap_size) + + if args.use_tsne == True: + visualize_embeddings_tsne(emb_l = dlrm.emb_l, + output_dir = output_dir, + max_size = args.max_tsne_size) + From fbabe615f12865929182050d9030446ce188499c Mon Sep 17 00:00:00 2001 From: Rachitha Prem Seelin <6088890+rachithayp@users.noreply.github.com> Date: Sat, 23 May 2020 03:10:51 -0700 Subject: [PATCH 16/57] Enable LR warmup and decay policy (#73) Co-authored-by: rpremsee --- dlrm_s_pytorch.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 9bf762b7..f81a1849 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -91,8 +91,34 @@ # import torch.nn.functional as Functional # from torch.nn.parameter import Parameter +from torch.optim.lr_scheduler import _LRScheduler + exc = getattr(builtins, "IOError", "FileNotFoundError") +class LRPolicyScheduler(_LRScheduler): + def __init__(self, optimizer, num_warmup_steps, decay_start_step, num_decay_steps): + self.num_warmup_steps = num_warmup_steps + self.decay_start_step = decay_start_step + self.num_decay_steps = num_decay_steps + + if self.decay_start_step < self.num_warmup_steps: + sys.exit("Learning rate warmup must finish before the decay starts") + + super(LRPolicyScheduler, self).__init__(optimizer) + + def get_lr(self): + step_count = self._step_count + if (self.num_warmup_steps > 0) and (step_count <= self.num_warmup_steps): + scale = 1.0 - (self.num_warmup_steps - step_count) / self.num_warmup_steps + lr = [base_lr * scale for base_lr in self.base_lrs] + elif self.num_decay_steps != 0 and step_count >= self.decay_start_step: + decayed_steps = step_count - self.decay_start_step + scale = ((self.num_decay_steps - decayed_steps) / self.num_decay_steps) ** 2 + min_lr = 0.0000001 + lr = [max(min_lr, base_lr * scale) for base_lr in self.base_lrs] + else: + lr = self.base_lrs + return lr ### define dlrm in PyTorch ### class DLRM_Net(nn.Module): @@ -519,6 +545,10 @@ def parallel_forward(self, dense_x, lS_o, lS_i): parser.add_argument("--mlperf-auc-threshold", type=float, default=0.0) parser.add_argument("--mlperf-bin-loader", action='store_true', default=False) parser.add_argument("--mlperf-bin-shuffle", action='store_true', default=False) + # LR policy + parser.add_argument("--lr-num-warmup-steps", type=int, default=0) + parser.add_argument("--lr-decay-start-step", type=int, default=0) + parser.add_argument("--lr-num-decay-steps", type=int, default=0) args = parser.parse_args() if args.mlperf_logging: @@ -752,6 +782,8 @@ def parallel_forward(self, dense_x, lS_o, lS_i): if not args.inference_only: # specify the optimizer algorithm optimizer = torch.optim.SGD(dlrm.parameters(), lr=args.learning_rate) + lr_scheduler = LRPolicyScheduler(optimizer, args.lr_num_warmup_steps, args.lr_decay_start_step, + args.lr_num_decay_steps) ### main loop ### def time_wrap(use_gpu): @@ -934,6 +966,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): # optimizer optimizer.step() + lr_scheduler.step() if args.mlperf_logging: total_time += iteration_time From cef3b73d9fb8f33926d2b6c3ad88e7b09afb3f81 Mon Sep 17 00:00:00 2001 From: dkorchevgithub <63178227+dkorchevgithub@users.noreply.github.com> Date: Wed, 27 May 2020 23:23:40 -0700 Subject: [PATCH 17/57] added more visualization options (#76) * adding script to visualize embedding tables * updated embedding visualization * updating visualization - adding data visualization - analysis of categorical variables * updated data visualization - mapping test data into manifold * created double plot for data visualization * added plots for each data class * aaded more plots: correct and erros, refactored the code * more refactoring, added z data * added intermidiate Z layers * updating visualization * updated output directory and plots * fixed silent bug --- tools/visualize.py | 483 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 453 insertions(+), 30 deletions(-) diff --git a/tools/visualize.py b/tools/visualize.py index 8d154328..b39def85 100644 --- a/tools/visualize.py +++ b/tools/visualize.py @@ -21,7 +21,7 @@ # The following command line arguments are available to the user: # # --load-model - DLRM model file -# --dataset - one of ["kaggle", "terabyte"] +# --data-set - one of ["kaggle", "terabyte"] # --max-ind-range - max index range used during the traning # --output-dir - output directory where output plots will be written, default will be on of these: ["kaggle_vis", "terabyte_vis"] # --max-umap-size - max number of points to visualize using UMAP, default=50000 @@ -29,14 +29,20 @@ # --max-tsne-size - max number of points to visualize using T-SNE, default=1000) # -import sys, os +import os import argparse import numpy as np import umap import json import torch +import matplotlib import matplotlib.pyplot as plt +from sklearn.metrics import accuracy_score +from sklearn.metrics import f1_score +from sklearn.metrics import precision_score +from sklearn.metrics import recall_score + from sklearn import manifold import dlrm_data_pytorch as dp @@ -49,55 +55,447 @@ def visualize_embeddings_umap(emb_l, for k in range(0, len(emb_l)): - E = dlrm.emb_l[k].weight.detach().cpu() + E = emb_l[k].weight.detach().cpu() print("umap", E.shape) if E.shape[0] < 20: print("Skipping small embedding") continue - + + n_vis = min(max_size, E.shape[0]) + # reducer = umap.UMAP(random_state=42, n_neighbors=25, min_dist=0.1) reducer = umap.UMAP(random_state=42) - Y = reducer.fit_transform(E[:max_size,:]) + Y = reducer.fit_transform(E[:n_vis,:]) plt.figure(figsize=(8,8)) - if Y.shape[0] > 2000: - size = 1 - else: - size = 5 - plt.scatter(-Y[:,0], -Y[:,1], s=size) + + linewidth = 0 + size = 1 + + if Y.shape[0] < 2500: + linewidth = 1 + size = 5 + + plt.scatter(-Y[:,0], -Y[:,1], s=size, marker='.', linewidth=linewidth) - n_vis = min(max_size, E.shape[0]) plt.title("UMAP: categorical var. "+str(k)+" ("+str(n_vis)+" of "+str(E.shape[0])+")") plt.savefig(output_dir+"/cat-"+str(k)+"-"+str(n_vis)+"-of-"+str(E.shape[0])+"-umap.png") plt.close() + def visualize_embeddings_tsne(emb_l, output_dir = "", max_size = 10000): for k in range(0, len(emb_l)): - E = dlrm.emb_l[k].weight.detach().cpu() + E = emb_l[k].weight.detach().cpu() print("tsne", E.shape) if E.shape[0] < 20: print("Skipping small embedding") continue + + n_vis = min(max_size, E.shape[0]) tsne = manifold.TSNE(init='pca', random_state=0, method='exact') - Y = tsne.fit_transform(E[:max_size,:]) - + Y = tsne.fit_transform(E[:n_vis,:]) + plt.figure(figsize=(8,8)) - plt.scatter(-Y[:,0], -Y[:,1]) + + linewidth = 0 + if Y.shape[0] < 5000: + linewidth = 1 + + plt.scatter(-Y[:,0], -Y[:,1], s=1, marker='.', linewidth=linewidth) - n_vis = min(max_size, E.shape[0]) plt.title("TSNE: categorical var. "+str(k)+" ("+str(n_vis)+" of "+str(E.shape[0])+")") plt.savefig(output_dir+"/cat-"+str(k)+"-"+str(n_vis)+"-of-"+str(E.shape[0])+"-tsne.png") plt.close() +def create_vis_data(dlrm, data_ld, max_size=50000, info=''): + + all_features = [] + all_X = [] + all_cat = [] + all_T = [] + all_c = [] + all_z = [] + all_pred = [] + + z_size = len(dlrm.top_l) + print('z_size', z_size) + for i in range(0, z_size): + all_z.append([]) + + for j, (X, lS_o, lS_i, T) in enumerate(data_ld): + + if j >= max_size: + break + + all_feat_vec = [] + all_cat_vec = [] + + x = dlrm.apply_mlp(X, dlrm.bot_l) + # debug prints + #print("intermediate") + #print(x[0].detach().cpu().numpy()) + all_feat_vec.append(x[0].detach().cpu().numpy()) + all_X.append(x[0].detach().cpu().numpy()) + + # process sparse features(using embeddings), resulting in a list of row vectors + ly = dlrm.apply_emb(lS_o, lS_i, dlrm.emb_l) + + for e in ly: + #print(e.detach().cpu().numpy()) + all_feat_vec.append(e[0].detach().cpu().numpy()) + all_cat_vec.append(e[0].detach().cpu().numpy()) + + all_feat_vec= np.concatenate(all_feat_vec, axis=0) + all_cat_vec= np.concatenate(all_cat_vec, axis=0) + + all_features.append(all_feat_vec) + all_cat.append(all_cat_vec) + all_T.append(int(T.detach().cpu().numpy()[0,0])) + + z = dlrm.interact_features(x, ly) + # print(z.detach().cpu().numpy()) + all_z[0].append(z.detach().cpu().numpy().flatten()) + + # obtain probability of a click (using top mlp) +# print(dlrm.top_l) +# p = dlrm.apply_mlp(z, dlrm.top_l) + + for i in range(0, z_size): + z = dlrm.top_l[i](z) + + if i < z_size-1: + curr_z = z.detach().cpu().numpy().flatten() + all_z[i+1].append(curr_z) + + # print('z',i, z.detach().cpu().numpy().flatten().shape) + + p = z + + # clamp output if needed + if 0.0 < dlrm.loss_threshold and dlrm.loss_threshold < 1.0: + z = torch.clamp(p, min=dlrm.loss_threshold, max=(1.0 - dlrm.loss_threshold)) + else: + z = p + + all_pred.append(int(z.detach().cpu().numpy()[0,0]+0.5)) + + #print(int(z.detach().cpu().numpy()[0,0]+0.5)) + if int(z.detach().cpu().numpy()[0,0]+0.5) == int(T.detach().cpu().numpy()[0,0]): + all_c.append(0) + else: + all_c.append(1) + + # calculate classifier metrics + ac = accuracy_score(all_T, all_pred) + f1 = f1_score(all_T, all_pred) + ps = precision_score(all_T, all_pred) + rc = recall_score(all_T, all_pred) + + print(info, 'accuracy', ac, 'f1', f1, 'precision', ps, 'recall', rc) + + return all_features, all_X, all_cat, all_T, all_z, all_c + +def plot_all_data(Y_train_data, + train_labels, + Y_test_data, + test_labels, + total_train_size = '', + total_test_size = '', + info = '', + output_dir = ''): + + size = 1 + colors = ['red','green'] + + fig, (ax1, ax2) = plt.subplots(1, 2) + fig.suptitle('UMAP: ' + info) + + ax1.scatter(-Y_train_data[:,0], -Y_train_data[:,1], s=size, c=train_labels, cmap=matplotlib.colors.ListedColormap(colors), marker='.', linewidth=0) + ax1.title.set_text('Train ('+str(len(train_labels))+' of '+ total_train_size+')') + if test_data is not None and test_labels is not None: + ax2.scatter(-Y_test_data[:,0], -Y_test_data[:,1], s=size, c=test_labels, cmap=matplotlib.colors.ListedColormap(colors), marker='.', linewidth=0) + ax2.title.set_text('Test ('+str(len(test_labels))+' of '+ total_test_size+')') + + plt.savefig(output_dir+"/"+info+'-umap.png') + plt.close() + + +def plot_one_class(Y_train_data, + train_labels, + Y_test_data, + test_labels, + label = 0, + col = 'red', + total_train_size = '', + total_test_size = '', + info = '', + output_dir = ''): + + size = 1 + + fig, (ax1, ax2) = plt.subplots(1, 2) + fig.suptitle('UMAP: '+ info ) + + ind_l_train = [i for i,x in enumerate(train_labels) if x == label] + Y_train_l = np.array([Y_train_data[i,:] for i in ind_l_train]) + + ax1.scatter(-Y_train_l[:,0], -Y_train_l[:,1], s=size, c=col, marker='.', linewidth=0) + ax1.title.set_text('Train, ('+str(len(train_labels))+' of '+ total_train_size+')') + if Y_test_data is not None and test_labels is not None: + ind_l_test = [i for i,x in enumerate(test_labels) if x == label] + Y_test_l = np.array([Y_test_data[i,:] for i in ind_l_test]) + + ax2.scatter(-Y_test_l[:,0], -Y_test_l[:,1], s=size, c=col, marker='.', linewidth=0) + ax2.title.set_text('Test, ('+str(len(test_labels))+' of '+ total_test_size+')') + + plt.savefig(output_dir+"/"+info+'-umap.png') + plt.close() + + +def visualize_umap(train_data, + train_c, + train_targets, + test_data = None, + test_c = None, + test_targets = None, + total_train_size = '', + total_test_size = '', + info = '', + output_dir = ''): + +# reducer = umap.UMAP(random_state=42, n_neighbors=25, min_dist=0.1) + reducer = umap.UMAP(random_state=42) + train_Y = reducer.fit_transform(train_data) + + if test_data is not None and test_targets is not None: + test_Y = reducer.transform(test_data) + + # all classes + plot_all_data(Y_train_data = train_Y, + train_labels = train_targets, + Y_test_data = test_Y, + test_labels = test_targets, + total_train_size = total_train_size, + total_test_size = total_test_size, + info = info, + output_dir = output_dir) + + # class 0 + plot_one_class(Y_train_data = train_Y, + train_labels = train_targets, + Y_test_data = test_Y, + test_labels = test_targets, + label = 0, + col = 'red', + total_train_size = total_train_size, + total_test_size = total_test_size, + info = info+' class ' + str(0), + output_dir = output_dir) + + # class 1 + plot_one_class(Y_train_data = train_Y, + train_labels = train_targets, + Y_test_data = test_Y, + test_labels = test_targets, + label = 1, + col = 'green', + total_train_size = total_train_size, + total_test_size = total_test_size, + info = info + ' class ' + str(1), + output_dir = output_dir) + + # correct classification + plot_one_class(Y_train_data = train_Y, + train_labels = train_c, + Y_test_data = test_Y, + test_labels = test_c, + label = 0, + col = 'green', + total_train_size = total_train_size, + total_test_size = total_test_size, + info = info + ' correct ', + output_dir = output_dir) + + # errors + plot_one_class(Y_train_data = train_Y, + train_labels = train_c, + Y_test_data = test_Y, + test_labels = test_c, + label = 1, + col = 'red', + total_train_size = total_train_size, + total_test_size = total_test_size, + info = info + ' errors ', + output_dir = output_dir) + + + +def visualize_data_umap(dlrm, + train_data_ld, + test_data_ld = None, + max_umap_size = 50000, + output_dir = ''): + + train_feat, train_X, train_cat, train_T, train_z, train_c = create_vis_data(dlrm=dlrm, data_ld=train_data_ld, max_size=max_umap_size, info='train') + + test_feat = None + test_X = None + test_cat = None + test_T = None + + if test_data_ld is not None: + test_feat, test_X, test_cat, test_T, test_z, test_c = create_vis_data(dlrm=dlrm, data_ld=test_data_ld, max_size=max_umap_size, info='test') + + visualize_umap(train_data = train_feat, + train_targets = train_T, + train_c = train_c, + test_data = test_feat, + test_c = test_c, + test_targets = test_T, + total_train_size = str(len(train_data_ld)), + total_test_size = str(len(test_data_ld)), + info = 'all-features', + output_dir = output_dir) + + visualize_umap(train_data = train_X, + train_c = train_c, + train_targets = train_T, + test_data = test_X, + test_c = test_c, + test_targets = test_T, + total_train_size = str(len(train_data_ld)), + total_test_size = str(len(test_data_ld)), + info = 'cont-features', + output_dir = output_dir) + + visualize_umap(train_data = train_cat, + train_c = train_c, + train_targets = train_T, + test_data = test_cat, + test_c = test_c, + test_targets = test_T, + total_train_size = str(len(train_data_ld)), + total_test_size = str(len(test_data_ld)), + info = 'cat-features', + output_dir = output_dir) + + # UMAP for z data + for i in range(0,len(test_z)): + visualize_umap(train_data = train_z[i], + train_targets = train_T, + train_c = train_c, + test_data = test_z[i], + test_c = test_c, + test_targets = test_T, + total_train_size = str(len(train_data_ld)), + total_test_size = str(len(test_data_ld)), + info = 'z-data-'+str(i), + output_dir = output_dir) + + + +def analyse_categorical_data(X_cat, n_days=10, output_dir=""): + + # analyse categorical variables + n_vec = len(X_cat) + n_cat = len(X_cat[0]) + n_days = n_days + + print('n_vec', n_vec, 'n_cat', n_cat) +# for c in train_data.X_cat: +# print(n_cat, c) + + all_cat = np.array(X_cat) + print('all_cat.shape', all_cat.shape) + day_size = all_cat.shape[0]/n_days + + for i in range(0,n_cat): + l_d = [] + l_s1 = [] + l_s2 = [] + l_int = [] + l_rem = [] + + cat = all_cat[:,i] + print('cat', i, cat.shape) + for d in range(1,n_days): + offset = int(d*day_size) + #print(offset) + cat1 = cat[:offset] + cat2 = cat[offset:] + + s1 = set(cat1) + s2 = set(cat2) + + intersect = list(s1 & s2) + #print(intersect) + l_d.append(d) + l_s1.append(len(s1)) + l_s2.append(len(s2)) + l_int.append(len(intersect)) + l_rem.append((len(s1)-len(intersect))) + + print(d, ',', len(s1), ',', len(s2), ',', len(intersect), ',', (len(s1)-len(intersect))) + + print("spit", l_d) + print("before", l_s1) + print("after", l_s2) + print("inters.", l_int) + print("removed", l_rem) + + plt.figure(figsize=(8,8)) + plt.plot(l_d, l_s1, 'g', label='before') + plt.plot(l_d, l_s2, 'r', label='after') + plt.plot(l_d, l_int, 'b', label='intersect') + plt.plot(l_d, l_rem, 'y', label='removed') + plt.title("categorical var. "+str(i)) + plt.legend() + plt.savefig(output_dir+"/cat-"+str(i).zfill(3)+".png") + plt.close() + + +def analyze_model_data(output_dir, + dlrm, + train_ld, + test_ld, + skip_embedding = False, + use_tsne = False, + max_umap_size = 50000, + max_tsne_size = 10000, + skip_categorical_analysis = False, + skip_data_plots = False): + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + if skip_embedding == False: + visualize_embeddings_umap(emb_l = dlrm.emb_l, + output_dir = output_dir, + max_size = max_umap_size) + + if use_tsne == True: + visualize_embeddings_tsne(emb_l = dlrm.emb_l, + output_dir = output_dir, + max_size = max_tsne_size) + + # data visualization and analysis + if skip_data_plots == False: + visualize_data_umap(dlrm=dlrm, train_data_ld=train_ld, test_data_ld=test_ld, max_umap_size=max_umap_size, output_dir=output_dir) + + # analyse categorical variables + if skip_categorical_analysis == False: + analyse_categorical_data(X_cat=train_data.X_cat, n_days=10, output_dir=output_dir) + + if __name__ == "__main__": output_dir = "" @@ -108,29 +506,43 @@ def visualize_embeddings_tsne(emb_l, ) parser.add_argument("--load-model", type=str, default="") - parser.add_argument("--dataset", choices=["kaggle", "terabyte"], help="dataset") + parser.add_argument("--data-set", choices=["kaggle", "terabyte"], help="dataset") # parser.add_argument("--dataset-path", required=True, help="path to the dataset") parser.add_argument("--max-ind-range", type=int, default=-1) # parser.add_argument("--mlperf-bin-loader", action='store_true', default=False) parser.add_argument("--output-dir", type=str, default="") - # umap related + parser.add_argument("--skip-embedding", action='store_true', default=False) + parser.add_argument("--skip-data-plots", action='store_true', default=False) + parser.add_argument("--skip-categorical-analysis", action='store_true', default=False) + + # umap relatet parser.add_argument("--max-umap-size", type=int, default=50000) # tsne related parser.add_argument("--use-tsne", action='store_true', default=False) parser.add_argument("--max-tsne-size", type=int, default=1000) + # data file related + parser.add_argument("--raw-data-file", type=str, default="") + parser.add_argument("--processed-data-file", type=str, default="") + parser.add_argument("--data-sub-sample-rate", type=float, default=0.0) # in [0, 1] + parser.add_argument("--data-randomize", type=str, default="none") # total or day or none + parser.add_argument("--memory-map", action="store_true", default=False) + parser.add_argument("--mini-batch-size", type=int, default=1) + parser.add_argument("--num-workers", type=int, default=0) + parser.add_argument("--test-mini-batch-size", type=int, default=1) + parser.add_argument("--test-num-workers", type=int, default=0) + parser.add_argument("--num-batches", type=int, default=0) + # mlperf logging (disables other output and stops early) + parser.add_argument("--mlperf-logging", action="store_true", default=False) args = parser.parse_args() print('command line args: ', json.dumps(vars(args))) if output_dir == "": - output_dir = args.dataset+"_vis" + output_dir = args.data_set+"_vis_all" print('output_dir:', output_dir) - if not os.path.exists(output_dir): - os.makedirs(output_dir) - - if args.dataset == "kaggle": + if args.data_set == "kaggle": # 1. Criteo Kaggle Display Advertisement Challenge Dataset (see ./bench/dlrm_s_criteo_kaggle.sh) m_spa=16 ln_emb=np.array([1460,583,10131227,2202608,305,24,12517,633,3,93145,5683,8351593,3194,27,14992,5461306,10,5652,2173,4,7046547,18,15,286181,105,142572]) @@ -185,13 +597,24 @@ def visualize_embeddings_tsne(emb_l, print("Model loaded", args.load_model) #print(dlrm) + + # load data + train_data = None + train_ld = None + test_data = None + test_ld = None - visualize_embeddings_umap(emb_l = dlrm.emb_l, - output_dir = output_dir, - max_size = args.max_umap_size) + if args.raw_data_file is not "" or args.processed_data_file is not "": + train_data, train_ld, test_data, test_ld = dp.make_criteo_data_and_loaders(args) - if args.use_tsne == True: - visualize_embeddings_tsne(emb_l = dlrm.emb_l, - output_dir = output_dir, - max_size = args.max_tsne_size) + analyze_model_data(output_dir = output_dir, + dlrm = dlrm, + train_ld = train_ld, + test_ld = test_ld, + skip_embedding = args.skip_embedding, + use_tsne = args.use_tsne, + max_umap_size = args.max_umap_size, + max_tsne_size = args.max_tsne_size, + skip_categorical_analysis = args.skip_categorical_analysis, + skip_data_plots = args.skip_data_plots) From 75b02cf59c1f664cda7b54b8aae113ca67f1beac Mon Sep 17 00:00:00 2001 From: mnaumovfb Date: Sun, 31 May 2020 16:54:08 -0700 Subject: [PATCH 18/57] Adjusting ONNX calls to work with large models (more than 2GB in size). --- dlrm_s_pytorch.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index f81a1849..e0517fc5 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -1216,11 +1216,11 @@ def loss_fn_wrap(Z, T, use_gpu, device): # export the model in onnx if args.save_onnx: - with open("dlrm_s_pytorch.onnx", "w+b") as dlrm_pytorch_onnx_file: - (X, lS_o, lS_i, _) = train_data[0] # get first batch of elements - torch.onnx._export( - dlrm, (X, lS_o, lS_i), dlrm_pytorch_onnx_file, verbose=True - ) + dlrm_pytorch_onnx_file = "dlrm_s_pytorch.onnx" + (X, lS_o, lS_i, _) = train_data[0] # get first batch of elements + torch.onnx.export( + dlrm, (X, lS_o, lS_i), dlrm_pytorch_onnx_file, verbose=True, use_external_data_format=True + ) # recover the model back dlrm_pytorch_onnx = onnx.load("dlrm_s_pytorch.onnx") # check the onnx model From 09017b80257cca866ae264c2cac6fbcc703322fd Mon Sep 17 00:00:00 2001 From: mnaumovfb Date: Fri, 5 Jun 2020 00:13:59 -0700 Subject: [PATCH 19/57] Adjusting the learning rate to freeze at last, when passed the decay interval. --- dlrm_s_pytorch.py | 17 +++- tools/visualize.py | 211 ++++++++++++++++++++++----------------------- 2 files changed, 118 insertions(+), 110 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index e0517fc5..1bb09784 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -99,6 +99,7 @@ class LRPolicyScheduler(_LRScheduler): def __init__(self, optimizer, num_warmup_steps, decay_start_step, num_decay_steps): self.num_warmup_steps = num_warmup_steps self.decay_start_step = decay_start_step + self.decay_end_step = decay_start_step + num_decay_steps self.num_decay_steps = num_decay_steps if self.decay_start_step < self.num_warmup_steps: @@ -108,16 +109,24 @@ def __init__(self, optimizer, num_warmup_steps, decay_start_step, num_decay_step def get_lr(self): step_count = self._step_count - if (self.num_warmup_steps > 0) and (step_count <= self.num_warmup_steps): + if step_count < self.num_warmup_steps: + # warmup scale = 1.0 - (self.num_warmup_steps - step_count) / self.num_warmup_steps lr = [base_lr * scale for base_lr in self.base_lrs] - elif self.num_decay_steps != 0 and step_count >= self.decay_start_step: + elif self.decay_start_step <= step_count and step_count < self.decay_end_step: + # decay decayed_steps = step_count - self.decay_start_step scale = ((self.num_decay_steps - decayed_steps) / self.num_decay_steps) ** 2 min_lr = 0.0000001 lr = [max(min_lr, base_lr * scale) for base_lr in self.base_lrs] + self.last_lr = lr else: - lr = self.base_lrs + if self.num_decay_steps > 0: + # freeze at last + lr = self.last_lr + else: + # do not adjust + lr = self.base_lrs return lr ### define dlrm in PyTorch ### @@ -783,7 +792,7 @@ def parallel_forward(self, dense_x, lS_o, lS_i): # specify the optimizer algorithm optimizer = torch.optim.SGD(dlrm.parameters(), lr=args.learning_rate) lr_scheduler = LRPolicyScheduler(optimizer, args.lr_num_warmup_steps, args.lr_decay_start_step, - args.lr_num_decay_steps) + args.lr_num_decay_steps) ### main loop ### def time_wrap(use_gpu): diff --git a/tools/visualize.py b/tools/visualize.py index b39def85..0ca5d436 100644 --- a/tools/visualize.py +++ b/tools/visualize.py @@ -4,29 +4,29 @@ # LICENSE file in the root directory of this source tree. # # -# This script performs the visualization of the embedding tables created in -# DLRM during the training procedure. We use two popular techniques for -# visualization: umap (https://umap-learn.readthedocs.io/en/latest/) -# and tsne (https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html). -# These links also provide instructions on how to install these packages +# This script performs the visualization of the embedding tables created in +# DLRM during the training procedure. We use two popular techniques for +# visualization: umap (https://umap-learn.readthedocs.io/en/latest/) and +# tsne (https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html). +# These links also provide instructions on how to install these packages # in different environments. # # Warning: the size of the data to be visualized depends on the RAM on your machine. # # -# A sample run of the code, with a kaggle model is shown below -# $ python ./tools/visualize.py –dataset=kaggle --load-model=./input/dlrm_kaggle.pytorch +# A sample run of the code, with a kaggle model is shown below +# $ python ./tools/visualize.py –dataset=kaggle --load-model=./input/dlrm_kaggle.pytorch # # # The following command line arguments are available to the user: # # --load-model - DLRM model file # --data-set - one of ["kaggle", "terabyte"] -# --max-ind-range - max index range used during the traning -# --output-dir - output directory where output plots will be written, default will be on of these: ["kaggle_vis", "terabyte_vis"] +# --max-ind-range - max index range used during the traning +# --output-dir - output directory where output plots will be written, default will be on of these: ["kaggle_vis", "terabyte_vis"] # --max-umap-size - max number of points to visualize using UMAP, default=50000 # --use-tsne - use T-SNE -# --max-tsne-size - max number of points to visualize using T-SNE, default=1000) +# --max-tsne-size - max number of points to visualize using T-SNE, default=1000) # import os @@ -49,13 +49,13 @@ from dlrm_s_pytorch import DLRM_Net -def visualize_embeddings_umap(emb_l, +def visualize_embeddings_umap(emb_l, output_dir = "", max_size = 500000): for k in range(0, len(emb_l)): - E = emb_l[k].weight.detach().cpu() + E = emb_l[k].weight.detach().cpu() print("umap", E.shape) if E.shape[0] < 20: @@ -64,17 +64,17 @@ def visualize_embeddings_umap(emb_l, n_vis = min(max_size, E.shape[0]) -# reducer = umap.UMAP(random_state=42, n_neighbors=25, min_dist=0.1) + # reducer = umap.UMAP(random_state=42, n_neighbors=25, min_dist=0.1) reducer = umap.UMAP(random_state=42) Y = reducer.fit_transform(E[:n_vis,:]) plt.figure(figsize=(8,8)) - + linewidth = 0 size = 1 - + if Y.shape[0] < 2500: - linewidth = 1 + linewidth = 1 size = 5 plt.scatter(-Y[:,0], -Y[:,1], s=size, marker='.', linewidth=linewidth) @@ -84,13 +84,13 @@ def visualize_embeddings_umap(emb_l, plt.close() -def visualize_embeddings_tsne(emb_l, +def visualize_embeddings_tsne(emb_l, output_dir = "", max_size = 10000): for k in range(0, len(emb_l)): - E = emb_l[k].weight.detach().cpu() + E = emb_l[k].weight.detach().cpu() print("tsne", E.shape) if E.shape[0] < 20: @@ -98,26 +98,26 @@ def visualize_embeddings_tsne(emb_l, continue n_vis = min(max_size, E.shape[0]) - + tsne = manifold.TSNE(init='pca', random_state=0, method='exact') - + Y = tsne.fit_transform(E[:n_vis,:]) plt.figure(figsize=(8,8)) linewidth = 0 if Y.shape[0] < 5000: - linewidth = 1 + linewidth = 1 plt.scatter(-Y[:,0], -Y[:,1], s=1, marker='.', linewidth=linewidth) - + plt.title("TSNE: categorical var. "+str(k)+" ("+str(n_vis)+" of "+str(E.shape[0])+")") plt.savefig(output_dir+"/cat-"+str(k)+"-"+str(n_vis)+"-of-"+str(E.shape[0])+"-tsne.png") plt.close() def create_vis_data(dlrm, data_ld, max_size=50000, info=''): - + all_features = [] all_X = [] all_cat = [] @@ -125,17 +125,17 @@ def create_vis_data(dlrm, data_ld, max_size=50000, info=''): all_c = [] all_z = [] all_pred = [] - + z_size = len(dlrm.top_l) print('z_size', z_size) for i in range(0, z_size): all_z.append([]) - + for j, (X, lS_o, lS_i, T) in enumerate(data_ld): if j >= max_size: break - + all_feat_vec = [] all_cat_vec = [] @@ -145,7 +145,7 @@ def create_vis_data(dlrm, data_ld, max_size=50000, info=''): #print(x[0].detach().cpu().numpy()) all_feat_vec.append(x[0].detach().cpu().numpy()) all_X.append(x[0].detach().cpu().numpy()) - + # process sparse features(using embeddings), resulting in a list of row vectors ly = dlrm.apply_emb(lS_o, lS_i, dlrm.emb_l) @@ -160,18 +160,18 @@ def create_vis_data(dlrm, data_ld, max_size=50000, info=''): all_features.append(all_feat_vec) all_cat.append(all_cat_vec) all_T.append(int(T.detach().cpu().numpy()[0,0])) - + z = dlrm.interact_features(x, ly) # print(z.detach().cpu().numpy()) all_z[0].append(z.detach().cpu().numpy().flatten()) - + # obtain probability of a click (using top mlp) -# print(dlrm.top_l) -# p = dlrm.apply_mlp(z, dlrm.top_l) - + # print(dlrm.top_l) + # p = dlrm.apply_mlp(z, dlrm.top_l) + for i in range(0, z_size): z = dlrm.top_l[i](z) - + if i < z_size-1: curr_z = z.detach().cpu().numpy().flatten() all_z[i+1].append(curr_z) @@ -179,7 +179,7 @@ def create_vis_data(dlrm, data_ld, max_size=50000, info=''): # print('z',i, z.detach().cpu().numpy().flatten().shape) p = z - + # clamp output if needed if 0.0 < dlrm.loss_threshold and dlrm.loss_threshold < 1.0: z = torch.clamp(p, min=dlrm.loss_threshold, max=(1.0 - dlrm.loss_threshold)) @@ -187,14 +187,14 @@ def create_vis_data(dlrm, data_ld, max_size=50000, info=''): z = p all_pred.append(int(z.detach().cpu().numpy()[0,0]+0.5)) - + #print(int(z.detach().cpu().numpy()[0,0]+0.5)) if int(z.detach().cpu().numpy()[0,0]+0.5) == int(T.detach().cpu().numpy()[0,0]): all_c.append(0) else: all_c.append(1) - - # calculate classifier metrics + + # calculate classifier metrics ac = accuracy_score(all_T, all_pred) f1 = f1_score(all_T, all_pred) ps = precision_score(all_T, all_pred) @@ -204,15 +204,15 @@ def create_vis_data(dlrm, data_ld, max_size=50000, info=''): return all_features, all_X, all_cat, all_T, all_z, all_c -def plot_all_data(Y_train_data, - train_labels, - Y_test_data, - test_labels, - total_train_size = '', - total_test_size = '', +def plot_all_data(Y_train_data, + train_labels, + Y_test_data, + test_labels, + total_train_size = '', + total_test_size = '', info = '', output_dir = ''): - + size = 1 colors = ['red','green'] @@ -231,17 +231,17 @@ def plot_all_data(Y_train_data, def plot_one_class(Y_train_data, train_labels, - Y_test_data, - test_labels, - label = 0, - col = 'red', - total_train_size = '', - total_test_size = '', + Y_test_data, + test_labels, + label = 0, + col = 'red', + total_train_size = '', + total_test_size = '', info = '', output_dir = ''): - + size = 1 - + fig, (ax1, ax2) = plt.subplots(1, 2) fig.suptitle('UMAP: '+ info ) @@ -253,7 +253,7 @@ def plot_one_class(Y_train_data, if Y_test_data is not None and test_labels is not None: ind_l_test = [i for i,x in enumerate(test_labels) if x == label] Y_test_l = np.array([Y_test_data[i,:] for i in ind_l_test]) - + ax2.scatter(-Y_test_l[:,0], -Y_test_l[:,1], s=size, c=col, marker='.', linewidth=0) ax2.title.set_text('Test, ('+str(len(test_labels))+' of '+ total_test_size+')') @@ -261,17 +261,17 @@ def plot_one_class(Y_train_data, plt.close() -def visualize_umap(train_data, +def visualize_umap(train_data, train_c, - train_targets, + train_targets, test_data = None, test_c = None, - test_targets = None, - total_train_size = '', - total_test_size = '', + test_targets = None, + total_train_size = '', + total_test_size = '', info = '', output_dir = ''): - + # reducer = umap.UMAP(random_state=42, n_neighbors=25, min_dist=0.1) reducer = umap.UMAP(random_state=42) train_Y = reducer.fit_transform(train_data) @@ -281,77 +281,77 @@ def visualize_umap(train_data, # all classes plot_all_data(Y_train_data = train_Y, - train_labels = train_targets, - Y_test_data = test_Y, - test_labels = test_targets, + train_labels = train_targets, + Y_test_data = test_Y, + test_labels = test_targets, total_train_size = total_train_size, total_test_size = total_test_size, info = info, output_dir = output_dir) - + # class 0 plot_one_class(Y_train_data = train_Y, train_labels = train_targets, - Y_test_data = test_Y, - test_labels = test_targets, - label = 0, - col = 'red', - total_train_size = total_train_size, - total_test_size = total_test_size, + Y_test_data = test_Y, + test_labels = test_targets, + label = 0, + col = 'red', + total_train_size = total_train_size, + total_test_size = total_test_size, info = info+' class ' + str(0), output_dir = output_dir) # class 1 plot_one_class(Y_train_data = train_Y, train_labels = train_targets, - Y_test_data = test_Y, - test_labels = test_targets, - label = 1, - col = 'green', - total_train_size = total_train_size, - total_test_size = total_test_size, + Y_test_data = test_Y, + test_labels = test_targets, + label = 1, + col = 'green', + total_train_size = total_train_size, + total_test_size = total_test_size, info = info + ' class ' + str(1), output_dir = output_dir) # correct classification plot_one_class(Y_train_data = train_Y, train_labels = train_c, - Y_test_data = test_Y, - test_labels = test_c, - label = 0, - col = 'green', - total_train_size = total_train_size, - total_test_size = total_test_size, + Y_test_data = test_Y, + test_labels = test_c, + label = 0, + col = 'green', + total_train_size = total_train_size, + total_test_size = total_test_size, info = info + ' correct ', output_dir = output_dir) # errors plot_one_class(Y_train_data = train_Y, train_labels = train_c, - Y_test_data = test_Y, - test_labels = test_c, - label = 1, - col = 'red', - total_train_size = total_train_size, - total_test_size = total_test_size, + Y_test_data = test_Y, + test_labels = test_c, + label = 1, + col = 'red', + total_train_size = total_train_size, + total_test_size = total_test_size, info = info + ' errors ', output_dir = output_dir) -def visualize_data_umap(dlrm, - train_data_ld, - test_data_ld = None, +def visualize_data_umap(dlrm, + train_data_ld, + test_data_ld = None, max_umap_size = 50000, output_dir = ''): train_feat, train_X, train_cat, train_T, train_z, train_c = create_vis_data(dlrm=dlrm, data_ld=train_data_ld, max_size=max_umap_size, info='train') - + test_feat = None test_X = None test_cat = None test_T = None - + if test_data_ld is not None: test_feat, test_X, test_cat, test_T, test_z, test_c = create_vis_data(dlrm=dlrm, data_ld=test_data_ld, max_size=max_umap_size, info='test') @@ -365,7 +365,7 @@ def visualize_data_umap(dlrm, total_test_size = str(len(test_data_ld)), info = 'all-features', output_dir = output_dir) - + visualize_umap(train_data = train_X, train_c = train_c, train_targets = train_T, @@ -376,7 +376,7 @@ def visualize_data_umap(dlrm, total_test_size = str(len(test_data_ld)), info = 'cont-features', output_dir = output_dir) - + visualize_umap(train_data = train_cat, train_c = train_c, train_targets = train_T, @@ -409,7 +409,7 @@ def analyse_categorical_data(X_cat, n_days=10, output_dir=""): n_vec = len(X_cat) n_cat = len(X_cat[0]) n_days = n_days - + print('n_vec', n_vec, 'n_cat', n_cat) # for c in train_data.X_cat: # print(n_cat, c) @@ -436,7 +436,7 @@ def analyse_categorical_data(X_cat, n_days=10, output_dir=""): s1 = set(cat1) s2 = set(cat2) - intersect = list(s1 & s2) + intersect = list(s1 & s2) #print(intersect) l_d.append(d) l_s1.append(len(s1)) @@ -499,7 +499,7 @@ def analyze_model_data(output_dir, if __name__ == "__main__": output_dir = "" - + ### parse arguments ### parser = argparse.ArgumentParser( description="Exploratory DLRM analysis" @@ -507,14 +507,14 @@ def analyze_model_data(output_dir, parser.add_argument("--load-model", type=str, default="") parser.add_argument("--data-set", choices=["kaggle", "terabyte"], help="dataset") -# parser.add_argument("--dataset-path", required=True, help="path to the dataset") + # parser.add_argument("--dataset-path", required=True, help="path to the dataset") parser.add_argument("--max-ind-range", type=int, default=-1) -# parser.add_argument("--mlperf-bin-loader", action='store_true', default=False) + # parser.add_argument("--mlperf-bin-loader", action='store_true', default=False) parser.add_argument("--output-dir", type=str, default="") parser.add_argument("--skip-embedding", action='store_true', default=False) parser.add_argument("--skip-data-plots", action='store_true', default=False) parser.add_argument("--skip-categorical-analysis", action='store_true', default=False) - + # umap relatet parser.add_argument("--max-umap-size", type=int, default=50000) # tsne related @@ -530,7 +530,7 @@ def analyze_model_data(output_dir, parser.add_argument("--num-workers", type=int, default=0) parser.add_argument("--test-mini-batch-size", type=int, default=1) parser.add_argument("--test-num-workers", type=int, default=0) - parser.add_argument("--num-batches", type=int, default=0) + parser.add_argument("--num-batches", type=int, default=0) # mlperf logging (disables other output and stops early) parser.add_argument("--mlperf-logging", action="store_true", default=False) @@ -541,20 +541,20 @@ def analyze_model_data(output_dir, if output_dir == "": output_dir = args.data_set+"_vis_all" print('output_dir:', output_dir) - + if args.data_set == "kaggle": # 1. Criteo Kaggle Display Advertisement Challenge Dataset (see ./bench/dlrm_s_criteo_kaggle.sh) m_spa=16 ln_emb=np.array([1460,583,10131227,2202608,305,24,12517,633,3,93145,5683,8351593,3194,27,14992,5461306,10,5652,2173,4,7046547,18,15,286181,105,142572]) ln_bot=np.array([13,512,256,64,16]) ln_top=np.array([367,512,256,1]) - + elif args.dataset == "terabyte": if args.max_ind_range == 10000000: # 2. Criteo Terabyte (see ./bench/dlrm_s_criteo_terabyte.sh [--sub-sample=0.875] --max-in-range=10000000) m_spa=64 - ln_emb=np.array([9980333,36084,17217,7378,20134,3,7112,1442,61, 9758201,1333352,313829,10,2208,11156,122,4,970,14, 9994222, 7267859, 9946608,415421,12420,101, 36]) + ln_emb=np.array([9980333,36084,17217,7378,20134,3,7112,1442,61,9758201,1333352,313829,10,2208,11156,122,4,970,14, 9994222, 7267859, 9946608,415421,12420,101, 36]) ln_bot=np.array([13,512,256,64]) ln_top=np.array([415,512,512,256,1]) elif args.max_ind_range == 40000000: @@ -603,7 +603,7 @@ def analyze_model_data(output_dir, train_ld = None test_data = None test_ld = None - + if args.raw_data_file is not "" or args.processed_data_file is not "": train_data, train_ld, test_data, test_ld = dp.make_criteo_data_and_loaders(args) @@ -617,4 +617,3 @@ def analyze_model_data(output_dir, max_tsne_size = args.max_tsne_size, skip_categorical_analysis = args.skip_categorical_analysis, skip_data_plots = args.skip_data_plots) - From 1f2589252d2bc251e8537422f42f71ecf1ffca80 Mon Sep 17 00:00:00 2001 From: tginart Date: Sun, 7 Jun 2020 23:14:29 -0700 Subject: [PATCH 20/57] Mixd Bugfixes (#87) * bugfixes for mixd * remove whitespace --- dlrm_s_pytorch.py | 4 ++-- tricks/md_embedding_bag.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 1bb09784..1955bb9c 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -179,9 +179,9 @@ def create_emb(self, m, ln): if self.qr_flag and n > self.qr_threshold: EE = QREmbeddingBag(n, m, self.qr_collisions, operation=self.qr_operation, mode="sum", sparse=True) - elif self.md_flag and n > self.md_threshold: - _m = m[i] + elif self.md_flag: base = max(m) + _m = m[i] if n > self.md_threshold else base EE = PrEmbeddingBag(n, _m, base) # use np initialization as below for consistency... W = np.random.uniform( diff --git a/tricks/md_embedding_bag.py b/tricks/md_embedding_bag.py index 53c9f7af..7c4071a2 100644 --- a/tricks/md_embedding_bag.py +++ b/tricks/md_embedding_bag.py @@ -34,7 +34,10 @@ def md_solver(n, alpha, d0=None, B=None, round_dim=True, k=None): d = alpha_power_rule(n.type(torch.float) / k, alpha, d0=d0, B=B) if round_dim: d = pow_2_round(d) - return d + undo_sort = [0] * len(indices) + for i, v in enumerate(indices): + undo_sort[v] = i + return d[undo_sort] def alpha_power_rule(n, alpha, d0=None, B=None): From 236e33110e5ce5671ce4ce4162a929c47e36cebc Mon Sep 17 00:00:00 2001 From: Taylan Bilal Date: Thu, 11 Jun 2020 20:47:09 -0700 Subject: [PATCH 21/57] Added gitignore from https://github.com/github/gitignore/blob/master/Python.gitignore . (#91) --- .gitignore | 138 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..a81c8ee1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,138 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ From 32181f77c6d5f6bba25c882cce200e21d0f1f53d Mon Sep 17 00:00:00 2001 From: mnaumovfb Date: Thu, 11 Jun 2020 23:17:33 -0700 Subject: [PATCH 22/57] Adjusting parameters for onnx.export to work with any data loader. --- dlrm_s_pytorch.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 1955bb9c..2c419bb7 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -918,6 +918,9 @@ def loss_fn_wrap(Z, T, use_gpu, device): previous_iteration_time = None for j, (X, lS_o, lS_i, T) in enumerate(train_ld): + if j == 0 and args.save_onnx: + (X_onnx, lS_o_onnx, lS_i_onnx) = (X, lS_o, lS_i) + if j < skip_upto_batch: continue @@ -1226,9 +1229,8 @@ def loss_fn_wrap(Z, T, use_gpu, device): # export the model in onnx if args.save_onnx: dlrm_pytorch_onnx_file = "dlrm_s_pytorch.onnx" - (X, lS_o, lS_i, _) = train_data[0] # get first batch of elements torch.onnx.export( - dlrm, (X, lS_o, lS_i), dlrm_pytorch_onnx_file, verbose=True, use_external_data_format=True + dlrm, (X_onnx, lS_o_onnx, lS_i_onnx), dlrm_pytorch_onnx_file, verbose=True, use_external_data_format=True ) # recover the model back dlrm_pytorch_onnx = onnx.load("dlrm_s_pytorch.onnx") From 6bd3adbb6846668c861f19483e949eeb3d0f53e8 Mon Sep 17 00:00:00 2001 From: mnaumovfb Date: Thu, 11 Jun 2020 23:39:28 -0700 Subject: [PATCH 23/57] Fixing a typo. --- tools/visualize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/visualize.py b/tools/visualize.py index 0ca5d436..7a219ee0 100644 --- a/tools/visualize.py +++ b/tools/visualize.py @@ -15,7 +15,7 @@ # # # A sample run of the code, with a kaggle model is shown below -# $ python ./tools/visualize.py –dataset=kaggle --load-model=./input/dlrm_kaggle.pytorch +# $python ./tools/visualize.py --dataset=kaggle --load-model=./input/dlrm_kaggle.pytorch # # # The following command line arguments are available to the user: From ae23fca0e622d4003c208b5f7f2ced159792f71f Mon Sep 17 00:00:00 2001 From: Tomasz Grel Date: Fri, 12 Jun 2020 21:55:34 +0200 Subject: [PATCH 24/57] Tgrel/tgrel mlperf fixes (#93) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix LR decay – allow a period of training with a constant base LR between warmup end step and decay start step * Bump pytorch version for multiGPU memory corruption bugfix --- Dockerfile | 2 ++ dlrm_s_pytorch.py | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c4b67626..0e4b7500 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,5 +9,7 @@ FROM ${FROM_IMAGE_NAME} ADD requirements.txt . RUN pip install -r requirements.txt +RUN pip install torch==1.3.1 + WORKDIR /code ADD . . diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 2c419bb7..344c1679 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -113,6 +113,7 @@ def get_lr(self): # warmup scale = 1.0 - (self.num_warmup_steps - step_count) / self.num_warmup_steps lr = [base_lr * scale for base_lr in self.base_lrs] + self.last_lr = lr elif self.decay_start_step <= step_count and step_count < self.decay_end_step: # decay decayed_steps = step_count - self.decay_start_step @@ -122,7 +123,8 @@ def get_lr(self): self.last_lr = lr else: if self.num_decay_steps > 0: - # freeze at last + # freeze at last, either because we're after decay + # or because we're between warmup and decay lr = self.last_lr else: # do not adjust From 3ecf64182e552e90641e07292c4042d16b96aff7 Mon Sep 17 00:00:00 2001 From: Hu Wan Date: Thu, 25 Jun 2020 10:03:45 +0900 Subject: [PATCH 25/57] Trimming trailing whitespaces (#100) --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8cc3c108..62433ae5 100644 --- a/README.md +++ b/README.md @@ -296,7 +296,7 @@ Benchmarking ``` ./bench/dlrm_s_criteo_kaggle.sh [--test-freq=1024] ``` - + 3) The code supports interface with the [Criteo Terabyte Dataset](https://labs.criteo.com/2013/12/download-terabyte-click-logs/). @@ -311,23 +311,23 @@ Benchmarking ./bench/dlrm_s_criteo_terabyte.sh ["--test-freq=10240 --memory-map --data-sub-sample-rate=0.875"] ``` - Corresponding pre-trained model is available under [CC-BY-NC license](https://creativecommons.org/licenses/by-nc/2.0/) and can be downloaded here - [dlrm_emb64_subsample0.875_maxindrange10M_pretrained.pt](https://dlrm.s3-us-west-1.amazonaws.com/models/tb0875_10M.pt) + [dlrm_emb64_subsample0.875_maxindrange10M_pretrained.pt](https://dlrm.s3-us-west-1.amazonaws.com/models/tb0875_10M.pt) *NOTE: Benchmarking scripts accept extra arguments which will be passed along to the model, such as --num-batches=100 to limit the number of data samples* -4) The code supports interface with [MLPerf benchmark](https://mlperf.org). +4) The code supports interface with [MLPerf benchmark](https://mlperf.org). - Please refer to the following training parameters ``` --mlperf-logging that keeps track of multiple metrics, including area under the curve (AUC) - + --mlperf-acc-threshold that allows early stopping based on accuracy metric - + --mlperf-auc-threshold that allows early stopping based on AUC metric - + --mlperf-bin-loader that enables preprocessing of data into a single binary file - + --mlperf-bin-shuffle that controls whether a random shuffle of mini-batches is performed ``` - The MLPerf training model is completely specified and can be trained using the following script From d54c813a086f09a911b80113830a487e1a8ec01a Mon Sep 17 00:00:00 2001 From: Hu Wan Date: Thu, 25 Jun 2020 10:04:24 +0900 Subject: [PATCH 26/57] Adding tqdm package in requirements (#99) --- README.md | 2 ++ requirements.txt | 1 + 2 files changed, 3 insertions(+) diff --git a/README.md b/README.md index 62433ae5..341f749e 100644 --- a/README.md +++ b/README.md @@ -367,6 +367,8 @@ pydot (*optional*) torchviz (*optional*) +tqdm + License ------- diff --git a/requirements.txt b/requirements.txt index c5cad56a..b198a127 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ pydot torch torchviz scikit-learn +tqdm From ce31edae8e7513276009f24a6f4672a77b2f27c7 Mon Sep 17 00:00:00 2001 From: dkorchevgithub <63178227+dkorchevgithub@users.noreply.github.com> Date: Mon, 29 Jun 2020 08:58:23 -0700 Subject: [PATCH 27/57] latest updates, 2020-06-27 (#103) * sunc 2020-06-26 * cleanup, format, testing after all updates --- tools/visualize.py | 1649 +++++++++++++++++++++++++++----------------- 1 file changed, 1030 insertions(+), 619 deletions(-) mode change 100644 => 100755 tools/visualize.py diff --git a/tools/visualize.py b/tools/visualize.py old mode 100644 new mode 100755 index 7a219ee0..f16504cb --- a/tools/visualize.py +++ b/tools/visualize.py @@ -1,619 +1,1030 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. -# -# -# This script performs the visualization of the embedding tables created in -# DLRM during the training procedure. We use two popular techniques for -# visualization: umap (https://umap-learn.readthedocs.io/en/latest/) and -# tsne (https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html). -# These links also provide instructions on how to install these packages -# in different environments. -# -# Warning: the size of the data to be visualized depends on the RAM on your machine. -# -# -# A sample run of the code, with a kaggle model is shown below -# $python ./tools/visualize.py --dataset=kaggle --load-model=./input/dlrm_kaggle.pytorch -# -# -# The following command line arguments are available to the user: -# -# --load-model - DLRM model file -# --data-set - one of ["kaggle", "terabyte"] -# --max-ind-range - max index range used during the traning -# --output-dir - output directory where output plots will be written, default will be on of these: ["kaggle_vis", "terabyte_vis"] -# --max-umap-size - max number of points to visualize using UMAP, default=50000 -# --use-tsne - use T-SNE -# --max-tsne-size - max number of points to visualize using T-SNE, default=1000) -# - -import os -import argparse -import numpy as np -import umap -import json -import torch -import matplotlib -import matplotlib.pyplot as plt - -from sklearn.metrics import accuracy_score -from sklearn.metrics import f1_score -from sklearn.metrics import precision_score -from sklearn.metrics import recall_score - -from sklearn import manifold - -import dlrm_data_pytorch as dp -from dlrm_s_pytorch import DLRM_Net - - -def visualize_embeddings_umap(emb_l, - output_dir = "", - max_size = 500000): - - for k in range(0, len(emb_l)): - - E = emb_l[k].weight.detach().cpu() - print("umap", E.shape) - - if E.shape[0] < 20: - print("Skipping small embedding") - continue - - n_vis = min(max_size, E.shape[0]) - - # reducer = umap.UMAP(random_state=42, n_neighbors=25, min_dist=0.1) - reducer = umap.UMAP(random_state=42) - Y = reducer.fit_transform(E[:n_vis,:]) - - plt.figure(figsize=(8,8)) - - linewidth = 0 - size = 1 - - if Y.shape[0] < 2500: - linewidth = 1 - size = 5 - - plt.scatter(-Y[:,0], -Y[:,1], s=size, marker='.', linewidth=linewidth) - - plt.title("UMAP: categorical var. "+str(k)+" ("+str(n_vis)+" of "+str(E.shape[0])+")") - plt.savefig(output_dir+"/cat-"+str(k)+"-"+str(n_vis)+"-of-"+str(E.shape[0])+"-umap.png") - plt.close() - - -def visualize_embeddings_tsne(emb_l, - output_dir = "", - max_size = 10000): - - for k in range(0, len(emb_l)): - - E = emb_l[k].weight.detach().cpu() - print("tsne", E.shape) - - if E.shape[0] < 20: - print("Skipping small embedding") - continue - - n_vis = min(max_size, E.shape[0]) - - tsne = manifold.TSNE(init='pca', random_state=0, method='exact') - - Y = tsne.fit_transform(E[:n_vis,:]) - - plt.figure(figsize=(8,8)) - - linewidth = 0 - if Y.shape[0] < 5000: - linewidth = 1 - - plt.scatter(-Y[:,0], -Y[:,1], s=1, marker='.', linewidth=linewidth) - - plt.title("TSNE: categorical var. "+str(k)+" ("+str(n_vis)+" of "+str(E.shape[0])+")") - plt.savefig(output_dir+"/cat-"+str(k)+"-"+str(n_vis)+"-of-"+str(E.shape[0])+"-tsne.png") - plt.close() - - -def create_vis_data(dlrm, data_ld, max_size=50000, info=''): - - all_features = [] - all_X = [] - all_cat = [] - all_T = [] - all_c = [] - all_z = [] - all_pred = [] - - z_size = len(dlrm.top_l) - print('z_size', z_size) - for i in range(0, z_size): - all_z.append([]) - - for j, (X, lS_o, lS_i, T) in enumerate(data_ld): - - if j >= max_size: - break - - all_feat_vec = [] - all_cat_vec = [] - - x = dlrm.apply_mlp(X, dlrm.bot_l) - # debug prints - #print("intermediate") - #print(x[0].detach().cpu().numpy()) - all_feat_vec.append(x[0].detach().cpu().numpy()) - all_X.append(x[0].detach().cpu().numpy()) - - # process sparse features(using embeddings), resulting in a list of row vectors - ly = dlrm.apply_emb(lS_o, lS_i, dlrm.emb_l) - - for e in ly: - #print(e.detach().cpu().numpy()) - all_feat_vec.append(e[0].detach().cpu().numpy()) - all_cat_vec.append(e[0].detach().cpu().numpy()) - - all_feat_vec= np.concatenate(all_feat_vec, axis=0) - all_cat_vec= np.concatenate(all_cat_vec, axis=0) - - all_features.append(all_feat_vec) - all_cat.append(all_cat_vec) - all_T.append(int(T.detach().cpu().numpy()[0,0])) - - z = dlrm.interact_features(x, ly) - # print(z.detach().cpu().numpy()) - all_z[0].append(z.detach().cpu().numpy().flatten()) - - # obtain probability of a click (using top mlp) - # print(dlrm.top_l) - # p = dlrm.apply_mlp(z, dlrm.top_l) - - for i in range(0, z_size): - z = dlrm.top_l[i](z) - - if i < z_size-1: - curr_z = z.detach().cpu().numpy().flatten() - all_z[i+1].append(curr_z) - - # print('z',i, z.detach().cpu().numpy().flatten().shape) - - p = z - - # clamp output if needed - if 0.0 < dlrm.loss_threshold and dlrm.loss_threshold < 1.0: - z = torch.clamp(p, min=dlrm.loss_threshold, max=(1.0 - dlrm.loss_threshold)) - else: - z = p - - all_pred.append(int(z.detach().cpu().numpy()[0,0]+0.5)) - - #print(int(z.detach().cpu().numpy()[0,0]+0.5)) - if int(z.detach().cpu().numpy()[0,0]+0.5) == int(T.detach().cpu().numpy()[0,0]): - all_c.append(0) - else: - all_c.append(1) - - # calculate classifier metrics - ac = accuracy_score(all_T, all_pred) - f1 = f1_score(all_T, all_pred) - ps = precision_score(all_T, all_pred) - rc = recall_score(all_T, all_pred) - - print(info, 'accuracy', ac, 'f1', f1, 'precision', ps, 'recall', rc) - - return all_features, all_X, all_cat, all_T, all_z, all_c - -def plot_all_data(Y_train_data, - train_labels, - Y_test_data, - test_labels, - total_train_size = '', - total_test_size = '', - info = '', - output_dir = ''): - - size = 1 - colors = ['red','green'] - - fig, (ax1, ax2) = plt.subplots(1, 2) - fig.suptitle('UMAP: ' + info) - - ax1.scatter(-Y_train_data[:,0], -Y_train_data[:,1], s=size, c=train_labels, cmap=matplotlib.colors.ListedColormap(colors), marker='.', linewidth=0) - ax1.title.set_text('Train ('+str(len(train_labels))+' of '+ total_train_size+')') - if test_data is not None and test_labels is not None: - ax2.scatter(-Y_test_data[:,0], -Y_test_data[:,1], s=size, c=test_labels, cmap=matplotlib.colors.ListedColormap(colors), marker='.', linewidth=0) - ax2.title.set_text('Test ('+str(len(test_labels))+' of '+ total_test_size+')') - - plt.savefig(output_dir+"/"+info+'-umap.png') - plt.close() - - -def plot_one_class(Y_train_data, - train_labels, - Y_test_data, - test_labels, - label = 0, - col = 'red', - total_train_size = '', - total_test_size = '', - info = '', - output_dir = ''): - - size = 1 - - fig, (ax1, ax2) = plt.subplots(1, 2) - fig.suptitle('UMAP: '+ info ) - - ind_l_train = [i for i,x in enumerate(train_labels) if x == label] - Y_train_l = np.array([Y_train_data[i,:] for i in ind_l_train]) - - ax1.scatter(-Y_train_l[:,0], -Y_train_l[:,1], s=size, c=col, marker='.', linewidth=0) - ax1.title.set_text('Train, ('+str(len(train_labels))+' of '+ total_train_size+')') - if Y_test_data is not None and test_labels is not None: - ind_l_test = [i for i,x in enumerate(test_labels) if x == label] - Y_test_l = np.array([Y_test_data[i,:] for i in ind_l_test]) - - ax2.scatter(-Y_test_l[:,0], -Y_test_l[:,1], s=size, c=col, marker='.', linewidth=0) - ax2.title.set_text('Test, ('+str(len(test_labels))+' of '+ total_test_size+')') - - plt.savefig(output_dir+"/"+info+'-umap.png') - plt.close() - - -def visualize_umap(train_data, - train_c, - train_targets, - test_data = None, - test_c = None, - test_targets = None, - total_train_size = '', - total_test_size = '', - info = '', - output_dir = ''): - -# reducer = umap.UMAP(random_state=42, n_neighbors=25, min_dist=0.1) - reducer = umap.UMAP(random_state=42) - train_Y = reducer.fit_transform(train_data) - - if test_data is not None and test_targets is not None: - test_Y = reducer.transform(test_data) - - # all classes - plot_all_data(Y_train_data = train_Y, - train_labels = train_targets, - Y_test_data = test_Y, - test_labels = test_targets, - total_train_size = total_train_size, - total_test_size = total_test_size, - info = info, - output_dir = output_dir) - - # class 0 - plot_one_class(Y_train_data = train_Y, - train_labels = train_targets, - Y_test_data = test_Y, - test_labels = test_targets, - label = 0, - col = 'red', - total_train_size = total_train_size, - total_test_size = total_test_size, - info = info+' class ' + str(0), - output_dir = output_dir) - - # class 1 - plot_one_class(Y_train_data = train_Y, - train_labels = train_targets, - Y_test_data = test_Y, - test_labels = test_targets, - label = 1, - col = 'green', - total_train_size = total_train_size, - total_test_size = total_test_size, - info = info + ' class ' + str(1), - output_dir = output_dir) - - # correct classification - plot_one_class(Y_train_data = train_Y, - train_labels = train_c, - Y_test_data = test_Y, - test_labels = test_c, - label = 0, - col = 'green', - total_train_size = total_train_size, - total_test_size = total_test_size, - info = info + ' correct ', - output_dir = output_dir) - - # errors - plot_one_class(Y_train_data = train_Y, - train_labels = train_c, - Y_test_data = test_Y, - test_labels = test_c, - label = 1, - col = 'red', - total_train_size = total_train_size, - total_test_size = total_test_size, - info = info + ' errors ', - output_dir = output_dir) - - - -def visualize_data_umap(dlrm, - train_data_ld, - test_data_ld = None, - max_umap_size = 50000, - output_dir = ''): - - train_feat, train_X, train_cat, train_T, train_z, train_c = create_vis_data(dlrm=dlrm, data_ld=train_data_ld, max_size=max_umap_size, info='train') - - test_feat = None - test_X = None - test_cat = None - test_T = None - - if test_data_ld is not None: - test_feat, test_X, test_cat, test_T, test_z, test_c = create_vis_data(dlrm=dlrm, data_ld=test_data_ld, max_size=max_umap_size, info='test') - - visualize_umap(train_data = train_feat, - train_targets = train_T, - train_c = train_c, - test_data = test_feat, - test_c = test_c, - test_targets = test_T, - total_train_size = str(len(train_data_ld)), - total_test_size = str(len(test_data_ld)), - info = 'all-features', - output_dir = output_dir) - - visualize_umap(train_data = train_X, - train_c = train_c, - train_targets = train_T, - test_data = test_X, - test_c = test_c, - test_targets = test_T, - total_train_size = str(len(train_data_ld)), - total_test_size = str(len(test_data_ld)), - info = 'cont-features', - output_dir = output_dir) - - visualize_umap(train_data = train_cat, - train_c = train_c, - train_targets = train_T, - test_data = test_cat, - test_c = test_c, - test_targets = test_T, - total_train_size = str(len(train_data_ld)), - total_test_size = str(len(test_data_ld)), - info = 'cat-features', - output_dir = output_dir) - - # UMAP for z data - for i in range(0,len(test_z)): - visualize_umap(train_data = train_z[i], - train_targets = train_T, - train_c = train_c, - test_data = test_z[i], - test_c = test_c, - test_targets = test_T, - total_train_size = str(len(train_data_ld)), - total_test_size = str(len(test_data_ld)), - info = 'z-data-'+str(i), - output_dir = output_dir) - - - -def analyse_categorical_data(X_cat, n_days=10, output_dir=""): - - # analyse categorical variables - n_vec = len(X_cat) - n_cat = len(X_cat[0]) - n_days = n_days - - print('n_vec', n_vec, 'n_cat', n_cat) -# for c in train_data.X_cat: -# print(n_cat, c) - - all_cat = np.array(X_cat) - print('all_cat.shape', all_cat.shape) - day_size = all_cat.shape[0]/n_days - - for i in range(0,n_cat): - l_d = [] - l_s1 = [] - l_s2 = [] - l_int = [] - l_rem = [] - - cat = all_cat[:,i] - print('cat', i, cat.shape) - for d in range(1,n_days): - offset = int(d*day_size) - #print(offset) - cat1 = cat[:offset] - cat2 = cat[offset:] - - s1 = set(cat1) - s2 = set(cat2) - - intersect = list(s1 & s2) - #print(intersect) - l_d.append(d) - l_s1.append(len(s1)) - l_s2.append(len(s2)) - l_int.append(len(intersect)) - l_rem.append((len(s1)-len(intersect))) - - print(d, ',', len(s1), ',', len(s2), ',', len(intersect), ',', (len(s1)-len(intersect))) - - print("spit", l_d) - print("before", l_s1) - print("after", l_s2) - print("inters.", l_int) - print("removed", l_rem) - - plt.figure(figsize=(8,8)) - plt.plot(l_d, l_s1, 'g', label='before') - plt.plot(l_d, l_s2, 'r', label='after') - plt.plot(l_d, l_int, 'b', label='intersect') - plt.plot(l_d, l_rem, 'y', label='removed') - plt.title("categorical var. "+str(i)) - plt.legend() - plt.savefig(output_dir+"/cat-"+str(i).zfill(3)+".png") - plt.close() - - -def analyze_model_data(output_dir, - dlrm, - train_ld, - test_ld, - skip_embedding = False, - use_tsne = False, - max_umap_size = 50000, - max_tsne_size = 10000, - skip_categorical_analysis = False, - skip_data_plots = False): - - if not os.path.exists(output_dir): - os.makedirs(output_dir) - - if skip_embedding == False: - visualize_embeddings_umap(emb_l = dlrm.emb_l, - output_dir = output_dir, - max_size = max_umap_size) - - if use_tsne == True: - visualize_embeddings_tsne(emb_l = dlrm.emb_l, - output_dir = output_dir, - max_size = max_tsne_size) - - # data visualization and analysis - if skip_data_plots == False: - visualize_data_umap(dlrm=dlrm, train_data_ld=train_ld, test_data_ld=test_ld, max_umap_size=max_umap_size, output_dir=output_dir) - - # analyse categorical variables - if skip_categorical_analysis == False: - analyse_categorical_data(X_cat=train_data.X_cat, n_days=10, output_dir=output_dir) - - -if __name__ == "__main__": - - output_dir = "" - - ### parse arguments ### - parser = argparse.ArgumentParser( - description="Exploratory DLRM analysis" - ) - - parser.add_argument("--load-model", type=str, default="") - parser.add_argument("--data-set", choices=["kaggle", "terabyte"], help="dataset") - # parser.add_argument("--dataset-path", required=True, help="path to the dataset") - parser.add_argument("--max-ind-range", type=int, default=-1) - # parser.add_argument("--mlperf-bin-loader", action='store_true', default=False) - parser.add_argument("--output-dir", type=str, default="") - parser.add_argument("--skip-embedding", action='store_true', default=False) - parser.add_argument("--skip-data-plots", action='store_true', default=False) - parser.add_argument("--skip-categorical-analysis", action='store_true', default=False) - - # umap relatet - parser.add_argument("--max-umap-size", type=int, default=50000) - # tsne related - parser.add_argument("--use-tsne", action='store_true', default=False) - parser.add_argument("--max-tsne-size", type=int, default=1000) - # data file related - parser.add_argument("--raw-data-file", type=str, default="") - parser.add_argument("--processed-data-file", type=str, default="") - parser.add_argument("--data-sub-sample-rate", type=float, default=0.0) # in [0, 1] - parser.add_argument("--data-randomize", type=str, default="none") # total or day or none - parser.add_argument("--memory-map", action="store_true", default=False) - parser.add_argument("--mini-batch-size", type=int, default=1) - parser.add_argument("--num-workers", type=int, default=0) - parser.add_argument("--test-mini-batch-size", type=int, default=1) - parser.add_argument("--test-num-workers", type=int, default=0) - parser.add_argument("--num-batches", type=int, default=0) - # mlperf logging (disables other output and stops early) - parser.add_argument("--mlperf-logging", action="store_true", default=False) - - args = parser.parse_args() - - print('command line args: ', json.dumps(vars(args))) - - if output_dir == "": - output_dir = args.data_set+"_vis_all" - print('output_dir:', output_dir) - - if args.data_set == "kaggle": - # 1. Criteo Kaggle Display Advertisement Challenge Dataset (see ./bench/dlrm_s_criteo_kaggle.sh) - m_spa=16 - ln_emb=np.array([1460,583,10131227,2202608,305,24,12517,633,3,93145,5683,8351593,3194,27,14992,5461306,10,5652,2173,4,7046547,18,15,286181,105,142572]) - ln_bot=np.array([13,512,256,64,16]) - ln_top=np.array([367,512,256,1]) - - elif args.dataset == "terabyte": - - if args.max_ind_range == 10000000: - # 2. Criteo Terabyte (see ./bench/dlrm_s_criteo_terabyte.sh [--sub-sample=0.875] --max-in-range=10000000) - m_spa=64 - ln_emb=np.array([9980333,36084,17217,7378,20134,3,7112,1442,61,9758201,1333352,313829,10,2208,11156,122,4,970,14, 9994222, 7267859, 9946608,415421,12420,101, 36]) - ln_bot=np.array([13,512,256,64]) - ln_top=np.array([415,512,512,256,1]) - elif args.max_ind_range == 40000000: - # 3. Criteo Terabyte MLPerf training (see ./bench/run_and_time.sh --max-in-range=40000000) - m_spa=128 - ln_emb=np.array([39884406,39043,17289,7420,20263,3,7120,1543,63,38532951,2953546,403346,10,2208,11938,155,4,976,14,39979771,25641295,39664984,585935,12972,108,36]) - ln_bot=np.array([13,512,256,128]) - ln_top=np.array([479,1024,1024,512,256,1]) - else: - raise ValueError("only --max-in-range 10M or 40M is supported") - else: - raise ValueError("only kaggle|terabyte dataset options are supported") - - dlrm = DLRM_Net( - m_spa, - ln_emb, - ln_bot, - ln_top, - arch_interaction_op="dot", - arch_interaction_itself=False, - sigmoid_bot=-1, - sigmoid_top=ln_top.size - 2, - sync_dense_params=True, - loss_threshold=0.0, - ndevices=-1, - qr_flag=False, - qr_operation=None, - qr_collisions=None, - qr_threshold=None, - md_flag=False, - md_threshold=None, - ) - - # Load model is specified - if not (args.load_model == ""): - print("Loading saved model {}".format(args.load_model)) - - ld_model = torch.load(args.load_model, map_location=torch.device('cpu')) - dlrm.load_state_dict(ld_model["state_dict"]) - - print("Model loaded", args.load_model) - #print(dlrm) - - # load data - train_data = None - train_ld = None - test_data = None - test_ld = None - - if args.raw_data_file is not "" or args.processed_data_file is not "": - train_data, train_ld, test_data, test_ld = dp.make_criteo_data_and_loaders(args) - - analyze_model_data(output_dir = output_dir, - dlrm = dlrm, - train_ld = train_ld, - test_ld = test_ld, - skip_embedding = args.skip_embedding, - use_tsne = args.use_tsne, - max_umap_size = args.max_umap_size, - max_tsne_size = args.max_tsne_size, - skip_categorical_analysis = args.skip_categorical_analysis, - skip_data_plots = args.skip_data_plots) +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +# +# +# This script performs the visualization of the embedding tables created in +# DLRM during the training procedure. We use two popular techniques for +# visualization: umap (https://umap-learn.readthedocs.io/en/latest/) and +# tsne (https://scikit-learn.org/stable/modules/generated/sklearn.manifold.TSNE.html). +# These links also provide instructions on how to install these packages +# in different environments. +# +# Warning: the size of the data to be visualized depends on the RAM on your machine. +# +# +# Connand line examples: +# +# Full analysis of embeddings and data representations for Criteo Kaggle data: +# $python ./tools/visualize.py --data-set=kaggle --load-model=../dlrm-2020-05-25/criteo.pytorch-e-0-i-110591 +# --raw-data-file=../../criteo/input/train.txt --skip-categorical-analysis +# --processed-data-file=../../criteo/input/kaggleAdDisplayChallenge_processed.npz +# +# +# To run just the analysis of categoricala data for Criteo Kaggle data set: +# $python ./tools/visualize.py --data-set=kaggle --load-model=../dlrm-2020-05-25/criteo.pytorch-e-0-i-110591 \ +# --raw-data-file=../../criteo/input/train.txt --data-randomize=none --processed-data-file=../../criteo/input/kaggleAdDisplayChallenge_processed.npz \ +# --skip-embedding --skip-data-plots +# +# +# The following command line arguments are available to the user: +# +# --load-model - DLRM model file +# --data-set - one of ["kaggle", "terabyte"] +# --max-ind-range - max index range used during the traning +# --output-dir - output directory, if not specified, it will be traeted from the model and datset names +# --max-umap-size - max number of points to visualize using UMAP, default=50000 +# --use-tsne - use T-SNE +# --max-tsne-size - max number of points to visualize using T-SNE, default=1000) +# --skip-embedding - skips analysis of embedding tables +# --umap-metric - metric for UMAP +# --skip-data-plots - skips data plots +# --skip-categorical-analysis - skips categorical analysis +# +# # data file related +# --raw-data-file +# --processed-data-file +# --data-sub-sample-rate +# --data-randomize +# --memory-map +# --mini-batch-size +# --num-workers +# --test-mini-batch-size +# --test-num-workers +# --num-batches +# --mlperf-logging + +import os +import sys +import argparse +import numpy as np +import umap +import hdbscan +import json +import torch +import math +import matplotlib +import matplotlib.pyplot as plt +import collections + +from sklearn.metrics import accuracy_score +from sklearn.metrics import f1_score +from sklearn.metrics import precision_score +from sklearn.metrics import recall_score + +from sklearn import manifold + +import dlrm_data_pytorch as dp +from dlrm_s_pytorch import DLRM_Net + + +def visualize_embeddings_umap(emb_l, + output_dir = "", + max_size = 500000, + umap_metric = "euclidean", + cat_counts = None, + use_max_count = True): + + for k in range(0, len(emb_l)): + + E = emb_l[k].weight.detach().cpu().numpy() + print("umap", E.shape) + + # create histogram of norms + bins = 50 + norms = [np.linalg.norm(E[i], ord=2) for i in range(0,E.shape[0])] +# plt.hist(norms, bins = bins) +# plt.title("Cat norm hist var. "+str(k)) + hist, bins = np.histogram(norms, bins=bins) + logbins = np.logspace(np.log10(bins[0]),np.log10(bins[-1]),len(bins)) + + plt.figure(figsize=(8,8)) + plt.title("Categorical norms: " + str(k) + " cardinality " + str(len(cat_counts[k]))) + plt.hist(norms, bins=logbins) + plt.xscale("log") +# plt.legend() + plt.savefig(output_dir+"/cat-norm-histogram-"+str(k)+".png") + plt.close() + + if E.shape[0] < 20: + print("Skipping small embedding") + continue + + n_vis = min(max_size, E.shape[0]) + min_cnt = 0 + +# reducer = umap.UMAP(random_state=42, n_neighbors=25, min_dist=0.1) + reducer = umap.UMAP(random_state=42, metric=umap_metric) + + if use_max_count is False or n_vis == E.shape[0]: + Y = reducer.fit_transform(E[:n_vis,:]) + else: + + # select values with couns > 1 + done = False + min_cnt = 1 + while done == False: + el_cnt = (cat_counts[k] > min_cnt).sum() + if el_cnt <= max_size: + done = True + else: + min_cnt = min_cnt+1 + + E1= [] + for i in range(0, E.shape[0]): + if cat_counts[k][i] > min_cnt: + E1.append(E[i,:]) + + print("max_count_len", len(E1), "mincount", min_cnt) + Y = reducer.fit_transform(np.array(E1)) + + n_vis = len(E1) + + plt.figure(figsize=(8,8)) + + linewidth = 0 + size = 1 + + if Y.shape[0] < 2500: + linewidth = 1 + size = 5 + + if cat_counts is None: + plt.scatter(-Y[:,0], -Y[:,1], s=size, marker=".", linewidth=linewidth) + else: + #print(cat_counts[k]) + n_disp = min(len(cat_counts[k]), Y.shape[0]) + cur_max = math.log(max(cat_counts[k])) + norm_cat_count = [math.log(cat_counts[k][i]+1)/cur_max for i in range(0, len(cat_counts[k]))] + plt.scatter(-Y[0:n_disp,0], -Y[0:n_disp,1], s=size, marker=".", linewidth=linewidth, c=np.array(norm_cat_count)[0:n_disp], cmap="viridis") + plt.colorbar() + + plt.title("UMAP: categorical var. " + str(k) + " (" + str(n_vis) + " of " + str(E.shape[0]) + ", min count " + str(min_cnt) + ")") + plt.savefig(output_dir + "/cat-" + str(k) + "-" + str(n_vis) + "-of-" + str(E.shape[0]) + "-umap.png") + plt.close() + + +def visualize_embeddings_tsne(emb_l, + output_dir = "", + max_size = 10000): + + for k in range(0, len(emb_l)): + + E = emb_l[k].weight.detach().cpu() + print("tsne", E.shape) + + if E.shape[0] < 20: + print("Skipping small embedding") + continue + + n_vis = min(max_size, E.shape[0]) + + tsne = manifold.TSNE(init="pca", random_state=0, method="exact") + + Y = tsne.fit_transform(E[:n_vis,:]) + + plt.figure(figsize=(8, 8)) + + linewidth = 0 + if Y.shape[0] < 5000: + linewidth = 1 + + plt.scatter(-Y[:,0], -Y[:,1], s=1, marker=".", linewidth=linewidth) + + plt.title("TSNE: categorical var. " + str(k) + " (" + str(n_vis) + " of " + str(E.shape[0]) + ")") + plt.savefig(output_dir + "/cat-" + str(k) + "-" + str(n_vis) + "-of-" + str(E.shape[0]) + "-tsne.png") + plt.close() + + +def analyse_categorical_data(X_cat, n_days=10, output_dir=""): + + # analyse categorical variables + n_vec = len(X_cat) + n_cat = len(X_cat[0]) + n_days = n_days + + print("n_vec", n_vec, "n_cat", n_cat) +# for c in train_data.X_cat: +# print(n_cat, c) + + all_cat = np.array(X_cat) + print("all_cat.shape", all_cat.shape) + day_size = all_cat.shape[0]/n_days + + for i in range(0,n_cat): + l_d = [] + l_s1 = [] + l_s2 = [] + l_int = [] + l_rem = [] + + cat = all_cat[:,i] + print("cat", i, cat.shape) + for d in range(1,n_days): + offset = int(d*day_size) + #print(offset) + cat1 = cat[:offset] + cat2 = cat[offset:] + + s1 = set(cat1) + s2 = set(cat2) + + intersect = list(s1 & s2) + #print(intersect) + l_d.append(d) + l_s1.append(len(s1)) + l_s2.append(len(s2)) + l_int.append(len(intersect)) + l_rem.append((len(s1)-len(intersect))) + + print(d, ",", len(s1), ",", len(s2), ",", len(intersect), ",", (len(s1)-len(intersect))) + + print("spit", l_d) + print("before", l_s1) + print("after", l_s2) + print("inters.", l_int) + print("removed", l_rem) + + plt.figure(figsize=(8,8)) + plt.plot(l_d, l_s1, "g", label="before") + plt.plot(l_d, l_s2, "r", label="after") + plt.plot(l_d, l_int, "b", label="intersect") + plt.plot(l_d, l_rem, "y", label="removed") + plt.title("categorical var. "+str(i)) + plt.legend() + plt.savefig(output_dir+"/cat-"+str(i).zfill(3)+".png") + plt.close() + + +def analyse_categorical_counts(X_cat, emb_l=None, output_dir=""): + + # analyse categorical variables + n_vec = len(X_cat) + n_cat = len(X_cat[0]) + + print("n_vec", n_vec, "n_cat", n_cat) +# for c in train_data.X_cat: +# print(n_cat, c) + + all_cat = np.array(X_cat) + print("all_cat.shape", all_cat.shape) + + all_counts = [] + + for i in range(0,n_cat): + + cat = all_cat[:,i] + if emb_l is None: + s = set(cat) + counts = np.zeros((len(s))) + print("cat", i, cat.shape, len(s)) + else: + s = emb_l[i].weight.detach().cpu().shape[0] + counts = np.zeros((s)) + print("cat", i, cat.shape, s) + + for d in range(0,n_vec): + cv = int(cat[d]) + counts[cv] = counts[cv]+1 + + all_counts.append(counts) + + if emb_l is None: + plt.figure(figsize=(8,8)) + plt.plot(counts) + plt.title("Categorical var "+str(i) + " cardinality " + str(len(counts))) + # plt.legend() + else: + E = emb_l[i].weight.detach().cpu().numpy() + norms = [np.linalg.norm(E[i], ord=2) for i in range(0,E.shape[0])] + + fig, (ax0, ax1) = plt.subplots(2, 1) + fig.suptitle("Categorical variable: " + str(i)+" cardinality "+str(len(counts))) + + ax0.plot(counts) + ax0.set_yscale("log") + ax0.set_title("Counts", fontsize=10) + + ax1.plot(norms) + ax1.set_title("Norms", fontsize=10) + + plt.savefig(output_dir+"/cat_counts-"+str(i).zfill(3)+".png") + plt.close() + + return all_counts + + +def dlrm_output_wrap(dlrm, X, lS_o, lS_i, T): + + all_feat_vec = [] + all_cat_vec = [] + x_vec = None + t_out = None + c_out = None + z_out = [] + p_out = None + + z_size = len(dlrm.top_l) + + x = dlrm.apply_mlp(X, dlrm.bot_l) + # debug prints + #print("intermediate") + #print(x[0].detach().cpu().numpy()) + x_vec = x[0].detach().cpu().numpy() + all_feat_vec.append(x_vec) +# all_X.append(x[0].detach().cpu().numpy()) + + # process sparse features(using embeddings), resulting in a list of row vectors + ly = dlrm.apply_emb(lS_o, lS_i, dlrm.emb_l) + + for e in ly: + #print(e.detach().cpu().numpy()) + all_feat_vec.append(e[0].detach().cpu().numpy()) + all_cat_vec.append(e[0].detach().cpu().numpy()) + + all_feat_vec= np.concatenate(all_feat_vec, axis=0) + all_cat_vec= np.concatenate(all_cat_vec, axis=0) + +# all_features.append(all_feat_vec) +# all_cat.append(all_cat_vec) + t_out = int(T.detach().cpu().numpy()[0,0]) +# all_T.append(int(T.detach().cpu().numpy()[0,0])) + + z = dlrm.interact_features(x, ly) + # print(z.detach().cpu().numpy()) +# z_out = z.detach().cpu().numpy().flatten() + z_out.append(z.detach().cpu().numpy().flatten()) +# all_z[0].append(z.detach().cpu().numpy().flatten()) + + # obtain probability of a click (using top mlp) +# print(dlrm.top_l) +# p = dlrm.apply_mlp(z, dlrm.top_l) + + for i in range(0, z_size): + z = dlrm.top_l[i](z) + +# if i < z_size-1: +# curr_z = z.detach().cpu().numpy().flatten() + z_out.append(z.detach().cpu().numpy().flatten()) +# all_z[i+1].append(curr_z) +# print("z append", i) + +# print("z",i, z.detach().cpu().numpy().flatten().shape) + + p = z + + # clamp output if needed + if 0.0 < dlrm.loss_threshold and dlrm.loss_threshold < 1.0: + z = torch.clamp(p, min=dlrm.loss_threshold, max=(1.0 - dlrm.loss_threshold)) + else: + z = p + + class_thresh = 0.0 #-0.25 + zp = z.detach().cpu().numpy()[0,0]+ class_thresh + + p_out = int(zp+0.5) + if p_out > 1: + p_out = 1 + if p_out < 0: + p_out = 0 + +# all_pred.append(int(z.detach().cpu().numpy()[0,0]+0.5)) + + #print(int(z.detach().cpu().numpy()[0,0]+0.5)) + if int(p_out) == t_out: + c_out = 0 + else: + c_out = 1 + + return all_feat_vec, x_vec, all_cat_vec, t_out, c_out, z_out, p_out + + +def create_umap_data(dlrm, data_ld, max_size=50000, offset=0, info=""): + + all_features = [] + all_X = [] + all_cat = [] + all_T = [] + all_c = [] + all_z = [] + all_pred = [] + + z_size = len(dlrm.top_l) + print("z_size", z_size) + for i in range(0, z_size): + all_z.append([]) + + for j, (X, lS_o, lS_i, T) in enumerate(data_ld): + + if j < offset: + continue + + if j >= max_size+offset: + break + + af, x, cat, t, c, z, p = dlrm_output_wrap(dlrm, X, lS_o, lS_i, T) + + all_features.append(af) + all_X.append(x) + all_cat.append(cat) + all_T.append(t) + all_c.append(c) + all_pred.append(p) + + for i in range(0, z_size): + all_z[i].append(z[i]) + +# # calculate classifier metrics + ac = accuracy_score(all_T, all_pred) + f1 = f1_score(all_T, all_pred) + ps = precision_score(all_T, all_pred) + rc = recall_score(all_T, all_pred) + + print(info, "accuracy", ac, "f1", f1, "precision", ps, "recall", rc) + + return all_features, all_X, all_cat, all_T, all_z, all_c, all_pred + + +def plot_all_data_3(umap_Y, + umap_T, + train_Y = None, + train_T = None, + test_Y = None, + test_T = None, + total_train_size = "", + total_test_size = "", + info = "", + output_dir = "", + orig_space_dim = 0): + + size = 1 + colors = ["red","green"] + + fig, (ax0, ax1, ax2) = plt.subplots(1, 3) + fig.suptitle("UMAP: " + info + " space dim "+str(orig_space_dim)) + + ax0.scatter(umap_Y[:,0], umap_Y[:,1], s=size, c=umap_T, cmap=matplotlib.colors.ListedColormap(colors), marker=".", linewidth=0) + ax0.set_title("UMAP ("+str(len(umap_T))+" of "+ total_train_size+")", fontsize=7) + + if train_Y is not None and train_T is not None: + ax1.scatter(train_Y[:,0], train_Y[:,1], s=size, c=train_T, cmap=matplotlib.colors.ListedColormap(colors), marker=".", linewidth=0) + ax1.set_title("Train ("+str(len(train_T))+" of "+ total_train_size+")", fontsize=7) + + if test_Y is not None and test_T is not None: + ax2.scatter(test_Y[:,0], test_Y[:,1], s=size, c=test_T, cmap=matplotlib.colors.ListedColormap(colors), marker=".", linewidth=0) + ax2.set_title("Test ("+str(len(test_T))+" of "+ total_test_size+")", fontsize=7) + + plt.savefig(output_dir+"/"+info+"-umap.png") + plt.close() + + +def plot_one_class_3(umap_Y, + umap_T, + train_Y, + train_T, + test_Y, + test_T, + target = 0, + col = "red", + total_train_size = "", + total_test_size = "", + info = "", + output_dir = "", + orig_space_dim = 0): + + size = 1 + + fig, (ax0, ax1, ax2) = plt.subplots(1, 3) + fig.suptitle("UMAP: "+ info + " space dim "+str(orig_space_dim)) + + ind_l_umap = [i for i,x in enumerate(umap_T) if x == target] + Y_umap_l = np.array([umap_Y[i,:] for i in ind_l_umap]) + + ax0.scatter(Y_umap_l[:,0], Y_umap_l[:,1], s=size, c=col, marker=".", linewidth=0) + ax0.set_title("UMAP, ("+str(len(umap_T))+" of "+ total_train_size+")", fontsize=7) + + if train_Y is not None and train_T is not None: + ind_l_test = [i for i,x in enumerate(train_T) if x == target] + Y_test_l = np.array([train_Y[i,:] for i in ind_l_test]) + + ax1.scatter(Y_test_l[:,0], Y_test_l[:,1], s=size, c=col, marker=".", linewidth=0) + ax1.set_title("Train, ("+str(len(train_T))+" of "+ total_train_size+")", fontsize=7) + + if test_Y is not None and test_T is not None: + ind_l_test = [i for i,x in enumerate(test_T) if x == target] + Y_test_l = np.array([test_Y[i,:] for i in ind_l_test]) + + ax2.scatter(Y_test_l[:,0], Y_test_l[:,1], s=size, c=col, marker=".", linewidth=0) + ax2.set_title("Test, ("+str(len(test_T))+" of "+ total_test_size+")", fontsize=7) + + plt.savefig(output_dir+"/"+info+"-umap.png") + plt.close() + + +def visualize_umap_data(umap_Y, + umap_T, + umap_C, + umap_P, + train_Y, + train_T, + train_C, + train_P, + test_Y = None, + test_T = None, + test_C = None, + test_P = None, + total_train_size = "", + total_test_size = "", + info = "", + output_dir = "", + orig_space_dim = 0): + + # all classes + plot_all_data_3(umap_Y = umap_Y, + umap_T = umap_T, + train_Y = train_Y, + train_T = train_T, + test_Y = test_Y, + test_T = test_T, + total_train_size = total_train_size, + total_test_size = total_test_size, + info = info, + output_dir = output_dir, + orig_space_dim = orig_space_dim) + + # all predictions + plot_all_data_3(umap_Y = umap_Y, + umap_T = umap_P, + train_Y = train_Y, + train_T = train_P, + test_Y = test_Y, + test_T = test_P, + total_train_size = total_train_size, + total_test_size = total_test_size, + info = info+", all-predictions", + output_dir = output_dir, + orig_space_dim = orig_space_dim) + + + # class 0 + plot_one_class_3(umap_Y = umap_Y, + umap_T = umap_T, + train_Y = train_Y, + train_T = train_T, + test_Y = test_Y, + test_T = test_T, + target = 0, + col = "red", + total_train_size = total_train_size, + total_test_size = total_test_size, + info = info+" class " + str(0), + output_dir = output_dir, + orig_space_dim = orig_space_dim) + + # class 1 + plot_one_class_3(umap_Y = umap_Y, + umap_T = umap_T, + train_Y = train_Y, + train_T = train_T, + test_Y = test_Y, + test_T = test_T, + target = 1, + col = "green", + total_train_size = total_train_size, + total_test_size = total_test_size, + info = info + " class " + str(1), + output_dir = output_dir, + orig_space_dim = orig_space_dim) + + # correct classification + plot_one_class_3(umap_Y = umap_Y, + umap_T = umap_C, + train_Y = train_Y, + train_T = train_C, + test_Y = test_Y, + test_T = test_C, + target = 0, + col = "green", + total_train_size = total_train_size, + total_test_size = total_test_size, + info = info + " correct ", + output_dir = output_dir, + orig_space_dim = orig_space_dim) + + # errors + plot_one_class_3(umap_Y = umap_Y, + umap_T = umap_C, + train_Y = train_Y, + train_T = train_C, + test_Y = test_Y, + test_T = test_C, + target = 1, + col = "red", + total_train_size = total_train_size, + total_test_size = total_test_size, + info = info + " errors ", + output_dir = output_dir, + orig_space_dim = orig_space_dim) + + # prediction 0 + plot_one_class_3(umap_Y = umap_Y, + umap_T = umap_P, + train_Y = train_Y, + train_T = train_P, + test_Y = test_Y, + test_T = test_P, + target = 0, + col = "red", + total_train_size = total_train_size, + total_test_size = total_test_size, + info = info + " predict-0 ", + output_dir = output_dir, + orig_space_dim = orig_space_dim) + + # prediction 1 + plot_one_class_3(umap_Y = umap_Y, + umap_T = umap_P, + train_Y = train_Y, + train_T = train_P, + test_Y = test_Y, + test_T = test_P, + target = 1, + col = "green", + total_train_size = total_train_size, + total_test_size = total_test_size, + info = info + " predict-1 ", + output_dir = output_dir, + orig_space_dim = orig_space_dim) + +def hdbscan_clustering(umap_data, train_data, test_data, info="", output_dir=""): + + clusterer = hdbscan.HDBSCAN(min_samples=10, min_cluster_size=500, prediction_data=True) + umap_labels = clusterer.fit_predict(umap_data) + train_labels, _ = hdbscan.approximate_predict(clusterer, train_data) + test_labels, _ = hdbscan.approximate_predict(clusterer, test_data) + + fig, ((ax00, ax01, ax02), (ax10, ax11, ax12)) = plt.subplots(2, 3) + fig.suptitle("HDBSCAN clastering: "+ info ) + + # plot umap data + umap_clustered = (umap_labels >= 0) + umap_coll = collections.Counter(umap_clustered) + print("umap_clustered", umap_coll) +# print("umap_data", umap_data.shape) +# print("~umap_clustered", umap_clustered.count(False), ~umap_clustered) + ax00.scatter(umap_data[~umap_clustered, 0], + umap_data[~umap_clustered, 1], + c=(0.5, 0.5, 0.5), + s=0.1, + alpha=0.5) + ax00.set_title("UMAP Outliers " + str(umap_coll[False]), fontsize=7) + ax10.scatter(umap_data[umap_clustered, 0], + umap_data[umap_clustered, 1], + c=umap_labels[umap_clustered], + s=0.1, + cmap="Spectral") + ax10.set_title("UMAP Inliers " + str(umap_coll[True]), fontsize=7) + + # plot train data + train_clustered = (train_labels >= 0) + train_coll = collections.Counter(train_clustered) + ax01.scatter(train_data[~train_clustered, 0], + train_data[~train_clustered, 1], + c=(0.5, 0.5, 0.5), + s=0.1, + alpha=0.5) + ax01.set_title("Train Outliers " + str(train_coll[False]), fontsize=7) + ax11.scatter(train_data[train_clustered, 0], + train_data[train_clustered, 1], + c=train_labels[train_clustered], + s=0.1, + cmap="Spectral") + ax11.set_title("Train Inliers " + str(train_coll[True]), fontsize=7) + + # plot test data + test_clustered = (test_labels >= 0) + test_coll = collections.Counter(test_clustered) + ax02.scatter(test_data[~test_clustered, 0], + test_data[~test_clustered, 1], + c=(0.5, 0.5, 0.5), + s=0.1, + alpha=0.5) + ax02.set_title("Tets Outliers " + str(test_coll[False]), fontsize=7) + ax12.scatter(test_data[test_clustered, 0], + test_data[test_clustered, 1], + c=test_labels[test_clustered], + s=0.1, + cmap="Spectral") + ax12.set_title("Test Inliers " + str(test_coll[True]), fontsize=7) + + plt.savefig(output_dir+"/"+info+"-hdbscan.png") + plt.close() + + +def visualize_all_data_umap(dlrm, + train_ld, + test_ld = None, + max_umap_size = 50000, + output_dir = "", + umap_metric = "euclidean"): + + data_ratio = 1 + + print("creating umap data") + umap_train_feat, umap_train_X, umap_train_cat, umap_train_T, umap_train_z, umap_train_c, umap_train_p = create_umap_data(dlrm=dlrm, data_ld=train_ld, max_size=max_umap_size, offset=0, info="umap") + + # transform train and test data + train_feat, train_X, train_cat, train_T, train_z, train_c, train_p = create_umap_data(dlrm=dlrm, data_ld=train_ld, max_size=max_umap_size*data_ratio, offset=max_umap_size, info="train") + test_feat, test_X, test_cat, test_T, test_z, test_c, test_p = create_umap_data(dlrm=dlrm, data_ld=test_ld, max_size=max_umap_size*data_ratio, offset=0, info="test") + + print("umap_train_feat", np.array(umap_train_feat).shape) + reducer_all_feat = umap.UMAP(random_state=42, metric=umap_metric) + umap_feat_Y = reducer_all_feat.fit_transform(umap_train_feat) + + train_feat_Y = reducer_all_feat.transform(train_feat) + test_feat_Y = reducer_all_feat.transform(test_feat) + + visualize_umap_data(umap_Y = umap_feat_Y, + umap_T = umap_train_T, + umap_C = umap_train_c, + umap_P = umap_train_p, + train_Y = train_feat_Y, + train_T = train_T, + train_C = train_c, + train_P = train_p, + test_Y = test_feat_Y, + test_T = test_T, + test_C = test_c, + test_P = test_p, + total_train_size = str(len(train_ld)), + total_test_size = str(len(test_ld)), + info = "all-features", + output_dir = output_dir, + orig_space_dim = np.array(umap_train_feat).shape[1]) + + hdbscan_clustering(umap_data = umap_feat_Y, + train_data = train_feat_Y, + test_data = test_feat_Y, + info = "umap-all-features", + output_dir = output_dir) + +# hdbscan_clustering(umap_data = np.array(umap_train_feat), +# train_data = np.array(train_feat), +# test_data = np.array(test_feat), +# info = "all-features", +# output_dir = output_dir) + + print("umap_train_X", np.array(umap_train_X).shape) + reducer_X = umap.UMAP(random_state=42, metric=umap_metric) + umap_X_Y = reducer_X.fit_transform(umap_train_X) + + train_X_Y = reducer_X.transform(train_X) + test_X_Y = reducer_X.transform(test_X) + + visualize_umap_data(umap_Y = umap_X_Y, + umap_T = umap_train_T, + umap_C = umap_train_c, + umap_P = umap_train_p, + train_Y = train_X_Y, + train_T = train_T, + train_C = train_c, + train_P = train_p, + test_Y = test_X_Y, + test_T = test_T, + test_C = test_c, + test_P = test_p, + total_train_size = str(len(train_ld)), + total_test_size = str(len(test_ld)), + info = "cont-features", + output_dir = output_dir, + orig_space_dim = np.array(umap_train_X).shape[1]) + + print("umap_train_cat", np.array(umap_train_cat).shape) + reducer_cat = umap.UMAP(random_state=42, metric=umap_metric) + umap_cat_Y = reducer_cat.fit_transform(umap_train_cat) + + train_cat_Y = reducer_cat.transform(train_cat) + test_cat_Y = reducer_cat.transform(test_cat) + + visualize_umap_data(umap_Y = umap_cat_Y, + umap_T = umap_train_T, + umap_C = umap_train_c, + umap_P = umap_train_p, + train_Y = train_cat_Y, + train_T = train_T, + train_C = train_c, + train_P = train_p, + test_Y = test_cat_Y, + test_T = test_T, + test_C = test_c, + test_P = test_p, + total_train_size = str(len(train_ld)), + total_test_size = str(len(test_ld)), + info = "cat-features", + output_dir = output_dir, + orig_space_dim = np.array(umap_train_cat).shape[1]) + + # UMAP for z data + for i in range(0,len(umap_train_z)): + print("z", i, np.array(umap_train_z[i]).shape) + reducer_z = umap.UMAP(random_state=42, metric=umap_metric) + umap_z_Y = reducer_z.fit_transform(umap_train_z[i]) + + train_z_Y = reducer_z.transform(train_z[i]) + test_z_Y = reducer_z.transform(test_z[i]) + + visualize_umap_data(umap_Y = umap_z_Y, + umap_T = umap_train_T, + umap_C = umap_train_c, + umap_P = umap_train_p, + train_Y = train_z_Y, + train_T = train_T, + train_C = train_c, + train_P = train_p, + test_Y = test_z_Y, + test_T = test_T, + test_C = test_c, + test_P = test_p, + total_train_size = str(len(train_ld)), + total_test_size = str(len(test_ld)), + info = "z-features-"+str(i), + output_dir = output_dir, + orig_space_dim = np.array(umap_train_z[i]).shape[1]) + + +def analyze_model_data(output_dir, + dlrm, + train_ld, + test_ld, + train_data, + skip_embedding = False, + use_tsne = False, + max_umap_size = 50000, + max_tsne_size = 10000, + skip_categorical_analysis = False, + skip_data_plots = False, + umap_metric = "euclidean"): + + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + if skip_embedding is False: + + cat_counts = None + + cat_counts = analyse_categorical_counts(X_cat=train_data.X_cat, emb_l=dlrm.emb_l, output_dir=output_dir) + + visualize_embeddings_umap(emb_l = dlrm.emb_l, + output_dir = output_dir, + max_size = max_umap_size, + umap_metric = umap_metric, + cat_counts = cat_counts) + + if use_tsne is True: + visualize_embeddings_tsne(emb_l = dlrm.emb_l, + output_dir = output_dir, + max_size = max_tsne_size) + + # data visualization and analysis + if skip_data_plots is False: + visualize_all_data_umap(dlrm=dlrm, train_ld=train_ld, test_ld=test_ld, max_umap_size=max_umap_size, output_dir=output_dir, umap_metric=umap_metric) + + # analyse categorical variables + if skip_categorical_analysis is False and args.data_randomize == "none": + analyse_categorical_data(X_cat=train_data.X_cat, n_days=10, output_dir=output_dir) + + + +if __name__ == "__main__": + + output_dir = "" + + ### parse arguments ### + parser = argparse.ArgumentParser( + description="Exploratory DLRM analysis" + ) + + parser.add_argument("--load-model", type=str, default="") + parser.add_argument("--data-set", choices=["kaggle", "terabyte"], help="dataset") +# parser.add_argument("--dataset-path", required=True, help="path to the dataset") + parser.add_argument("--max-ind-range", type=int, default=-1) +# parser.add_argument("--mlperf-bin-loader", action="store_true", default=False) + parser.add_argument("--output-dir", type=str, default="") + parser.add_argument("--skip-embedding", action="store_true", default=False) + parser.add_argument("--umap-metric", type=str, default="euclidean") + parser.add_argument("--skip-data-plots", action="store_true", default=False) + parser.add_argument("--skip-categorical-analysis", action="store_true", default=False) + + # umap relatet + parser.add_argument("--max-umap-size", type=int, default=50000) + # tsne related + parser.add_argument("--use-tsne", action="store_true", default=False) + parser.add_argument("--max-tsne-size", type=int, default=1000) + # data file related + parser.add_argument("--raw-data-file", type=str, default="") + parser.add_argument("--processed-data-file", type=str, default="") + parser.add_argument("--data-sub-sample-rate", type=float, default=0.0) # in [0, 1] + parser.add_argument("--data-randomize", type=str, default="total") # none, total or day or none + parser.add_argument("--memory-map", action="store_true", default=False) + parser.add_argument("--mini-batch-size", type=int, default=1) + parser.add_argument("--num-workers", type=int, default=0) + parser.add_argument("--test-mini-batch-size", type=int, default=1) + parser.add_argument("--test-num-workers", type=int, default=0) + parser.add_argument("--num-batches", type=int, default=0) + # mlperf logging (disables other output and stops early) + parser.add_argument("--mlperf-logging", action="store_true", default=False) + + args = parser.parse_args() + + print("command line args: ", json.dumps(vars(args))) + + if output_dir == "": + output_dir = args.data_set+"-"+os.path.split(args.load_model)[-1]+"-vis_all" + print("output_dir:", output_dir) + + if args.data_set == "kaggle": + # 1. Criteo Kaggle Display Advertisement Challenge Dataset (see ./bench/dlrm_s_criteo_kaggle.sh) + m_spa=16 + ln_emb=np.array([1460,583,10131227,2202608,305,24,12517,633,3,93145,5683,8351593,3194,27,14992,5461306,10,5652,2173,4,7046547,18,15,286181,105,142572]) + ln_bot=np.array([13,512,256,64,16]) + ln_top=np.array([367,512,256,1]) + + elif args.dataset == "terabyte": + + if args.max_ind_range == 10000000: + # 2. Criteo Terabyte (see ./bench/dlrm_s_criteo_terabyte.sh [--sub-sample=0.875] --max-in-range=10000000) + m_spa=64 + ln_emb=np.array([9980333,36084,17217,7378,20134,3,7112,1442,61, 9758201,1333352,313829,10,2208,11156,122,4,970,14, 9994222, 7267859, 9946608,415421,12420,101, 36]) + ln_bot=np.array([13,512,256,64]) + ln_top=np.array([415,512,512,256,1]) + elif args.max_ind_range == 40000000: + # 3. Criteo Terabyte MLPerf training (see ./bench/run_and_time.sh --max-in-range=40000000) + m_spa=128 + ln_emb=np.array([39884406,39043,17289,7420,20263,3,7120,1543,63,38532951,2953546,403346,10,2208,11938,155,4,976,14,39979771,25641295,39664984,585935,12972,108,36]) + ln_bot=np.array([13,512,256,128]) + ln_top=np.array([479,1024,1024,512,256,1]) + else: + raise ValueError("only --max-in-range 10M or 40M is supported") + else: + raise ValueError("only kaggle|terabyte dataset options are supported") + + # check input parameters + if args.data_randomize != "none" and args.skip_categorical_analysis is not True: + print("Incorrect option for categoricat analysis, use: --data-randomize=none") + sys.exit(-1) + + dlrm = DLRM_Net( + m_spa, + ln_emb, + ln_bot, + ln_top, + arch_interaction_op="dot", + arch_interaction_itself=False, + sigmoid_bot=-1, + sigmoid_top=ln_top.size - 2, + sync_dense_params=True, + loss_threshold=0.0, + ndevices=-1, + qr_flag=False, + qr_operation=None, + qr_collisions=None, + qr_threshold=None, + md_flag=False, + md_threshold=None, + ) + + # Load model is specified + if not (args.load_model == ""): + print("Loading saved model {}".format(args.load_model)) + + ld_model = torch.load(args.load_model, map_location=torch.device("cpu")) + dlrm.load_state_dict(ld_model["state_dict"]) + + print("Model loaded", args.load_model) + #print(dlrm) + + z_size = len(dlrm.top_l) + for i in range(0, z_size): + print("z", i, dlrm.top_l[i]) + + # load data + train_data = None + test_data = None + + if args.raw_data_file is not "" or args.processed_data_file is not "": + train_data, train_ld, test_data, test_ld = dp.make_criteo_data_and_loaders(args) + + analyze_model_data(output_dir = output_dir, + dlrm = dlrm, + train_ld = train_ld, + test_ld = test_ld, + train_data = train_data, + skip_embedding = args.skip_embedding, + use_tsne = args.use_tsne, + max_umap_size = args.max_umap_size, + max_tsne_size = args.max_tsne_size, + skip_categorical_analysis = args.skip_categorical_analysis, + skip_data_plots = args.skip_data_plots, + umap_metric = args.umap_metric) + From f8bf6abc45d732db83093a380a2ec1fc93362ad7 Mon Sep 17 00:00:00 2001 From: mnaumovfb Date: Tue, 7 Jul 2020 02:04:04 -0700 Subject: [PATCH 28/57] Fixing saving of model protobuf with types and shapes in caffe2 version. --- dlrm_s_caffe2.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dlrm_s_caffe2.py b/dlrm_s_caffe2.py index 47b27d61..eb3e3638 100644 --- a/dlrm_s_caffe2.py +++ b/dlrm_s_caffe2.py @@ -79,6 +79,7 @@ # caffe2 from caffe2.proto import caffe2_pb2 from caffe2.python import brew, core, dyndep, model_helper, net_drawer, workspace +# from caffe2.python.predictor import mobile_exporter """ # auxiliary routine used to split input on the mini-bacth dimension @@ -607,6 +608,9 @@ def create_model(self, X, S_lengths, S_indices, T): tril_indices = np.array([j + i * num_fea for i in range(num_fea) for j in range(i + offset)]) self.FeedBlobWrapper(self.tint + "_tril_indices", tril_indices) + if self.save_onnx: + tish = tril_indices.shape + self.onnx_tsd[self.tint + "_tril_indices"] = (onnx.TensorProto.INT32, tish) # create compute graph if T is not None: From 29445216299e60484e0b74cf230d6b714b5240f0 Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Sun, 12 Jul 2020 01:46:45 -0700 Subject: [PATCH 29/57] modifications for FAIR cluster --- dlrm_data_pytorch.py | 6 ++--- dlrm_s_pytorch.py | 3 ++- extend_distributed.py | 60 ++++++++++++++++++++++++++++++++++++------- 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/dlrm_data_pytorch.py b/dlrm_data_pytorch.py index 6cbe382a..032f03cc 100644 --- a/dlrm_data_pytorch.py +++ b/dlrm_data_pytorch.py @@ -34,7 +34,7 @@ import torch from torch.utils.data import Dataset, RandomSampler -import data_loader_terabyte +## import data_loader_terabyte # Kaggle Display Advertising Challenge Dataset @@ -1007,7 +1007,7 @@ def read_dist_from_file(file_path): with open(file_path, "r") as f: lines = f.read().splitlines() except Exception: - print("Wrong file or file path") + print("Wrong file or file path in read: ", file_path) # read unique accesses unique_accesses = [int(el) for el in lines[0].split(", ")] # read cumulative distribution (elements are passed as two separate lists) @@ -1030,7 +1030,7 @@ def write_dist_to_file(file_path, unique_accesses, list_sd, cumm_sd): s = str(cumm_sd) f.write(s[1 : len(s) - 1] + "\n") except Exception: - print("Wrong file or file path") + print("Wrong file or file path in write: ", file_path) if __name__ == "__main__": diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 3aeeec0c..51f3d7f8 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -72,10 +72,11 @@ import warnings with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) -import onnx +## import onnx # pytorch import torch +from torch import onnx import torch.nn as nn from torch.nn.parallel.parallel_apply import parallel_apply from torch.nn.parallel.replicate import replicate diff --git a/extend_distributed.py b/extend_distributed.py index d7fd9dd1..152bde4a 100644 --- a/extend_distributed.py +++ b/extend_distributed.py @@ -39,6 +39,45 @@ def get_split_lengths(n): my_len = splits[my_rank] return (my_len, splits) +def get_world_rank_from_env(): + return env2int( + ["RANK", + "PMI_RANK", + "OMPI_COMM_WORLD_RANK", + "MV2_COMM_WORLD_RANK", + "SLURM_PROCID"], + -1 + ) + +def get_world_size_from_env(): + return env2int( + ["WORLD_SIZE", + "PMI_SIZE", + "OMPI_COMM_WORLD_SIZE", + "MV2_COMM_WORLD_SIZE", + "SLURM_NPROCS"], + -1 + ) + +def get_local_rank_from_env(): + return env2int( + ["MPI_LOCALRANKID", + "OMPI_COMM_WORLD_LOCAL_RANK", + "MV2_COMM_WORLD_LOCAL_RANK", + "SLURM_LOCALID", + ], + -1, + ) + +def get_local_size_from_env(): + return env2int( + ["MPI_LOCALNRANKS", + "OMPI_COMM_WORLD_LOCAL_SIZE", + "MV2_COMM_WORLD_LOCAL_SIZE", + ], + -1, + ) + def init_distributed(rank = -1, size = -1, backend=''): global myreq global my_rank @@ -62,25 +101,28 @@ def init_distributed(rank = -1, size = -1, backend=''): if backend != '': #guess Rank and size if rank == -1: - rank = env2int(['PMI_RANK', 'OMPI_COMM_WORLD_RANK', 'MV2_COMM_WORLD_RANK', 'RANK'], 0) + rank = get_world_rank_from_env() if size == -1: - size = env2int(['PMI_SIZE', 'OMPI_COMM_WORLD_SIZE', 'MV2_COMM_WORLD_SIZE', 'WORLD_SIZE'], 1) + size = get_world_size_from_env() if not os.environ.get('RANK', None) and rank != -1: os.environ['RANK'] = str(rank) if not os.environ.get('WORLD_SIZE', None) and size != -1: os.environ['WORLD_SIZE'] = str(size) if not os.environ.get('MASTER_PORT', None): os.environ['MASTER_PORT'] = '29500' if not os.environ.get('MASTER_ADDR', None): - local_size = env2int(['MPI_LOCALNRANKS', 'OMPI_COMM_WORLD_LOCAL_SIZE', 'MV2_COMM_WORLD_LOCAL_SIZE'], 1) - if local_size != size and backend != 'mpi': - print("Warning: Looks like distributed multinode run but MASTER_ADDR env not set, using '127.0.0.1' as default") - print("If this run hangs, try exporting rank 0's hostname as MASTER_ADDR") - os.environ['MASTER_ADDR'] = '127.0.0.1' + if "SLURM_NODELIST" in os.environ: + master_addr = os.environ["SLURM_NODELIST"].split('-')[0].replace("[", "") + elif "HOSTNAME" in os.environ: + # handle other cases ? + master_addr = os.environ["HOSTNAME"] + else: + master_addr = "127.0.0.1" + os.environ["MASTER_ADDR"] = master_addr if size > 1: dist.init_process_group(backend, rank=rank, world_size=size) my_rank = dist.get_rank() my_size = dist.get_world_size() - my_local_rank = env2int(['MPI_LOCALRANKID', 'OMPI_COMM_WORLD_LOCAL_RANK', 'MV2_COMM_WORLD_LOCAL_RANK'], 0) - my_local_size = env2int(['MPI_LOCALNRANKS', 'OMPI_COMM_WORLD_LOCAL_SIZE', 'MV2_COMM_WORLD_LOCAL_SIZE'], 1) + my_local_rank = get_local_rank_from_env() + my_local_size = get_local_size_from_env() if my_rank == 0: print("Running on %d ranks using %s backend" % (my_size, backend)) if hasattr(dist, 'all_to_all_single'): try: From e6009d4e2dbed459da13c0a9890f5411d462b543 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 19 Jul 2020 19:58:42 -0700 Subject: [PATCH 30/57] add projection --- dlrm_data_pytorch.py | 2 +- dlrm_s_pytorch.py | 105 +++++++++++++++++++++++++++++++++---------- 2 files changed, 82 insertions(+), 25 deletions(-) diff --git a/dlrm_data_pytorch.py b/dlrm_data_pytorch.py index 032f03cc..40f603ea 100644 --- a/dlrm_data_pytorch.py +++ b/dlrm_data_pytorch.py @@ -813,7 +813,7 @@ def generate_synthetic_input_batch( # sparse indices to be used per embedding file_path = trace_file line_accesses, list_sd, cumm_sd = read_dist_from_file( - file_path.replace("j", str(i)) + file_path.replace("j", str(0)) ) # debug prints # print("input") diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 51f3d7f8..2b30c69d 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -92,6 +92,8 @@ import sklearn.metrics +import uuid + # from torchviz import make_dot # import torch.nn.functional as Functional # from torch.nn.parameter import Parameter @@ -193,12 +195,36 @@ def create_emb(self, m, ln): np.random.set_state(np_rand_state) return emb_l + def create_proj(self, n, m): + # build MLP layer by layer + layers = nn.ModuleList() + # construct fully connected operator + LL = nn.Linear(int(n), int(m), bias=True) + + # initialize the weights + # with torch.no_grad(): + # custom Xavier input, output or two-sided fill + mean = 0.0 # std_dev = np.sqrt(variance) + std_dev = np.sqrt(2 / (m + n)) # np.sqrt(1 / m) # np.sqrt(1 / n) + W = np.random.normal(mean, std_dev, size=(m, n)).astype(np.float32) + std_dev = np.sqrt(1 / m) # np.sqrt(2 / (m + 1)) + bt = np.random.normal(mean, std_dev, size=m).astype(np.float32) + # approach 1 + LL.weight.data = torch.tensor(W, requires_grad=True) + LL.bias.data = torch.tensor(bt, requires_grad=True) + # approach 2: constant value ? + layers.append(LL) + + return torch.nn.Sequential(*layers) + + def __init__( self, m_spa=None, ln_emb=None, ln_bot=None, ln_top=None, + proj_size = 0, arch_interaction_op=None, arch_interaction_itself=False, sigmoid_bot=-1, @@ -224,6 +250,7 @@ def __init__( ): # save arguments + self.proj_size = proj_size self.ndevices = ndevices self.output_d = 0 self.parallel_model_batch_size = -1 @@ -260,6 +287,8 @@ def __init__( self.emb_l = self.create_emb(m_spa, ln_emb) self.bot_l = self.create_mlp(ln_bot, sigmoid_bot) self.top_l = self.create_mlp(ln_top, sigmoid_top) + if (proj_size > 0): + self.proj_l = self.create_proj(len(ln_emb)+1, proj_size) def apply_mlp(self, x, layers): # approach 1: use ModuleList @@ -269,6 +298,14 @@ def apply_mlp(self, x, layers): # approach 2: use Sequential container to wrap all layers return layers(x) + def apply_proj(self, x, layers): + # approach 1: use ModuleList + # for layer in layers: + # x = layer(x) + # return x + # approach 2: use Sequential container to wrap all layers + return layers(x) + def apply_emb(self, lS_o, lS_i, emb_l): # WARNING: notice that we are processing the batch at once. We implicitly # assume that the data is laid out such that: @@ -299,22 +336,31 @@ def interact_features(self, x, ly): (batch_size, d) = x.shape T = torch.cat([x] + ly, dim=1).view((batch_size, -1, d)) # perform a dot product - Z = torch.bmm(T, torch.transpose(T, 1, 2)) - # append dense feature with the interactions (into a row vector) - # approach 1: all - # Zflat = Z.view((batch_size, -1)) - # approach 2: unique - _, ni, nj = Z.shape - # approach 1: tril_indices - # offset = 0 if self.arch_interaction_itself else -1 - # li, lj = torch.tril_indices(ni, nj, offset=offset) - # approach 2: custom - offset = 1 if self.arch_interaction_itself else 0 - li = torch.tensor([i for i in range(ni) for j in range(i + offset)]) - lj = torch.tensor([j for i in range(nj) for j in range(i + offset)]) - Zflat = Z[:, li, lj] - # concatenate dense features and interactions - R = torch.cat([x] + [Zflat], dim=1) + if (self.proj_size > 0): + TT = torch.transpose(T, 1, 2) + TS = torch.reshape(TT, (-1, len(ly)+1)) + TC = self.apply_mlp(TS, self.proj_l) + TR = torch.reshape(TC, (-1, d ,self.proj_size)) + Z = torch.bmm(T, TR) + Zflat = Z.view((batch_size, -1)) + R = torch.cat([x] + [Zflat], dim=1) + else: + Z = torch.bmm(T, torch.transpose(T, 1, 2)) + # append dense feature with the interactions (into a row vector) + # approach 1: all + # Zflat = Z.view((batch_size, -1)) + # approach 2: unique + _, ni, nj = Z.shape + # approach 1: tril_indices + # offset = 0 if self.arch_interaction_itself else -1 + # li, lj = torch.tril_indices(ni, nj, offset=offset) + # approach 2: custom + offset = 1 if self.arch_interaction_itself else 0 + li = torch.tensor([i for i in range(ni) for j in range(i + offset)]) + lj = torch.tensor([j for i in range(nj) for j in range(i + offset)]) + Zflat = Z[:, li, lj] + # concatenate dense features and interactions + R = torch.cat([x] + [Zflat], dim=1) elif self.arch_interaction_op == "cat": # concatenation features (into a row vector) R = torch.cat([x] + ly, dim=1) @@ -548,6 +594,7 @@ def parallel_forward(self, dense_x, lS_o, lS_i): # model related parameters parser.add_argument("--arch-sparse-feature-size", type=int, default=2) parser.add_argument("--arch-embedding-size", type=str, default="4-3-2") + parser.add_argument("--arch-project-size", type=int, default=0) # j will be replaced with the table number parser.add_argument("--arch-mlp-bot", type=str, default="4-3-2") parser.add_argument("--arch-mlp-top", type=str, default="4-2-1") @@ -622,6 +669,7 @@ def parallel_forward(self, dense_x, lS_o, lS_i): parser.add_argument("--mlperf-auc-threshold", type=float, default=0.0) parser.add_argument("--mlperf-bin-loader", action='store_true', default=False) parser.add_argument("--mlperf-bin-shuffle", action='store_true', default=False) + args = parser.parse_args() ext_dist.init_distributed(backend=args.dist_backend) @@ -698,10 +746,13 @@ def parallel_forward(self, dense_x, lS_o, lS_i): # approach 1: all # num_int = num_fea * num_fea + m_den_out # approach 2: unique - if args.arch_interaction_itself: - num_int = (num_fea * (num_fea + 1)) // 2 + m_den_out + if (args.arch_project_size > 0): + num_int = num_fea * args.arch_project_size + m_den_out else: - num_int = (num_fea * (num_fea - 1)) // 2 + m_den_out + if args.arch_interaction_itself: + num_int = (num_fea * (num_fea + 1)) // 2 + m_den_out + else: + num_int = (num_fea * (num_fea - 1)) // 2 + m_den_out elif args.arch_interaction_op == "cat": num_int = num_fea * m_den_out else: @@ -825,6 +876,7 @@ def parallel_forward(self, dense_x, lS_o, lS_i): ln_emb, ln_bot, ln_top, + args.arch_project_size, arch_interaction_op=args.arch_interaction_op, arch_interaction_itself=args.arch_interaction_itself, sigmoid_bot=-1, @@ -999,7 +1051,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): ext_dist.barrier() print("time/loss/accuracy (if enabled):") - with torch.autograd.profiler.profile(args.enable_profiling, use_gpu) as prof: + with torch.autograd.profiler.profile(args.enable_profiling, use_gpu, record_shapes=True) as prof: while k < args.nepochs: if k < skip_upto_epoch: continue @@ -1304,10 +1356,15 @@ def loss_fn_wrap(Z, T, use_gpu, device): # profiling if args.enable_profiling: os.makedirs(args.out_dir, exist_ok=True) - with open("%s.prof" % file_prefix, "w") as prof_f: - prof_f.write(prof.key_averages().table(sort_by="cpu_time_total")) - prof.export_chrome_trace("./%s.json" % file_prefix) - # print(prof.key_averages().table(sort_by="cpu_time_total")) + with open("TT"+str(uuid.uuid4().hex), "w") as prof_f: + prof_f.write(prof.key_averages(group_by_input_shape=True).table( + sort_by="self_cpu_time_total" + )) + +# with open("%s.prof" % file_prefix, "w") as prof_f: +# prof_f.write(prof.key_averages().table(sort_by="cpu_time_total")) +# prof.export_chrome_trace("./%s.json" % file_prefix) +# # print(prof.key_averages().table(sort_by="cpu_time_total")) # plot compute graph if args.plot_compute_graph: From 3170d3503b5c3c515a5dbb6b118fd73a65e4f535 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 28 Jul 2020 02:08:39 -0700 Subject: [PATCH 31/57] change output file size --- dlrm_s_pytorch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 2b30c69d..a821570c 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -1358,7 +1358,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): os.makedirs(args.out_dir, exist_ok=True) with open("TT"+str(uuid.uuid4().hex), "w") as prof_f: prof_f.write(prof.key_averages(group_by_input_shape=True).table( - sort_by="self_cpu_time_total" + sort_by="self_cpu_time_total", row_limit=200 )) # with open("%s.prof" % file_prefix, "w") as prof_f: From 18996f69d5d70b4ffc7a229ee6aa91aebbee54ed Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Tue, 28 Jul 2020 06:15:33 -0700 Subject: [PATCH 32/57] test --- dlrm_s_pytorch.py | 13 +++++++++++++ extend_distributed.py | 35 ++++++++++++++++++++++++++++++----- 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index a821570c..0fcc9013 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -65,6 +65,7 @@ # numpy import numpy as np +import socket # onnx # The onnx import causes deprecation warnings every time workers @@ -672,8 +673,20 @@ def parallel_forward(self, dense_x, lS_o, lS_i): args = parser.parse_args() + print("=== Get Env ===") + print(socket.gethostname()) +# myenv = os.environ +# for e in myenv: +# print(e, "=", myenv[e]) +# print("=== Done ===") + ext_dist.init_distributed(backend=args.dist_backend) + print("success size= ", ext_dist.my_size, ext_dist.my_rank) + + ext_dist.barrier() + print("passed barrier") + if args.mlperf_logging: print('command line args: ', json.dumps(vars(args))) diff --git a/extend_distributed.py b/extend_distributed.py index 152bde4a..81239bbe 100644 --- a/extend_distributed.py +++ b/extend_distributed.py @@ -61,7 +61,8 @@ def get_world_size_from_env(): def get_local_rank_from_env(): return env2int( - ["MPI_LOCALRANKID", + ["LOCAL_RANK", + "MPI_LOCALRANKID", "OMPI_COMM_WORLD_LOCAL_RANK", "MV2_COMM_WORLD_LOCAL_RANK", "SLURM_LOCALID", @@ -71,7 +72,8 @@ def get_local_rank_from_env(): def get_local_size_from_env(): return env2int( - ["MPI_LOCALNRANKS", + ["LOCAL_SIZE", + "MPI_LOCALNRANKS", "OMPI_COMM_WORLD_LOCAL_SIZE", "MV2_COMM_WORLD_LOCAL_SIZE", ], @@ -123,11 +125,34 @@ def init_distributed(rank = -1, size = -1, backend=''): my_size = dist.get_world_size() my_local_rank = get_local_rank_from_env() my_local_size = get_local_size_from_env() - if my_rank == 0: print("Running on %d ranks using %s backend" % (my_size, backend)) + if my_local_size == -1: + if "SLURM_TASKS_PER_NODE" in os.environ: + size = os.environ["SLURM_TASKS_PER_NODE"].split("(")[0] + my_local_size = int(size) + + if my_rank >= 0: print("Running on %d ranks using %s backend" % (my_size, backend)) if hasattr(dist, 'all_to_all_single'): try: - dist.all_to_all_single(torch.empty([0]), torch.empty([0])) + a = torch.arange(my_size) + my_rank * my_size + b = torch.zeros(my_size).to(torch.int64) + c = torch.zeros(my_size).to(torch.int64) + for i in range(my_size): + c[i] = my_rank + i * my_size + + if (torch.cuda.is_available): + dev = torch.device('cuda', my_local_rank) + a = a.to(dev) + b = b.to(dev) + c = c.to(dev) + dist.all_to_all_single(b, a) + print("alltoall on rank :", my_rank, "a = ", a, " b = ", b) + else: + dist.all_to_all_single(b, a) + if torch.equal(b, c): alltoall_supported = True + print("All to all single test passed for rank ", my_rank) + else: + print("Failed alltoall single test! for rank= ", my_rank) except RuntimeError: pass if a2a_impl == 'alltoall' and alltoall_supported == False: @@ -416,7 +441,7 @@ def alltoall(inputs, per_rank_split_lengths): a2ai.S = sum(per_rank_split_lengths) if per_rank_split_lengths else a2ai.lS * my_size if a2a_impl == '' and alltoall_supported or a2a_impl == 'alltoall': - #print("Using All2All_Req") + print("Using All2All_Req") output = All2All_Req.apply(a2ai, *inputs) myreq.WaitFunction = All2All_Wait elif a2a_impl == '' or a2a_impl == 'scatter': From 99516992f9e30435cd4c247ba090d272cfd80e6f Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Tue, 28 Jul 2020 19:06:54 -0700 Subject: [PATCH 33/57] bug fix in projection --- dlrm_s_pytorch.py | 8 ++++---- extend_distributed.py | 25 +++++++++++++++++++------ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 0fcc9013..7244b889 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -339,7 +339,7 @@ def interact_features(self, x, ly): # perform a dot product if (self.proj_size > 0): TT = torch.transpose(T, 1, 2) - TS = torch.reshape(TT, (-1, len(ly)+1)) + TS = torch.reshape(TT, (-1, TT.size(2))) TC = self.apply_mlp(TS, self.proj_l) TR = torch.reshape(TC, (-1, d ,self.proj_size)) Z = torch.bmm(T, TR) @@ -678,7 +678,7 @@ def parallel_forward(self, dense_x, lS_o, lS_i): # myenv = os.environ # for e in myenv: # print(e, "=", myenv[e]) -# print("=== Done ===") + print("=== Done ===") ext_dist.init_distributed(backend=args.dist_backend) @@ -1371,13 +1371,13 @@ def loss_fn_wrap(Z, T, use_gpu, device): os.makedirs(args.out_dir, exist_ok=True) with open("TT"+str(uuid.uuid4().hex), "w") as prof_f: prof_f.write(prof.key_averages(group_by_input_shape=True).table( - sort_by="self_cpu_time_total", row_limit=200 + sort_by="self_cpu_time_total", )) # with open("%s.prof" % file_prefix, "w") as prof_f: # prof_f.write(prof.key_averages().table(sort_by="cpu_time_total")) # prof.export_chrome_trace("./%s.json" % file_prefix) -# # print(prof.key_averages().table(sort_by="cpu_time_total")) +# print(prof.key_averages().table(sort_by="cpu_time_total")) # plot compute graph if args.plot_compute_graph: diff --git a/extend_distributed.py b/extend_distributed.py index 81239bbe..7bc0ffbf 100644 --- a/extend_distributed.py +++ b/extend_distributed.py @@ -106,12 +106,15 @@ def init_distributed(rank = -1, size = -1, backend=''): rank = get_world_rank_from_env() if size == -1: size = get_world_size_from_env() + assert rank >= 0 + assert size > 0 + if not os.environ.get('RANK', None) and rank != -1: os.environ['RANK'] = str(rank) if not os.environ.get('WORLD_SIZE', None) and size != -1: os.environ['WORLD_SIZE'] = str(size) if not os.environ.get('MASTER_PORT', None): os.environ['MASTER_PORT'] = '29500' if not os.environ.get('MASTER_ADDR', None): if "SLURM_NODELIST" in os.environ: - master_addr = os.environ["SLURM_NODELIST"].split('-')[0].replace("[", "") + master_addr = os.environ["SLURM_NODELIST"].split(',')[0].replace("[", "") elif "HOSTNAME" in os.environ: # handle other cases ? master_addr = os.environ["HOSTNAME"] @@ -119,16 +122,26 @@ def init_distributed(rank = -1, size = -1, backend=''): master_addr = "127.0.0.1" os.environ["MASTER_ADDR"] = master_addr + myenv = os.environ + for e in myenv: + print(e, "=", myenv[e]) + print("=== Done ===") + if size > 1: - dist.init_process_group(backend, rank=rank, world_size=size) - my_rank = dist.get_rank() - my_size = dist.get_world_size() my_local_rank = get_local_rank_from_env() my_local_size = get_local_size_from_env() if my_local_size == -1: if "SLURM_TASKS_PER_NODE" in os.environ: - size = os.environ["SLURM_TASKS_PER_NODE"].split("(")[0] - my_local_size = int(size) + locsize = os.environ["SLURM_TASKS_PER_NODE"].split("(")[0] + my_local_size = int(locsize) + + assert(my_local_rank >= 0) + assert(my_local_size >= 0) + print("Check local rank ", my_local_rank, " size ", my_local_size) + + dist.init_process_group(backend, rank=rank, world_size=size) + my_rank = dist.get_rank() + my_size = dist.get_world_size() if my_rank >= 0: print("Running on %d ranks using %s backend" % (my_size, backend)) if hasattr(dist, 'all_to_all_single'): From cb4467405be4b583dd74acaa48fad378d0d1467e Mon Sep 17 00:00:00 2001 From: mneilly-et <55827703+mneilly-et@users.noreply.github.com> Date: Fri, 31 Jul 2020 20:49:29 -0700 Subject: [PATCH 34/57] Add validation checks to arguments using dash separated lists and check arch-interaction-op for valid choice (#113) --- dlrm_s_pytorch.py | 39 ++++++++++++++++++++++++++++++++++----- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 344c1679..9c778461 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -473,6 +473,30 @@ def parallel_forward(self, dense_x, lS_o, lS_i): return z0 +def dash_separated_ints(value): + vals = value.split('-') + for val in vals: + try: + int(val) + except ValueError: + raise argparse.ArgumentTypeError( + "%s is not a valid dash separated list of ints" % value) + + return value + + +def dash_separated_floats(value): + vals = value.split('-') + for val in vals: + try: + float(val) + except ValueError: + raise argparse.ArgumentTypeError( + "%s is not a valid dash separated list of floats" % value) + + return value + + if __name__ == "__main__": ### import packages ### import sys @@ -484,11 +508,15 @@ def parallel_forward(self, dense_x, lS_o, lS_i): ) # model related parameters parser.add_argument("--arch-sparse-feature-size", type=int, default=2) - parser.add_argument("--arch-embedding-size", type=str, default="4-3-2") + parser.add_argument( + "--arch-embedding-size", type=dash_separated_ints, default="4-3-2") # j will be replaced with the table number - parser.add_argument("--arch-mlp-bot", type=str, default="4-3-2") - parser.add_argument("--arch-mlp-top", type=str, default="4-2-1") - parser.add_argument("--arch-interaction-op", type=str, default="dot") + parser.add_argument( + "--arch-mlp-bot", type=dash_separated_ints, default="4-3-2") + parser.add_argument( + "--arch-mlp-top", type=dash_separated_ints, default="4-2-1") + parser.add_argument( + "--arch-interaction-op", type=str, choices=['dot', 'cat'], default="dot") parser.add_argument("--arch-interaction-itself", action="store_true", default=False) # embedding table options parser.add_argument("--md-flag", action="store_true", default=False) @@ -502,7 +530,8 @@ def parallel_forward(self, dense_x, lS_o, lS_i): # activations and loss parser.add_argument("--activation-function", type=str, default="relu") parser.add_argument("--loss-function", type=str, default="mse") # or bce or wbce - parser.add_argument("--loss-weights", type=str, default="1.0-1.0") # for wbce + parser.add_argument( + "--loss-weights", type=dash_separated_floats, default="1.0-1.0") # for wbce parser.add_argument("--loss-threshold", type=float, default=0.0) # 1.0e-7 parser.add_argument("--round-targets", type=bool, default=False) # data From 3a9b5cf3e4fee4c90447cbc01603a957f5285bfd Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Thu, 6 Aug 2020 04:29:51 -0700 Subject: [PATCH 35/57] add gaussian distribution --- dlrm_data_pytorch.py | 84 ++++++++++++++++++++++++++++++++++++++++++-- dlrm_s_pytorch.py | 8 +++++ 2 files changed, 90 insertions(+), 2 deletions(-) diff --git a/dlrm_data_pytorch.py b/dlrm_data_pytorch.py index 40f603ea..1d9845ba 100644 --- a/dlrm_data_pytorch.py +++ b/dlrm_data_pytorch.py @@ -537,6 +537,13 @@ def __init__( trace_file="", enable_padding=False, reset_seed_on_access=False, + + rand_data_dist="uniform", + rand_data_min=1, + rand_data_max=1, + rand_data_mu=-1, + rand_data_sigma=1, + rand_seed=0 ): # compute batch size @@ -561,6 +568,12 @@ def __init__( self.enable_padding = enable_padding self.reset_seed_on_access = reset_seed_on_access self.rand_seed = rand_seed + + self.rand_data_dist = rand_data_dist + self.rand_data_min = rand_data_min + self.rand_data_max = rand_data_max + self.rand_data_mu = rand_data_mu + self.rand_data_sigma = rand_data_sigma def reset_numpy_seed(self, numpy_rand_seed): np.random.seed(numpy_rand_seed) @@ -585,12 +598,18 @@ def __getitem__(self, index): # generate a batch of dense and sparse features if self.data_generation == "random": - (X, lS_o, lS_i) = generate_uniform_input_batch( + (X, lS_o, lS_i) = generate_dist_input_batch( self.m_den, self.ln_emb, n, self.num_indices_per_lookup, - self.num_indices_per_lookup_fixed + self.num_indices_per_lookup_fixed, + + rand_data_dist=self.rand_data_dist, + rand_data_min=self.rand_data_min, + rand_data_max=self.rand_data_max, + rand_data_mu=self.rand_data_mu, + rand_data_sigma=self.rand_data_sigma, ) elif self.data_generation == "synthetic": (X, lS_o, lS_i) = generate_synthetic_input_batch( @@ -778,6 +797,67 @@ def generate_uniform_input_batch( return (Xt, lS_emb_offsets, lS_emb_indices) +# random data from uniform or gaussian ditribution (input data) +def generate_dist_input_batch( + m_den, + ln_emb, + n, + num_indices_per_lookup, + num_indices_per_lookup_fixed, + rand_data_dist, + rand_data_min, + rand_data_max, + rand_data_mu, + rand_data_sigma, +): + # dense feature + Xt = torch.tensor(ra.rand(n, m_den).astype(np.float32)) + + # sparse feature (sparse indices) + lS_emb_offsets = [] + lS_emb_indices = [] + # for each embedding generate a list of n lookups, + # where each lookup is composed of multiple sparse indices + for size in ln_emb: + lS_batch_offsets = [] + lS_batch_indices = [] + offset = 0 + for _ in range(n): + # num of sparse indices to be used per embedding (between + if num_indices_per_lookup_fixed: + sparse_group_size = np.int64(num_indices_per_lookup) + else: + # random between [1,num_indices_per_lookup]) + r = ra.random(1) + sparse_group_size = np.int64( + np.round(max([1.0], r * min(size, num_indices_per_lookup))) + ) + # sparse indices to be used per embedding + if rand_data_dist == "gaussian": + if rand_data_mu == -1: + rand_data_mu = (rand_data_max + rand_data_min) / 2.0 + r = ra.normal(rand_data_mu, rand_data_sigma, sparse_group_size) + sparse_group = np.clip(r, rand_data_min, rand_data_max) + sparse_group = np.unique(sparse_group).astype(np.int64) + elif rand_data_dist == "uniform": + r = ra.random(sparse_group_size) + sparse_group = np.unique(np.round(r * (size - 1)).astype(np.int64)) + else: + raise(rand_data_dist, "distribution is not supported. \ + please select uniform or gaussian") + + # reset sparse_group_size in case some index duplicates were removed + sparse_group_size = np.int64(sparse_group.size) + # store lengths and indices + lS_batch_offsets += [offset] + lS_batch_indices += sparse_group.tolist() + # update offset for next iteration + offset += sparse_group_size + lS_emb_offsets.append(torch.tensor(lS_batch_offsets)) + lS_emb_indices.append(torch.tensor(lS_batch_indices)) + + return (Xt, lS_emb_offsets, lS_emb_indices) + # synthetic distribution (input data) def generate_synthetic_input_batch( m_den, diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 7244b889..da79c720 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -622,6 +622,14 @@ def parallel_forward(self, dense_x, lS_o, lS_i): parser.add_argument( "--data-generation", type=str, default="random" ) # synthetic or dataset + + # add Gaussian distribution + parser.add_argument("--rand-data-dist", type=str, default="uniform") # uniform or gaussian + parser.add_argument("--rand-data-min", type=float, default=0) + parser.add_argument("--rand-data-max", type=float, default=1) + parser.add_argument("--rand-data-mu", type=float, default=-1) + parser.add_argument("--rand-data-sigma", type=float, default=1) + parser.add_argument("--data-trace-file", type=str, default="./input/dist_emb_j.log") parser.add_argument("--data-set", type=str, default="kaggle") # or terabyte parser.add_argument("--raw-data-file", type=str, default="") From 53bf84b5f404d0fbe9d26bd024934b7d75e67a6e Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Thu, 6 Aug 2020 11:43:59 -0700 Subject: [PATCH 36/57] add synthetic data --- dlrm_s_pytorch.py | 11 + fb_emb_trace_writer.py | 264 +++++++++++++++++++++ fb_synthetic_data_pytorch.py | 439 +++++++++++++++++++++++++++++++++++ 3 files changed, 714 insertions(+) create mode 100644 fb_emb_trace_writer.py create mode 100644 fb_synthetic_data_pytorch.py diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index da79c720..f3668eea 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -95,6 +95,8 @@ import uuid +import fb_synthetic_data_pytorch as fb_syn_data + # from torchviz import make_dot # import torch.nn.functional as Functional # from torch.nn.parameter import Parameter @@ -752,6 +754,15 @@ def parallel_forward(self, dense_x, lS_o, lS_i): ))) m_den = train_data.m_den ln_bot[0] = m_den + + elif args.data_generation == "fb_synthetic": + # input and target at random + ln_emb = np.fromstring(args.arch_embedding_size, dtype=int, sep="-") + m_den = ln_bot[0] + train_data, train_ld = fb_syn_data.make_random_data_and_loader(args, ln_emb, m_den) + nbatches = args.num_batches if args.num_batches > 0 else len(train_ld) + table_feature_map = None # {idx : idx for idx in range(len(ln_emb))} + else: # input and target at random ln_emb = np.fromstring(args.arch_embedding_size, dtype=int, sep="-") diff --git a/fb_emb_trace_writer.py b/fb_emb_trace_writer.py new file mode 100644 index 00000000..538e1cd0 --- /dev/null +++ b/fb_emb_trace_writer.py @@ -0,0 +1,264 @@ +# Description: using torch internal koski reader to write two traces +# for each embedding table. The first one is trace_{i}.log which represents +# indices of each sample for embedding table i. The second one is size_trace_{i}.log +# which stores the number of indices for each sample for embedding table i. +# These traces would be used in fb synthetic data generation which imitate fb data +# for benchmarking DLRM in external infrastructures. + + +from __future__ import absolute_import, division, print_function, unicode_literals + +import argparse +import builtins +import argparse +import os + +# data generation +import dlrm_data_pytorch as dp + +# numpy +import numpy as np + +# pytorch +import torch + +# The following import is needed only for runs that use production data +# For those runs, the proper dependencies should be set up before the run. +try: + import fb_data_pytorch as fbdata +except ImportError: + print("Production libs are not set up.") + print("Note: Not all runs need production libs.") + pass + +exc = getattr(builtins, "IOError", "FileNotFoundError") + + +if __name__ == "__main__": + ### parse arguments ### + parser = argparse.ArgumentParser( + description="Train Deep Learning Recommendation Model (DLRM)" + ) + # model related parameters + parser.add_argument("--arch-sparse-feature-size", type=int, default=2) + parser.add_argument("--arch-embedding-size", type=str, default="4-3-2") + # j will be replaced with the table number + parser.add_argument("--arch-mlp-bot", type=str, default="4-3-2") + parser.add_argument("--arch-mlp-top", type=str, default="4-2-1") + parser.add_argument("--arch-interaction-op", type=str, default="dot") + parser.add_argument("--arch-interaction-itself", action="store_true", default=False) + parser.add_argument("--weighted-pooling", type=str, default=None) + # embedding table options + parser.add_argument("--md-flag", action="store_true", default=False) + parser.add_argument("--md-threshold", type=int, default=200) + parser.add_argument("--md-temperature", type=float, default=0.3) + parser.add_argument("--md-round-dims", action="store_true", default=False) + parser.add_argument("--qr-flag", action="store_true", default=False) + parser.add_argument("--qr-threshold", type=int, default=200) + parser.add_argument("--qr-operation", type=str, default="mult") + parser.add_argument("--qr-collisions", type=int, default=4) + parser.add_argument("--hype-flag", action="store_true", default=False) + parser.add_argument("--cluster-fb-preproc", action="store_true", default=False) + # activations and loss + parser.add_argument("--activation-function", type=str, default="relu") + parser.add_argument("--loss-function", type=str, default="mse") # or bce or wbce + parser.add_argument("--loss-weights", type=str, default="1.0-1.0") # for wbce + parser.add_argument("--loss-threshold", type=float, default=0.0) # 1.0e-7 + parser.add_argument("--round-targets", type=bool, default=False) + # data + parser.add_argument("--data-size", type=int, default=1) + parser.add_argument("--num-batches", type=int, default=0) + parser.add_argument( + "--data-generation", type=str, default="random" + ) # synthetic or dataset + parser.add_argument("--data-trace-file", type=str, default="./input/dist_emb_j.log") + parser.add_argument("--data-set", type=str, default="kaggle") # or terabyte + parser.add_argument("--raw-data-file", type=str, default="") + parser.add_argument("--processed-data-file", type=str, default="") + parser.add_argument("--data-randomize", type=str, default="total") # or day or none + parser.add_argument("--data-trace-enable-padding", type=bool, default=False) + parser.add_argument("--max-ind-range", type=int, default=-1) + parser.add_argument("--data-sub-sample-rate", type=float, default=0.0) # in [0, 1] + parser.add_argument("--num-indices-per-lookup", type=int, default=10) + parser.add_argument("--num-indices-per-lookup-fixed", type=bool, default=False) + parser.add_argument("--num-workers", type=int, default=0) + parser.add_argument("--memory-map", action="store_true", default=False) + # training + parser.add_argument("--mini-batch-size", type=int, default=1) + parser.add_argument("--nepochs", type=int, default=1) + parser.add_argument("--learning-rate", type=float, default=0.01) + parser.add_argument("--print-precision", type=int, default=5) + parser.add_argument("--numpy-rand-seed", type=int, default=123) + parser.add_argument("--sync-dense-params", type=bool, default=True) + parser.add_argument("--optimizer", type=str, default="sgd") + # inference + parser.add_argument("--inference-only", action="store_true", default=False) + # quantize + parser.add_argument("--quantize-with-bit", type=int, default=32) + # onnx + parser.add_argument("--save-onnx", action="store_true", default=False) + # gpu + parser.add_argument("--use-gpu", action="store_true", default=False) + # debugging and profiling + parser.add_argument("--print-freq", type=int, default=1) + parser.add_argument("--test-freq", type=int, default=-1) + parser.add_argument("--test-mini-batch-size", type=int, default=-1) + parser.add_argument("--test-num-workers", type=int, default=-1) + parser.add_argument("--fb-test-sample-limit", type=int, default=5000) + parser.add_argument("--fb-train-sample-limit", type=int, default=500000) + parser.add_argument("--fb-test-ds", type=str, default="") + parser.add_argument("--fb-train-ds", type=str, default="") + parser.add_argument("--fb-stream-train-samples", action="store_true", default=False) + parser.add_argument("--fb-stream-test-samples", action="store_true", default=False) + parser.add_argument("--fb-stream-prefetch-size", type=int, default=128) + parser.add_argument("--fb-preprocess", action="store_true", default=False) + parser.add_argument("--fb-report-ne", action="store_true", default=False) + parser.add_argument("--fb-window-size", type=int, default=-1) + parser.add_argument("--fb-loss-bias", type=float, default=1.0) # shifted BCE loss c + parser.add_argument("--fb-run-id", type=int, default=-1) + parser.add_argument("--print-time", action="store_true", default=False) + parser.add_argument("--debug-mode", action="store_true", default=False) + parser.add_argument("--enable-profiling", action="store_true", default=False) + parser.add_argument("--plot-compute-graph", action="store_true", default=False) + parser.add_argument("--tensor-board-filename", type=str, default="run_kaggle_pt") + # store/load model + parser.add_argument("--save-model", type=str, default="") + parser.add_argument("--load-model", type=str, default="") + # mlperf logging (disables other output and stops early) + parser.add_argument("--mlperf-logging", action="store_true", default=False) + # stop at target accuracy Kaggle 0.789, Terabyte (sub-sampled=0.875) 0.8107 + parser.add_argument("--mlperf-acc-threshold", type=float, default=0.0) + # stop at target AUC Terabyte (no subsampling) 0.8025 + parser.add_argument("--mlperf-auc-threshold", type=float, default=0.0) + parser.add_argument("--trace-folder", type=str, default="fb_traces/") + args = parser.parse_args() + print(args) + + if (args.test_mini_batch_size < 0): + # if the parameter is not set, use the training batch size + args.test_mini_batch_size = args.mini_batch_size + if (args.test_num_workers < 0): + # if the parameter is not set, use the same parameter for training + args.test_num_workers = args.num_workers + + ### prepare training data ### + ln_bot = np.fromstring(args.arch_mlp_bot, dtype=int, sep="-") + # input data + if args.data_generation == "dataset": + + if (args.data_set == "fb"): + train_data, train_ld, test_data, test_ld, table_feature_map = \ + fbdata.make_fb_data_and_loaders(args) + nbatches = args.num_batches if args.num_batches > 0 \ + else train_data.num_batches + # TODO: Handle the cases where fb_train_sample_limit is set to more + # than the number of samples in the table. + nbatches_test = test_data.num_batches + else: # criteo kaggle or terabyte + train_data, train_ld, test_data, test_ld = \ + dp.make_criteo_data_and_loaders(args) + nbatches = args.num_batches if args.num_batches > 0 else len(train_ld) + nbatches_test = len(test_ld) + + ln_emb = train_data.counts + # enforce maximum limit on number of vectors per embedding + if args.max_ind_range > 0: + ln_emb = np.array(list(map( + lambda x: x if x < args.max_ind_range else args.max_ind_range, + ln_emb + ))) + else: + ln_emb = np.array(ln_emb) + + m_den = train_data.m_den + ln_bot[0] = m_den + + else: + # input and target at random + ln_emb = np.fromstring(args.arch_embedding_size, dtype=int, sep="-") + m_den = ln_bot[0] + train_data, train_ld = dp.make_random_data_and_loader(args, ln_emb, m_den) + nbatches = args.num_batches if args.num_batches > 0 else len(train_ld) + + # For FB data, each batch has a label tensor packed, while the others don't. + # The following function is a wrapper to avoid checking this multiple times in th + # loop below. + def unpack_batch(b): + if args.data_generation == "dataset" and args.data_set == "fb": + # Experiment with weighted samples + return b[0], b[1], b[2], b[3], b[4] + else: + # Experiment with unweighted samples + return b[0], b[1], b[2], b[3], torch.ones(b[3].size()) + + ### parse command line arguments ### + num_fea = ln_emb.size + 1 # num sparse + num dense features + + print( + "mlp bot arch " + + str(ln_bot.size - 1) + + " layers, with input to output dimensions:" + ) + print(ln_bot) + print("# of features (sparse and dense)") + print(num_fea) + print("dense feature size") + print(m_den) + print("sparse feature size") + print( + "# of embeddings (= # of sparse features) " + + str(ln_emb.size) + + ", with dimensions " + + "x:" + ) + print(ln_emb) + + f_traces = [-1 for i in range(ln_emb.size)] + f_size_trace = [-1 for i in range(ln_emb.size)] + if not os.path.exists(args.trace_folder): + os.mkdir(args.trace_folder) + + for i in range(ln_emb.size): + f_traces[i] = open(f'{args.trace_folder}/trace_{i}.log', 'w') + f_size_trace[i] = open(f'{args.trace_folder}/size_trace_{i}.log', 'w') + + def f(x): + return x.item() + + for j, inputBatch in enumerate(train_ld): + X, lS_o, lS_i, T, W = unpack_batch(inputBatch) + + # sparse index traces + for table_ind, S_i in enumerate(lS_i): + + for item_ind, item in enumerate(S_i): + elm = f"{item}, " + if j == nbatches - 1 and item_ind == len(S_i) - 1: # the final last item + elm = f"{item}" + + f_traces[table_ind].write(elm) + + # size traces + for table_ind, S_o in enumerate(lS_o): + + s_i = lS_i[table_ind] + for ind, _item in enumerate(S_o): + if ind == len(S_o) - 1: # the last item in this batch + size_trace_elm = f"{len(s_i) - f(S_o[-1])}, " + if j == nbatches - 1 : # the final last item + size_trace_elm = f"{len(s_i) - f(S_o[-1])}" + else: + itm = f(S_o[ind + 1]) - f(S_o[ind]) + size_trace_elm = str(itm) + ', ' + + f_size_trace[table_ind].write(size_trace_elm) + + if (j + 1) % args.print_freq == 0: + print(f"mini-batch: {j + 1}") + + # closing trace files + for i in range(ln_emb.size): + f_traces[i].close() + f_size_trace[i].close() + + print(f"Trace and size_trace files are in the folder {args.trace_folder}") diff --git a/fb_synthetic_data_pytorch.py b/fb_synthetic_data_pytorch.py new file mode 100644 index 00000000..367f1d1c --- /dev/null +++ b/fb_synthetic_data_pytorch.py @@ -0,0 +1,439 @@ +#!/usr/bin/env python3 + +# Description: Generating synthetic data from the same distribution of the fb data. + +from __future__ import absolute_import, division, print_function, unicode_literals + +import torch +import sys +import numpy as np +import os +import sys +import operator +from numpy import random as ra +from collections import deque +import collections +import bisect + +# WARNING: global define, must be consistent across all synthetic functions +cache_line_size = 1 + +from torch.utils.data import Dataset + + +def collate_wrapper_random(list_of_tuples): + # where each tuple is (X, lS_o, lS_i, T) + (X, lS_o, lS_i, T) = list_of_tuples[0] + return (X, + torch.stack(lS_o), + lS_i, + T) + + +# auxiliary read routines +def read_trace_from_file(file_path, ignore_error=True, trace_file_binary_type=False): + try: + with open(file_path) as f: + if trace_file_binary_type: + array = np.fromfile(f, dtype=np.uint64) + trace = array.astype(np.uint64).tolist() + else: + line = f.readline() + if line == '': + trace = [] + else : + trace = list(map(lambda x: np.uint64(x), line.split(", "))) + return trace + except Exception as e: + if ignore_error: + print(f"Can not read '{file_path}' . {e}.") + else: + raise (e) + + +# auxiliary write routines +def write_trace_to_file(file_path, trace, trace_file_binary_type=False): + try: + if trace_file_binary_type: + with open(file_path, "wb+") as f: + np.array(trace).astype(np.uint64).tofile(f) + else: + with open(file_path, "w+") as f: + s = str(trace) + f.write(s[1 : len(s) - 1]) + except Exception as e: + print(e, "\n Unable to write trace file ", file_path) + + +def trace_profile(trace, enable_padding=False): + rstack = deque() # S + stack_distances = deque() # SDS + line_accesses = deque() # L + for x in trace: + r = np.uint64(x / cache_line_size) + l = len(rstack) + try: # found # + i = rstack.index(r) + # WARNING: I believe below is the correct depth in terms of meaning of the + # algorithm, but that is not what seems to be in the paper alg. + # -1 can be subtracted if we defined the distance between + # consecutive accesses (e.g. r, r) as 0 rather than 1. + sd = l - i # - 1 + # push r to the end of stack_distances + stack_distances.appendleft(sd) + # remove r from its position and insert to the top of stack + del rstack[i] # rstack.remove(r) + rstack.append(r) + except ValueError: # not found # + sd = 0 # -1 + # push r to the end of stack_distances/line_accesses + stack_distances.appendleft(sd) + line_accesses.appendleft(r) + # push r to the top of stack + rstack.append(r) + + if enable_padding: + # WARNING: notice that as the ratio between the number of samples (l) + # and cardinality (c) of a sample increases the probability of + # generating a sample gets smaller and smaller because there are + # few new samples compared to repeated samples. This means that for a + # long trace with relatively small cardinality it will take longer to + # generate all new samples and therefore obtain full distribution support + # and hence it takes longer for distribution to resemble the original. + # Therefore, we may pad the number of new samples to be on par with + # average number of samples l/c artificially. + l = len(stack_distances) + c = max(stack_distances) + padding = int(np.ceil(l / c)) + stack_distances = stack_distances + [0] * padding + + return (rstack, stack_distances, line_accesses) + + +def trace_generate_lru( + line_accesses, list_sd, cumm_sd, out_trace_len, enable_padding=False +): + max_sd = list_sd[-1] + l = len(line_accesses) + i = 0 + ztrace = deque() + for _ in range(out_trace_len): + sd = generate_stack_distance(list_sd, cumm_sd, max_sd, i, enable_padding) + mem_ref_within_line = 0 # floor(ra.rand(1)*cache_line_size) #0 + + # generate memory reference + if sd == 0: # new reference # + line_ref = line_accesses[0] + del line_accesses[0] + line_accesses.append(line_ref) + mem_ref = np.uint64(line_ref * cache_line_size + mem_ref_within_line) + i += 1 + else: # existing reference # + line_ref = line_accesses[l - sd] + mem_ref = np.uint64(line_ref * cache_line_size + mem_ref_within_line) + del line_accesses[l - sd] + line_accesses.append(line_ref) + # save generated memory reference + ztrace.append(mem_ref) + + return ztrace + + +def syn_trace_from_trace( + trace_file, + syn_trace_len, + prep_folder="stack_dists_line_aces/", + trace_file_binary_type=False, + trace_enable_padding=False, + numpy_rand_seed=123, + print_precision=5 +): + + ### some basic setup ### + if numpy_rand_seed != -1: + np.random.seed(numpy_rand_seed) + np.set_printoptions(precision=print_precision) + uni_name = '_'.join(trace_file[:-4].split('/')) + + ### profile trace ### + dist_file = prep_folder + "dist_" + uni_name + ".npy" + if not os.path.exists(dist_file): + if not os.path.exists(prep_folder): + os.mkdir(prep_folder) + + ### read trace ### + trace = read_trace_from_file(trace_file) + + if trace == []: + return [] + + (_, stack_distances, line_accesses) = trace_profile( + trace, trace_enable_padding + ) + stack_distances.reverse() + line_accesses.reverse() + + ### compute probability distribution ### + # count items + l = len(stack_distances) + dc = sorted( + collections.Counter(stack_distances).items(), key=operator.itemgetter(0) + ) + + # create a distribution + list_sd = list(map(lambda tuple_x_k: tuple_x_k[0], dc)) # x = tuple_x_k[0] + # dist_sd = list( + # map(lambda tuple_x_k: tuple_x_k[1] / float(l), dc) + # ) # k = tuple_x_k[1] + cumm_sd = deque() # np.cumsum(dc).tolist() #prefixsum + for i, (_, k) in enumerate(dc): + if i == 0: + cumm_sd.append(k / float(l)) + else: + # add the 2nd element of the i-th tuple in the dist_sd list + cumm_sd.append(cumm_sd[i - 1] + (k / float(l))) + + ### write stack_distance and line_accesses to a file ### + # dist_file_log = prep_folder + "dist_" + uni_name + '.log' + # write_dist_to_file(dist_file_log, line_accesses, list_sd, cumm_sd) + with open(dist_file, 'wb') as f: + np.save(f, np.array(line_accesses)) + np.save(f, np.array(list_sd)) + np.save(f, np.array(cumm_sd)) + print('line_acs, list_sd, cumm_sd saved to ', dist_file) + + else: + with open(dist_file, 'rb') as f: + line_accesses = deque(np.load(f)) + list_sd = deque(np.load(f)) + cumm_sd = deque(np.load(f)) + print('line_acs, list_sd, cumm_sd loaded from ', dist_file) + + ### generate correspondinf synthetic ### + synthetic_trace = trace_generate_lru( + line_accesses, list_sd, cumm_sd, syn_trace_len, trace_enable_padding + ) + # synthetic_trace = trace_generate_rand( + # line_accesses, list_sd, cumm_sd, len(trace), args.trace_enable_padding + # ) + # write synthetic trace to a file + # synthetic_file = prep_folder + "syn_" + uni_name + '.log' + # write_trace_to_file(synthetic_file, synthetic_trace) + return synthetic_trace + + +def trace_generate_rand( + line_accesses, list_sd, cumm_sd, out_trace_len, enable_padding=False +): + max_sd = list_sd[-1] + l = len(line_accesses) # !!!Unique, + i = 0 + ztrace = [] + for _ in range(out_trace_len): + sd = generate_stack_distance(list_sd, cumm_sd, max_sd, i, enable_padding) + mem_ref_within_line = 0 # floor(ra.rand(1)*cache_line_size) #0 + # generate memory reference + if sd == 0: # new reference # + line_ref = line_accesses.pop(0) + line_accesses.append(line_ref) + mem_ref = np.uint64(line_ref * cache_line_size + mem_ref_within_line) + i += 1 + else: # existing reference # + line_ref = line_accesses[l - sd] + mem_ref = np.uint64(line_ref * cache_line_size + mem_ref_within_line) + ztrace.append(mem_ref) + + return ztrace + + +def generate_stack_distance(cumm_val, cumm_dist, max_i, i, enable_padding=False): + u = ra.rand(1) + if i < max_i: + # only generate stack distances up to the number of new references seen so far + j = bisect.bisect(cumm_val, i) - 1 + fi = cumm_dist[j] + u *= fi # shrink distribution support to exclude last values + elif enable_padding: + # WARNING: disable generation of new references (once all have been seen) + fi = cumm_dist[0] + u = (1.0 - fi) * u + fi # remap distribution support to exclude first value + + for (j, f) in enumerate(cumm_dist): + if u <= f: + return cumm_val[j] + + +# fb synthetic distribution (input data) +def fb_generate_synthetic_input_batch( + m_den, + ln_emb, + mini_batch_size, + enable_padding=False, + trace_folder="fb_traces/" +): + # dense feature + Xt = torch.tensor(ra.rand(mini_batch_size, m_den).astype(np.float32)) + + # sparse feature (sparse indices) + lS_emb_offsets = deque() + lS_emb_indices = deque() + + # for each embedding generate a list of n lookups, + # where each lookup is composed of multiple sparse indices + + def is_available(trace_file): + if not os.path.exists(trace_file): + print("To generate trace_[i].log and size_trace_[i].log \ + files run emb_trace_writer.py ") + sys.exit(f"{trace_file} is not available.") + + for emb_tab_ind, _size in enumerate(ln_emb): + lS_batch_offsets = deque() + lS_batch_indices = deque() + + trace_file = trace_folder + f'size_trace_{emb_tab_ind}.log' + is_available(trace_file) + + lS_batch_sizes = np.array(syn_trace_from_trace(trace_file, mini_batch_size)).astype(np.int64) + + lS_batch_offsets = [0 for i in range(len(lS_batch_sizes))] + for key, val in enumerate(lS_batch_sizes[:-1]): + lS_batch_offsets[key + 1] = lS_batch_offsets[key] + val + + trace_file = trace_folder + f'trace_{emb_tab_ind}.log' + is_available(trace_file) + lS_batch_indices = np.array(syn_trace_from_trace(trace_file, + lS_batch_offsets[-1] + lS_batch_sizes[-1])).astype(np.int64) + + lS_emb_offsets.append(torch.tensor(lS_batch_offsets)) + lS_emb_indices.append(torch.tensor(lS_batch_indices)) + + return (Xt, list(lS_emb_offsets), list(lS_emb_indices)) + + +def generate_random_output_batch(n, num_targets, round_targets=False): + # target (probability of a click) + if round_targets: + P = np.round(ra.rand(n, num_targets).astype(np.float32)).astype(np.float32) + else: + P = ra.rand(n, num_targets).astype(np.float32) + + return torch.tensor(P) + + +class RandomDataset(Dataset): + + def __init__( + self, + m_den, + ln_emb, + data_size, + num_batches, + mini_batch_size, + num_indices_per_lookup, + num_indices_per_lookup_fixed, + num_targets=1, + round_targets=False, + data_generation="random", + trace_file="", + enable_padding=False, + reset_seed_on_access=False, + rand_seed=0 + ): + # compute batch size + nbatches = int(np.ceil((data_size * 1.0) / mini_batch_size)) + if num_batches != 0: + nbatches = num_batches + data_size = nbatches * mini_batch_size + # print("Total number of batches %d" % nbatches) + + # save args (recompute data_size if needed) + self.m_den = m_den + self.ln_emb = ln_emb + self.data_size = data_size + self.num_batches = nbatches + self.mini_batch_size = mini_batch_size + self.num_indices_per_lookup = num_indices_per_lookup + self.num_indices_per_lookup_fixed = num_indices_per_lookup_fixed + self.num_targets = num_targets + self.round_targets = round_targets + self.data_generation = data_generation + self.trace_file = trace_file + self.enable_padding = enable_padding + self.reset_seed_on_access = reset_seed_on_access + self.rand_seed = rand_seed + + def reset_numpy_seed(self, numpy_rand_seed): + np.random.seed(numpy_rand_seed) + # torch.manual_seed(numpy_rand_seed) + + def __getitem__(self, index): + + if isinstance(index, slice): + return [ + self[idx] for idx in range( + index.start or 0, index.stop or len(self), index.step or 1 + ) + ] + + # WARNING: reset seed on access to first element + # (e.g. if same random samples needed across epochs) + if self.reset_seed_on_access and index == 0: + self.reset_numpy_seed(self.rand_seed) + + # number of data points in a batch + n = min(self.mini_batch_size, self.data_size - (index * self.mini_batch_size)) + + # generate a batch of dense and sparse features + if self.data_generation == "fb_synthetic": + (X, lS_o, lS_i) = fb_generate_synthetic_input_batch( + self.m_den, + self.ln_emb, + self.mini_batch_size, + self.enable_padding + ) + else: + sys.exit( + "ERROR: --data-generation=" + self.data_generation + " is not supported" + ) + + # generate a batch of target (probability of a click) + T = generate_random_output_batch(n, self.num_targets, self.round_targets) + + return (X, lS_o, lS_i, T) + + def __len__(self): + # WARNING: note that we produce bacthes of outputs in __getitem__ + # therefore we should use num_batches rather than data_size below + return self.num_batches + + +def make_random_data_and_loader(args, ln_emb, m_den): + + print("Running with synthetic data!") + train_data = RandomDataset( + m_den, + ln_emb, + args.data_size, + args.num_batches, + args.mini_batch_size, + args.num_indices_per_lookup, + args.num_indices_per_lookup_fixed, + 1, # num_targets + args.round_targets, + args.data_generation, + args.data_trace_file, + args.data_trace_enable_padding, + reset_seed_on_access=True, + rand_seed=args.numpy_rand_seed + ) # WARNING: generates a batch of lookups at once + train_loader = torch.utils.data.DataLoader( + train_data, + batch_size=1, + shuffle=False, + num_workers=args.num_workers, + collate_fn=collate_wrapper_random, + pin_memory=False, + drop_last=False, # True + ) + return train_data, train_loader From eaee70cd58adde0d3461f562395c83f1c7a510e9 Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Tue, 11 Aug 2020 02:05:13 -0700 Subject: [PATCH 37/57] small fix --- dlrm_s_pytorch.py | 2 +- extend_distributed.py | 8 +++--- fb_synthetic_data_pytorch.py | 47 +++++++++++++++++++++++++++++------- 3 files changed, 43 insertions(+), 14 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 0ba0d414..a6e11a08 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -1235,7 +1235,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): # optimizer optimizer.step() - lr_scheduler.step() + ### lr_scheduler.step() if args.mlperf_logging: total_time += iteration_time diff --git a/extend_distributed.py b/extend_distributed.py index 7bc0ffbf..c995981a 100644 --- a/extend_distributed.py +++ b/extend_distributed.py @@ -122,10 +122,10 @@ def init_distributed(rank = -1, size = -1, backend=''): master_addr = "127.0.0.1" os.environ["MASTER_ADDR"] = master_addr - myenv = os.environ - for e in myenv: - print(e, "=", myenv[e]) - print("=== Done ===") +# myenv = os.environ +# for e in myenv: +# print(e, "=", myenv[e]) +# print("=== Done ===") if size > 1: my_local_rank = get_local_rank_from_env() diff --git a/fb_synthetic_data_pytorch.py b/fb_synthetic_data_pytorch.py index 367f1d1c..8af766a9 100644 --- a/fb_synthetic_data_pytorch.py +++ b/fb_synthetic_data_pytorch.py @@ -111,7 +111,8 @@ def trace_profile(trace, enable_padding=False): def trace_generate_lru( - line_accesses, list_sd, cumm_sd, out_trace_len, enable_padding=False + line_accesses, list_sd, cumm_sd, out_trace_len, enable_padding=False, + max_value=-1 ): max_sd = list_sd[-1] l = len(line_accesses) @@ -134,6 +135,7 @@ def trace_generate_lru( del line_accesses[l - sd] line_accesses.append(line_ref) # save generated memory reference + mem_ref = mem_ref % max_value if max_value > -1 else mem_ref ztrace.append(mem_ref) return ztrace @@ -146,6 +148,7 @@ def syn_trace_from_trace( trace_file_binary_type=False, trace_enable_padding=False, numpy_rand_seed=123, + max_value=-1, print_precision=5 ): @@ -211,7 +214,8 @@ def syn_trace_from_trace( ### generate correspondinf synthetic ### synthetic_trace = trace_generate_lru( - line_accesses, list_sd, cumm_sd, syn_trace_len, trace_enable_padding + line_accesses, list_sd, cumm_sd, syn_trace_len, trace_enable_padding, + max_value ) # synthetic_trace = trace_generate_rand( # line_accesses, list_sd, cumm_sd, len(trace), args.trace_enable_padding @@ -226,7 +230,7 @@ def trace_generate_rand( line_accesses, list_sd, cumm_sd, out_trace_len, enable_padding=False ): max_sd = list_sd[-1] - l = len(line_accesses) # !!!Unique, + l = len(line_accesses) i = 0 ztrace = [] for _ in range(out_trace_len): @@ -287,12 +291,32 @@ def is_available(trace_file): files run emb_trace_writer.py ") sys.exit(f"{trace_file} is not available.") - for emb_tab_ind, _size in enumerate(ln_emb): + num_aval_traces = 0 + print_num_aval_traces = True + for emb_tab_ind, size in enumerate(ln_emb): lS_batch_offsets = deque() lS_batch_indices = deque() trace_file = trace_folder + f'size_trace_{emb_tab_ind}.log' - is_available(trace_file) + if not os.path.exists(trace_file): + if emb_tab_ind == 0: + print("To generate trace_[i].log and size_trace_[i].log \ + files run emb_trace_writer.py ") + sys.exit(f"{trace_file} is not available.") + + if print_num_aval_traces: + print(f"Number of recognized trace files is {num_aval_traces}") + print_num_aval_traces = False + + trace_file_id = emb_tab_ind % num_aval_traces + trace_file_name = f'size_trace_{trace_file_id}.log' + print(f"Trace file {trace_folder}{trace_file_name} " + f"used instead of {trace_file}") + trace_file = trace_folder + trace_file_name + + else: + num_aval_traces += 1 + trace_file_id = emb_tab_ind lS_batch_sizes = np.array(syn_trace_from_trace(trace_file, mini_batch_size)).astype(np.int64) @@ -300,10 +324,15 @@ def is_available(trace_file): for key, val in enumerate(lS_batch_sizes[:-1]): lS_batch_offsets[key + 1] = lS_batch_offsets[key] + val - trace_file = trace_folder + f'trace_{emb_tab_ind}.log' + trace_file = trace_folder + f'trace_{trace_file_id}.log' is_available(trace_file) - lS_batch_indices = np.array(syn_trace_from_trace(trace_file, - lS_batch_offsets[-1] + lS_batch_sizes[-1])).astype(np.int64) + lS_batch_indices = np.array( + syn_trace_from_trace( + trace_file, + lS_batch_offsets[-1] + lS_batch_sizes[-1], + max_value=size + ) + ).astype(np.int64) lS_emb_offsets.append(torch.tensor(lS_batch_offsets)) lS_emb_indices.append(torch.tensor(lS_batch_indices)) @@ -410,7 +439,6 @@ def __len__(self): def make_random_data_and_loader(args, ln_emb, m_den): - print("Running with synthetic data!") train_data = RandomDataset( m_den, ln_emb, @@ -437,3 +465,4 @@ def make_random_data_and_loader(args, ln_emb, m_den): drop_last=False, # True ) return train_data, train_loader + From d32dfd7a2076ff0ea302a9f95a621a0ff61fcd53 Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Tue, 11 Aug 2020 03:48:54 -0700 Subject: [PATCH 38/57] clean files --- fb_emb_trace_writer.py | 264 -------------------- fb_synthetic_data_pytorch.py | 468 ----------------------------------- 2 files changed, 732 deletions(-) delete mode 100644 fb_emb_trace_writer.py delete mode 100644 fb_synthetic_data_pytorch.py diff --git a/fb_emb_trace_writer.py b/fb_emb_trace_writer.py deleted file mode 100644 index 538e1cd0..00000000 --- a/fb_emb_trace_writer.py +++ /dev/null @@ -1,264 +0,0 @@ -# Description: using torch internal koski reader to write two traces -# for each embedding table. The first one is trace_{i}.log which represents -# indices of each sample for embedding table i. The second one is size_trace_{i}.log -# which stores the number of indices for each sample for embedding table i. -# These traces would be used in fb synthetic data generation which imitate fb data -# for benchmarking DLRM in external infrastructures. - - -from __future__ import absolute_import, division, print_function, unicode_literals - -import argparse -import builtins -import argparse -import os - -# data generation -import dlrm_data_pytorch as dp - -# numpy -import numpy as np - -# pytorch -import torch - -# The following import is needed only for runs that use production data -# For those runs, the proper dependencies should be set up before the run. -try: - import fb_data_pytorch as fbdata -except ImportError: - print("Production libs are not set up.") - print("Note: Not all runs need production libs.") - pass - -exc = getattr(builtins, "IOError", "FileNotFoundError") - - -if __name__ == "__main__": - ### parse arguments ### - parser = argparse.ArgumentParser( - description="Train Deep Learning Recommendation Model (DLRM)" - ) - # model related parameters - parser.add_argument("--arch-sparse-feature-size", type=int, default=2) - parser.add_argument("--arch-embedding-size", type=str, default="4-3-2") - # j will be replaced with the table number - parser.add_argument("--arch-mlp-bot", type=str, default="4-3-2") - parser.add_argument("--arch-mlp-top", type=str, default="4-2-1") - parser.add_argument("--arch-interaction-op", type=str, default="dot") - parser.add_argument("--arch-interaction-itself", action="store_true", default=False) - parser.add_argument("--weighted-pooling", type=str, default=None) - # embedding table options - parser.add_argument("--md-flag", action="store_true", default=False) - parser.add_argument("--md-threshold", type=int, default=200) - parser.add_argument("--md-temperature", type=float, default=0.3) - parser.add_argument("--md-round-dims", action="store_true", default=False) - parser.add_argument("--qr-flag", action="store_true", default=False) - parser.add_argument("--qr-threshold", type=int, default=200) - parser.add_argument("--qr-operation", type=str, default="mult") - parser.add_argument("--qr-collisions", type=int, default=4) - parser.add_argument("--hype-flag", action="store_true", default=False) - parser.add_argument("--cluster-fb-preproc", action="store_true", default=False) - # activations and loss - parser.add_argument("--activation-function", type=str, default="relu") - parser.add_argument("--loss-function", type=str, default="mse") # or bce or wbce - parser.add_argument("--loss-weights", type=str, default="1.0-1.0") # for wbce - parser.add_argument("--loss-threshold", type=float, default=0.0) # 1.0e-7 - parser.add_argument("--round-targets", type=bool, default=False) - # data - parser.add_argument("--data-size", type=int, default=1) - parser.add_argument("--num-batches", type=int, default=0) - parser.add_argument( - "--data-generation", type=str, default="random" - ) # synthetic or dataset - parser.add_argument("--data-trace-file", type=str, default="./input/dist_emb_j.log") - parser.add_argument("--data-set", type=str, default="kaggle") # or terabyte - parser.add_argument("--raw-data-file", type=str, default="") - parser.add_argument("--processed-data-file", type=str, default="") - parser.add_argument("--data-randomize", type=str, default="total") # or day or none - parser.add_argument("--data-trace-enable-padding", type=bool, default=False) - parser.add_argument("--max-ind-range", type=int, default=-1) - parser.add_argument("--data-sub-sample-rate", type=float, default=0.0) # in [0, 1] - parser.add_argument("--num-indices-per-lookup", type=int, default=10) - parser.add_argument("--num-indices-per-lookup-fixed", type=bool, default=False) - parser.add_argument("--num-workers", type=int, default=0) - parser.add_argument("--memory-map", action="store_true", default=False) - # training - parser.add_argument("--mini-batch-size", type=int, default=1) - parser.add_argument("--nepochs", type=int, default=1) - parser.add_argument("--learning-rate", type=float, default=0.01) - parser.add_argument("--print-precision", type=int, default=5) - parser.add_argument("--numpy-rand-seed", type=int, default=123) - parser.add_argument("--sync-dense-params", type=bool, default=True) - parser.add_argument("--optimizer", type=str, default="sgd") - # inference - parser.add_argument("--inference-only", action="store_true", default=False) - # quantize - parser.add_argument("--quantize-with-bit", type=int, default=32) - # onnx - parser.add_argument("--save-onnx", action="store_true", default=False) - # gpu - parser.add_argument("--use-gpu", action="store_true", default=False) - # debugging and profiling - parser.add_argument("--print-freq", type=int, default=1) - parser.add_argument("--test-freq", type=int, default=-1) - parser.add_argument("--test-mini-batch-size", type=int, default=-1) - parser.add_argument("--test-num-workers", type=int, default=-1) - parser.add_argument("--fb-test-sample-limit", type=int, default=5000) - parser.add_argument("--fb-train-sample-limit", type=int, default=500000) - parser.add_argument("--fb-test-ds", type=str, default="") - parser.add_argument("--fb-train-ds", type=str, default="") - parser.add_argument("--fb-stream-train-samples", action="store_true", default=False) - parser.add_argument("--fb-stream-test-samples", action="store_true", default=False) - parser.add_argument("--fb-stream-prefetch-size", type=int, default=128) - parser.add_argument("--fb-preprocess", action="store_true", default=False) - parser.add_argument("--fb-report-ne", action="store_true", default=False) - parser.add_argument("--fb-window-size", type=int, default=-1) - parser.add_argument("--fb-loss-bias", type=float, default=1.0) # shifted BCE loss c - parser.add_argument("--fb-run-id", type=int, default=-1) - parser.add_argument("--print-time", action="store_true", default=False) - parser.add_argument("--debug-mode", action="store_true", default=False) - parser.add_argument("--enable-profiling", action="store_true", default=False) - parser.add_argument("--plot-compute-graph", action="store_true", default=False) - parser.add_argument("--tensor-board-filename", type=str, default="run_kaggle_pt") - # store/load model - parser.add_argument("--save-model", type=str, default="") - parser.add_argument("--load-model", type=str, default="") - # mlperf logging (disables other output and stops early) - parser.add_argument("--mlperf-logging", action="store_true", default=False) - # stop at target accuracy Kaggle 0.789, Terabyte (sub-sampled=0.875) 0.8107 - parser.add_argument("--mlperf-acc-threshold", type=float, default=0.0) - # stop at target AUC Terabyte (no subsampling) 0.8025 - parser.add_argument("--mlperf-auc-threshold", type=float, default=0.0) - parser.add_argument("--trace-folder", type=str, default="fb_traces/") - args = parser.parse_args() - print(args) - - if (args.test_mini_batch_size < 0): - # if the parameter is not set, use the training batch size - args.test_mini_batch_size = args.mini_batch_size - if (args.test_num_workers < 0): - # if the parameter is not set, use the same parameter for training - args.test_num_workers = args.num_workers - - ### prepare training data ### - ln_bot = np.fromstring(args.arch_mlp_bot, dtype=int, sep="-") - # input data - if args.data_generation == "dataset": - - if (args.data_set == "fb"): - train_data, train_ld, test_data, test_ld, table_feature_map = \ - fbdata.make_fb_data_and_loaders(args) - nbatches = args.num_batches if args.num_batches > 0 \ - else train_data.num_batches - # TODO: Handle the cases where fb_train_sample_limit is set to more - # than the number of samples in the table. - nbatches_test = test_data.num_batches - else: # criteo kaggle or terabyte - train_data, train_ld, test_data, test_ld = \ - dp.make_criteo_data_and_loaders(args) - nbatches = args.num_batches if args.num_batches > 0 else len(train_ld) - nbatches_test = len(test_ld) - - ln_emb = train_data.counts - # enforce maximum limit on number of vectors per embedding - if args.max_ind_range > 0: - ln_emb = np.array(list(map( - lambda x: x if x < args.max_ind_range else args.max_ind_range, - ln_emb - ))) - else: - ln_emb = np.array(ln_emb) - - m_den = train_data.m_den - ln_bot[0] = m_den - - else: - # input and target at random - ln_emb = np.fromstring(args.arch_embedding_size, dtype=int, sep="-") - m_den = ln_bot[0] - train_data, train_ld = dp.make_random_data_and_loader(args, ln_emb, m_den) - nbatches = args.num_batches if args.num_batches > 0 else len(train_ld) - - # For FB data, each batch has a label tensor packed, while the others don't. - # The following function is a wrapper to avoid checking this multiple times in th - # loop below. - def unpack_batch(b): - if args.data_generation == "dataset" and args.data_set == "fb": - # Experiment with weighted samples - return b[0], b[1], b[2], b[3], b[4] - else: - # Experiment with unweighted samples - return b[0], b[1], b[2], b[3], torch.ones(b[3].size()) - - ### parse command line arguments ### - num_fea = ln_emb.size + 1 # num sparse + num dense features - - print( - "mlp bot arch " - + str(ln_bot.size - 1) - + " layers, with input to output dimensions:" - ) - print(ln_bot) - print("# of features (sparse and dense)") - print(num_fea) - print("dense feature size") - print(m_den) - print("sparse feature size") - print( - "# of embeddings (= # of sparse features) " - + str(ln_emb.size) - + ", with dimensions " - + "x:" - ) - print(ln_emb) - - f_traces = [-1 for i in range(ln_emb.size)] - f_size_trace = [-1 for i in range(ln_emb.size)] - if not os.path.exists(args.trace_folder): - os.mkdir(args.trace_folder) - - for i in range(ln_emb.size): - f_traces[i] = open(f'{args.trace_folder}/trace_{i}.log', 'w') - f_size_trace[i] = open(f'{args.trace_folder}/size_trace_{i}.log', 'w') - - def f(x): - return x.item() - - for j, inputBatch in enumerate(train_ld): - X, lS_o, lS_i, T, W = unpack_batch(inputBatch) - - # sparse index traces - for table_ind, S_i in enumerate(lS_i): - - for item_ind, item in enumerate(S_i): - elm = f"{item}, " - if j == nbatches - 1 and item_ind == len(S_i) - 1: # the final last item - elm = f"{item}" - - f_traces[table_ind].write(elm) - - # size traces - for table_ind, S_o in enumerate(lS_o): - - s_i = lS_i[table_ind] - for ind, _item in enumerate(S_o): - if ind == len(S_o) - 1: # the last item in this batch - size_trace_elm = f"{len(s_i) - f(S_o[-1])}, " - if j == nbatches - 1 : # the final last item - size_trace_elm = f"{len(s_i) - f(S_o[-1])}" - else: - itm = f(S_o[ind + 1]) - f(S_o[ind]) - size_trace_elm = str(itm) + ', ' - - f_size_trace[table_ind].write(size_trace_elm) - - if (j + 1) % args.print_freq == 0: - print(f"mini-batch: {j + 1}") - - # closing trace files - for i in range(ln_emb.size): - f_traces[i].close() - f_size_trace[i].close() - - print(f"Trace and size_trace files are in the folder {args.trace_folder}") diff --git a/fb_synthetic_data_pytorch.py b/fb_synthetic_data_pytorch.py deleted file mode 100644 index 8af766a9..00000000 --- a/fb_synthetic_data_pytorch.py +++ /dev/null @@ -1,468 +0,0 @@ -#!/usr/bin/env python3 - -# Description: Generating synthetic data from the same distribution of the fb data. - -from __future__ import absolute_import, division, print_function, unicode_literals - -import torch -import sys -import numpy as np -import os -import sys -import operator -from numpy import random as ra -from collections import deque -import collections -import bisect - -# WARNING: global define, must be consistent across all synthetic functions -cache_line_size = 1 - -from torch.utils.data import Dataset - - -def collate_wrapper_random(list_of_tuples): - # where each tuple is (X, lS_o, lS_i, T) - (X, lS_o, lS_i, T) = list_of_tuples[0] - return (X, - torch.stack(lS_o), - lS_i, - T) - - -# auxiliary read routines -def read_trace_from_file(file_path, ignore_error=True, trace_file_binary_type=False): - try: - with open(file_path) as f: - if trace_file_binary_type: - array = np.fromfile(f, dtype=np.uint64) - trace = array.astype(np.uint64).tolist() - else: - line = f.readline() - if line == '': - trace = [] - else : - trace = list(map(lambda x: np.uint64(x), line.split(", "))) - return trace - except Exception as e: - if ignore_error: - print(f"Can not read '{file_path}' . {e}.") - else: - raise (e) - - -# auxiliary write routines -def write_trace_to_file(file_path, trace, trace_file_binary_type=False): - try: - if trace_file_binary_type: - with open(file_path, "wb+") as f: - np.array(trace).astype(np.uint64).tofile(f) - else: - with open(file_path, "w+") as f: - s = str(trace) - f.write(s[1 : len(s) - 1]) - except Exception as e: - print(e, "\n Unable to write trace file ", file_path) - - -def trace_profile(trace, enable_padding=False): - rstack = deque() # S - stack_distances = deque() # SDS - line_accesses = deque() # L - for x in trace: - r = np.uint64(x / cache_line_size) - l = len(rstack) - try: # found # - i = rstack.index(r) - # WARNING: I believe below is the correct depth in terms of meaning of the - # algorithm, but that is not what seems to be in the paper alg. - # -1 can be subtracted if we defined the distance between - # consecutive accesses (e.g. r, r) as 0 rather than 1. - sd = l - i # - 1 - # push r to the end of stack_distances - stack_distances.appendleft(sd) - # remove r from its position and insert to the top of stack - del rstack[i] # rstack.remove(r) - rstack.append(r) - except ValueError: # not found # - sd = 0 # -1 - # push r to the end of stack_distances/line_accesses - stack_distances.appendleft(sd) - line_accesses.appendleft(r) - # push r to the top of stack - rstack.append(r) - - if enable_padding: - # WARNING: notice that as the ratio between the number of samples (l) - # and cardinality (c) of a sample increases the probability of - # generating a sample gets smaller and smaller because there are - # few new samples compared to repeated samples. This means that for a - # long trace with relatively small cardinality it will take longer to - # generate all new samples and therefore obtain full distribution support - # and hence it takes longer for distribution to resemble the original. - # Therefore, we may pad the number of new samples to be on par with - # average number of samples l/c artificially. - l = len(stack_distances) - c = max(stack_distances) - padding = int(np.ceil(l / c)) - stack_distances = stack_distances + [0] * padding - - return (rstack, stack_distances, line_accesses) - - -def trace_generate_lru( - line_accesses, list_sd, cumm_sd, out_trace_len, enable_padding=False, - max_value=-1 -): - max_sd = list_sd[-1] - l = len(line_accesses) - i = 0 - ztrace = deque() - for _ in range(out_trace_len): - sd = generate_stack_distance(list_sd, cumm_sd, max_sd, i, enable_padding) - mem_ref_within_line = 0 # floor(ra.rand(1)*cache_line_size) #0 - - # generate memory reference - if sd == 0: # new reference # - line_ref = line_accesses[0] - del line_accesses[0] - line_accesses.append(line_ref) - mem_ref = np.uint64(line_ref * cache_line_size + mem_ref_within_line) - i += 1 - else: # existing reference # - line_ref = line_accesses[l - sd] - mem_ref = np.uint64(line_ref * cache_line_size + mem_ref_within_line) - del line_accesses[l - sd] - line_accesses.append(line_ref) - # save generated memory reference - mem_ref = mem_ref % max_value if max_value > -1 else mem_ref - ztrace.append(mem_ref) - - return ztrace - - -def syn_trace_from_trace( - trace_file, - syn_trace_len, - prep_folder="stack_dists_line_aces/", - trace_file_binary_type=False, - trace_enable_padding=False, - numpy_rand_seed=123, - max_value=-1, - print_precision=5 -): - - ### some basic setup ### - if numpy_rand_seed != -1: - np.random.seed(numpy_rand_seed) - np.set_printoptions(precision=print_precision) - uni_name = '_'.join(trace_file[:-4].split('/')) - - ### profile trace ### - dist_file = prep_folder + "dist_" + uni_name + ".npy" - if not os.path.exists(dist_file): - if not os.path.exists(prep_folder): - os.mkdir(prep_folder) - - ### read trace ### - trace = read_trace_from_file(trace_file) - - if trace == []: - return [] - - (_, stack_distances, line_accesses) = trace_profile( - trace, trace_enable_padding - ) - stack_distances.reverse() - line_accesses.reverse() - - ### compute probability distribution ### - # count items - l = len(stack_distances) - dc = sorted( - collections.Counter(stack_distances).items(), key=operator.itemgetter(0) - ) - - # create a distribution - list_sd = list(map(lambda tuple_x_k: tuple_x_k[0], dc)) # x = tuple_x_k[0] - # dist_sd = list( - # map(lambda tuple_x_k: tuple_x_k[1] / float(l), dc) - # ) # k = tuple_x_k[1] - cumm_sd = deque() # np.cumsum(dc).tolist() #prefixsum - for i, (_, k) in enumerate(dc): - if i == 0: - cumm_sd.append(k / float(l)) - else: - # add the 2nd element of the i-th tuple in the dist_sd list - cumm_sd.append(cumm_sd[i - 1] + (k / float(l))) - - ### write stack_distance and line_accesses to a file ### - # dist_file_log = prep_folder + "dist_" + uni_name + '.log' - # write_dist_to_file(dist_file_log, line_accesses, list_sd, cumm_sd) - with open(dist_file, 'wb') as f: - np.save(f, np.array(line_accesses)) - np.save(f, np.array(list_sd)) - np.save(f, np.array(cumm_sd)) - print('line_acs, list_sd, cumm_sd saved to ', dist_file) - - else: - with open(dist_file, 'rb') as f: - line_accesses = deque(np.load(f)) - list_sd = deque(np.load(f)) - cumm_sd = deque(np.load(f)) - print('line_acs, list_sd, cumm_sd loaded from ', dist_file) - - ### generate correspondinf synthetic ### - synthetic_trace = trace_generate_lru( - line_accesses, list_sd, cumm_sd, syn_trace_len, trace_enable_padding, - max_value - ) - # synthetic_trace = trace_generate_rand( - # line_accesses, list_sd, cumm_sd, len(trace), args.trace_enable_padding - # ) - # write synthetic trace to a file - # synthetic_file = prep_folder + "syn_" + uni_name + '.log' - # write_trace_to_file(synthetic_file, synthetic_trace) - return synthetic_trace - - -def trace_generate_rand( - line_accesses, list_sd, cumm_sd, out_trace_len, enable_padding=False -): - max_sd = list_sd[-1] - l = len(line_accesses) - i = 0 - ztrace = [] - for _ in range(out_trace_len): - sd = generate_stack_distance(list_sd, cumm_sd, max_sd, i, enable_padding) - mem_ref_within_line = 0 # floor(ra.rand(1)*cache_line_size) #0 - # generate memory reference - if sd == 0: # new reference # - line_ref = line_accesses.pop(0) - line_accesses.append(line_ref) - mem_ref = np.uint64(line_ref * cache_line_size + mem_ref_within_line) - i += 1 - else: # existing reference # - line_ref = line_accesses[l - sd] - mem_ref = np.uint64(line_ref * cache_line_size + mem_ref_within_line) - ztrace.append(mem_ref) - - return ztrace - - -def generate_stack_distance(cumm_val, cumm_dist, max_i, i, enable_padding=False): - u = ra.rand(1) - if i < max_i: - # only generate stack distances up to the number of new references seen so far - j = bisect.bisect(cumm_val, i) - 1 - fi = cumm_dist[j] - u *= fi # shrink distribution support to exclude last values - elif enable_padding: - # WARNING: disable generation of new references (once all have been seen) - fi = cumm_dist[0] - u = (1.0 - fi) * u + fi # remap distribution support to exclude first value - - for (j, f) in enumerate(cumm_dist): - if u <= f: - return cumm_val[j] - - -# fb synthetic distribution (input data) -def fb_generate_synthetic_input_batch( - m_den, - ln_emb, - mini_batch_size, - enable_padding=False, - trace_folder="fb_traces/" -): - # dense feature - Xt = torch.tensor(ra.rand(mini_batch_size, m_den).astype(np.float32)) - - # sparse feature (sparse indices) - lS_emb_offsets = deque() - lS_emb_indices = deque() - - # for each embedding generate a list of n lookups, - # where each lookup is composed of multiple sparse indices - - def is_available(trace_file): - if not os.path.exists(trace_file): - print("To generate trace_[i].log and size_trace_[i].log \ - files run emb_trace_writer.py ") - sys.exit(f"{trace_file} is not available.") - - num_aval_traces = 0 - print_num_aval_traces = True - for emb_tab_ind, size in enumerate(ln_emb): - lS_batch_offsets = deque() - lS_batch_indices = deque() - - trace_file = trace_folder + f'size_trace_{emb_tab_ind}.log' - if not os.path.exists(trace_file): - if emb_tab_ind == 0: - print("To generate trace_[i].log and size_trace_[i].log \ - files run emb_trace_writer.py ") - sys.exit(f"{trace_file} is not available.") - - if print_num_aval_traces: - print(f"Number of recognized trace files is {num_aval_traces}") - print_num_aval_traces = False - - trace_file_id = emb_tab_ind % num_aval_traces - trace_file_name = f'size_trace_{trace_file_id}.log' - print(f"Trace file {trace_folder}{trace_file_name} " - f"used instead of {trace_file}") - trace_file = trace_folder + trace_file_name - - else: - num_aval_traces += 1 - trace_file_id = emb_tab_ind - - lS_batch_sizes = np.array(syn_trace_from_trace(trace_file, mini_batch_size)).astype(np.int64) - - lS_batch_offsets = [0 for i in range(len(lS_batch_sizes))] - for key, val in enumerate(lS_batch_sizes[:-1]): - lS_batch_offsets[key + 1] = lS_batch_offsets[key] + val - - trace_file = trace_folder + f'trace_{trace_file_id}.log' - is_available(trace_file) - lS_batch_indices = np.array( - syn_trace_from_trace( - trace_file, - lS_batch_offsets[-1] + lS_batch_sizes[-1], - max_value=size - ) - ).astype(np.int64) - - lS_emb_offsets.append(torch.tensor(lS_batch_offsets)) - lS_emb_indices.append(torch.tensor(lS_batch_indices)) - - return (Xt, list(lS_emb_offsets), list(lS_emb_indices)) - - -def generate_random_output_batch(n, num_targets, round_targets=False): - # target (probability of a click) - if round_targets: - P = np.round(ra.rand(n, num_targets).astype(np.float32)).astype(np.float32) - else: - P = ra.rand(n, num_targets).astype(np.float32) - - return torch.tensor(P) - - -class RandomDataset(Dataset): - - def __init__( - self, - m_den, - ln_emb, - data_size, - num_batches, - mini_batch_size, - num_indices_per_lookup, - num_indices_per_lookup_fixed, - num_targets=1, - round_targets=False, - data_generation="random", - trace_file="", - enable_padding=False, - reset_seed_on_access=False, - rand_seed=0 - ): - # compute batch size - nbatches = int(np.ceil((data_size * 1.0) / mini_batch_size)) - if num_batches != 0: - nbatches = num_batches - data_size = nbatches * mini_batch_size - # print("Total number of batches %d" % nbatches) - - # save args (recompute data_size if needed) - self.m_den = m_den - self.ln_emb = ln_emb - self.data_size = data_size - self.num_batches = nbatches - self.mini_batch_size = mini_batch_size - self.num_indices_per_lookup = num_indices_per_lookup - self.num_indices_per_lookup_fixed = num_indices_per_lookup_fixed - self.num_targets = num_targets - self.round_targets = round_targets - self.data_generation = data_generation - self.trace_file = trace_file - self.enable_padding = enable_padding - self.reset_seed_on_access = reset_seed_on_access - self.rand_seed = rand_seed - - def reset_numpy_seed(self, numpy_rand_seed): - np.random.seed(numpy_rand_seed) - # torch.manual_seed(numpy_rand_seed) - - def __getitem__(self, index): - - if isinstance(index, slice): - return [ - self[idx] for idx in range( - index.start or 0, index.stop or len(self), index.step or 1 - ) - ] - - # WARNING: reset seed on access to first element - # (e.g. if same random samples needed across epochs) - if self.reset_seed_on_access and index == 0: - self.reset_numpy_seed(self.rand_seed) - - # number of data points in a batch - n = min(self.mini_batch_size, self.data_size - (index * self.mini_batch_size)) - - # generate a batch of dense and sparse features - if self.data_generation == "fb_synthetic": - (X, lS_o, lS_i) = fb_generate_synthetic_input_batch( - self.m_den, - self.ln_emb, - self.mini_batch_size, - self.enable_padding - ) - else: - sys.exit( - "ERROR: --data-generation=" + self.data_generation + " is not supported" - ) - - # generate a batch of target (probability of a click) - T = generate_random_output_batch(n, self.num_targets, self.round_targets) - - return (X, lS_o, lS_i, T) - - def __len__(self): - # WARNING: note that we produce bacthes of outputs in __getitem__ - # therefore we should use num_batches rather than data_size below - return self.num_batches - - -def make_random_data_and_loader(args, ln_emb, m_den): - - train_data = RandomDataset( - m_den, - ln_emb, - args.data_size, - args.num_batches, - args.mini_batch_size, - args.num_indices_per_lookup, - args.num_indices_per_lookup_fixed, - 1, # num_targets - args.round_targets, - args.data_generation, - args.data_trace_file, - args.data_trace_enable_padding, - reset_seed_on_access=True, - rand_seed=args.numpy_rand_seed - ) # WARNING: generates a batch of lookups at once - train_loader = torch.utils.data.DataLoader( - train_data, - batch_size=1, - shuffle=False, - num_workers=args.num_workers, - collate_fn=collate_wrapper_random, - pin_memory=False, - drop_last=False, # True - ) - return train_data, train_loader - From f1d301c50f6570e64ea2559e58021bd6521acf58 Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Mon, 17 Aug 2020 09:35:37 -0700 Subject: [PATCH 39/57] add readme for param branch --- README.params | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 README.params diff --git a/README.params b/README.params new file mode 100644 index 00000000..339c9b0f --- /dev/null +++ b/README.params @@ -0,0 +1,37 @@ + +This note is intended for branch params_dlrm. The purpose of this branch is to test DLRM with large data models. + +The data is configured with total 1359 embedding tables, including user related 815 tables with hash size 26000000 and +ads related tables with hash size 14000000. You could set up the parameters in the shell script as following: + +large_arch_emb_usr=$(printf '26000000%.0s' {1..815}) +large_arch_emb_usr=${large_arch_emb_usr//"03"/"0-3"} + +large_arch_emb_ads=$(printf '14000000%.0s' {1..544}) +large_arch_emb_ads=${large_arch_emb_ads//"04"/"0-4"} + +large_arch_emb="$large_arch_emb_usr-$large_arch_emb_ads" + +There are a new parameter which do not exist in master branch: +--arch-project-size : reduces the number of interaction features for the dot operation. + This is mainly due to the memory concern. It reduces the memory size needed for top MLP. + +Also, the data-generation supports one more data model "fb_synthetic". + +Here are the command line to run the large model dlrm: +python dlrm_s_pytorch.py + --arch-sparse-feature-size=128 + --arch-mlp-bot="2000-1024-1024-1024-1024-1024-1024-1024-1024-1024-1024-512-128" + --arch-mlp-top="4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-1" + --arch-embedding-size=$large_arch_emb + --data-generation=rando + --loss-function=bce + --round-targets=True + --learning-rate=0.1 + --mini-batch-size=2048 + --print-freq=10240 + --print-time + --test-mini-batch-size=16384 + --test-num-workers=16 + --arch-projection-size 30 + From 2fe7f8106f50cce72de23f0100be2107491cb86d Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Tue, 18 Aug 2020 13:19:18 -0700 Subject: [PATCH 40/57] put project into separate file --- README.params | 9 +++++++++ dlrm_s_pytorch.py | 41 ++++++++++------------------------------- project.py | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 31 deletions(-) create mode 100644 project.py diff --git a/README.params b/README.params index 339c9b0f..6abca3db 100644 --- a/README.params +++ b/README.params @@ -18,6 +18,11 @@ There are a new parameter which do not exist in master branch: Also, the data-generation supports one more data model "fb_synthetic". +The model parameter size depends on the mini-batch-size and data types. +Suppose, we set mini-batch-size=2048, data type=float32, +The model size for embedding tables are about 14TB. +The MLP model size are about 8.5GB per copy. + Here are the command line to run the large model dlrm: python dlrm_s_pytorch.py --arch-sparse-feature-size=128 @@ -33,5 +38,9 @@ python dlrm_s_pytorch.py --print-time --test-mini-batch-size=16384 --test-num-workers=16 + --num-indices-per-lookup-fixed=1 + --num-indices-per-lookup=28 --arch-projection-size 30 + --use-gpu + diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index a6e11a08..2ca6ded9 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -94,6 +94,7 @@ import sklearn.metrics import uuid +import project import fb_synthetic_data_pytorch as fb_syn_data @@ -235,29 +236,6 @@ def create_emb(self, m, ln): np.random.set_state(np_rand_state) return emb_l - def create_proj(self, n, m): - # build MLP layer by layer - layers = nn.ModuleList() - # construct fully connected operator - LL = nn.Linear(int(n), int(m), bias=True) - - # initialize the weights - # with torch.no_grad(): - # custom Xavier input, output or two-sided fill - mean = 0.0 # std_dev = np.sqrt(variance) - std_dev = np.sqrt(2 / (m + n)) # np.sqrt(1 / m) # np.sqrt(1 / n) - W = np.random.normal(mean, std_dev, size=(m, n)).astype(np.float32) - std_dev = np.sqrt(1 / m) # np.sqrt(2 / (m + 1)) - bt = np.random.normal(mean, std_dev, size=m).astype(np.float32) - # approach 1 - LL.weight.data = torch.tensor(W, requires_grad=True) - LL.bias.data = torch.tensor(bt, requires_grad=True) - # approach 2: constant value ? - layers.append(LL) - - return torch.nn.Sequential(*layers) - - def __init__( self, m_spa=None, @@ -328,7 +306,7 @@ def __init__( self.bot_l = self.create_mlp(ln_bot, sigmoid_bot) self.top_l = self.create_mlp(ln_top, sigmoid_top) if (proj_size > 0): - self.proj_l = self.create_proj(len(ln_emb)+1, proj_size) + self.proj_l = project.create_proj(len(ln_emb)+1, proj_size) def apply_mlp(self, x, layers): # approach 1: use ModuleList @@ -377,13 +355,14 @@ def interact_features(self, x, ly): T = torch.cat([x] + ly, dim=1).view((batch_size, -1, d)) # perform a dot product if (self.proj_size > 0): - TT = torch.transpose(T, 1, 2) - TS = torch.reshape(TT, (-1, TT.size(2))) - TC = self.apply_mlp(TS, self.proj_l) - TR = torch.reshape(TC, (-1, d ,self.proj_size)) - Z = torch.bmm(T, TR) - Zflat = Z.view((batch_size, -1)) - R = torch.cat([x] + [Zflat], dim=1) + R = project.project(T, self.proj_size, batch_size, d, x, self.proj_l) + #TT = torch.transpose(T, 1, 2) + #TS = torch.reshape(TT, (-1, TT.size(2))) + #TC = self.apply_mlp(TS, self.proj_l) + #TR = torch.reshape(TC, (-1, d ,self.proj_size)) + #Z = torch.bmm(T, TR) + #Zflat = Z.view((batch_size, -1)) + #R = torch.cat([x] + [Zflat], dim=1) else: Z = torch.bmm(T, torch.transpose(T, 1, 2)) # append dense feature with the interactions (into a row vector) diff --git a/project.py b/project.py new file mode 100644 index 00000000..cc0ad0a8 --- /dev/null +++ b/project.py @@ -0,0 +1,40 @@ + +import sys +import torch +import torch.nn as nn +import numpy as np + +def project(T, project_size, batch_size, d, x, layer): + + TT = torch.transpose(T, 1, 2) + TS = torch.reshape(TT, (-1, TT.size(2))) + TC = layer(TS) + TR = torch.reshape(TC, (-1, d, project_size)) + Z = torch.bmm(T, TR) + Zflat = Z.view((batch_size, -1)) + R = torch.cat([x] + [Zflat], dim=1) + + return R + +def create_proj(n, m): + # build MLP layer by layer + layers = nn.ModuleList() + # construct fully connected operator + LL = nn.Linear(int(n), int(m), bias=True) + + # initialize the weights + # with torch.no_grad(): + # custom Xavier input, output or two-sided fill + mean = 0.0 # std_dev = np.sqrt(variance) + std_dev = np.sqrt(2 / (m + n)) # np.sqrt(1 / m) # np.sqrt(1 / n) + W = np.random.normal(mean, std_dev, size=(m, n)).astype(np.float32) + std_dev = np.sqrt(1 / m) # np.sqrt(2 / (m + 1)) + bt = np.random.normal(mean, std_dev, size=m).astype(np.float32) + # approach 1 + LL.weight.data = torch.tensor(W, requires_grad=True) + LL.bias.data = torch.tensor(bt, requires_grad=True) + # approach 2: constant value ? + layers.append(LL) + + return torch.nn.Sequential(*layers) + From 788dc43b6f13749261dd2d2b1f4de6fb42a4279e Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Fri, 11 Sep 2020 19:10:45 -0700 Subject: [PATCH 41/57] add fb_synthetic data --- dlrm_s_pytorch.py | 15 ++++++++- extend_distributed.py | 18 ++++++---- job.all.sh | 50 ++++++++++++++++++++++++++++ synthetic_data_loader.py | 72 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 148 insertions(+), 7 deletions(-) create mode 100644 job.all.sh create mode 100644 synthetic_data_loader.py diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 2ca6ded9..f8483165 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -96,7 +96,8 @@ import uuid import project -import fb_synthetic_data_pytorch as fb_syn_data +# import fb_synthetic_data_pytorch as fb_syn_data +import synthetic_data_loader as fb_syn_data # from torchviz import make_dot # import torch.nn.functional as Functional @@ -489,6 +490,7 @@ def distributed_forward(self, dense_x, lS_o, lS_i): (_, batch_split_lengths) = ext_dist.get_split_lengths(batch_size) z = ext_dist.all_gather(z, batch_split_lengths) #print("Z: %s" % z) + return z def parallel_forward(self, dense_x, lS_o, lS_i): @@ -1213,6 +1215,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): # print(l.weight.grad.norm().item()) # optimizer + ### ext_dist.barrier() optimizer.step() ### lr_scheduler.step() @@ -1441,8 +1444,18 @@ def loss_fn_wrap(Z, T, use_gpu, device): + " reached, stop training") break + if (ext_dist.my_rank == 0 and should_print): + print("ITER : ", j) + os.system("nvidia-smi") + k += 1 # nepochs + if (ext_dist.my_rank == 0): + print("MEMORY INFO") + print(torch.cuda.memory_allocated(0)) + print(torch.cuda.memory_summary(0)) + os.system("nvidia-smi") + file_prefix = "%s/dlrm_s_pytorch_r%d" % (args.out_dir, ext_dist.my_rank) # profiling if args.enable_profiling: diff --git a/extend_distributed.py b/extend_distributed.py index c995981a..43ec4578 100644 --- a/extend_distributed.py +++ b/extend_distributed.py @@ -9,6 +9,7 @@ except ImportError as e: #print(e) torch_ccl = False +import time my_rank = -1 my_size = -1 @@ -114,7 +115,7 @@ def init_distributed(rank = -1, size = -1, backend=''): if not os.environ.get('MASTER_PORT', None): os.environ['MASTER_PORT'] = '29500' if not os.environ.get('MASTER_ADDR', None): if "SLURM_NODELIST" in os.environ: - master_addr = os.environ["SLURM_NODELIST"].split(',')[0].replace("[", "") + master_addr = os.environ["SLURM_NODELIST"].replace('-', ',').split(',')[0].replace("[", "") elif "HOSTNAME" in os.environ: # handle other cases ? master_addr = os.environ["HOSTNAME"] @@ -137,7 +138,7 @@ def init_distributed(rank = -1, size = -1, backend=''): assert(my_local_rank >= 0) assert(my_local_size >= 0) - print("Check local rank ", my_local_rank, " size ", my_local_size) + print("Check local rank ", my_local_rank, " size ", my_local_size, os.environ["MASTER_ADDR"], os.environ["MASTER_PORT"]) dist.init_process_group(backend, rank=rank, world_size=size) my_rank = dist.get_rank() @@ -152,6 +153,7 @@ def init_distributed(rank = -1, size = -1, backend=''): for i in range(my_size): c[i] = my_rank + i * my_size + t1 = time.time() if (torch.cuda.is_available): dev = torch.device('cuda', my_local_rank) a = a.to(dev) @@ -161,11 +163,13 @@ def init_distributed(rank = -1, size = -1, backend=''): print("alltoall on rank :", my_rank, "a = ", a, " b = ", b) else: dist.all_to_all_single(b, a) + t2 = time.time() + if torch.equal(b, c): alltoall_supported = True - print("All to all single test passed for rank ", my_rank) + print("All to all single test passed for rank ", my_rank, " time ", t2 - t1) else: - print("Failed alltoall single test! for rank= ", my_rank) + print("Failed alltoall single test! for rank= ", my_rank, " time ", t2 - t1) except RuntimeError: pass if a2a_impl == 'alltoall' and alltoall_supported == False: @@ -367,7 +371,7 @@ class All2All_Wait(Function): @staticmethod def forward(ctx, *output): global myreq - #print("All2All_Wait:forward") + # print("All2All_Wait:forward") a2ai = myreq.a2ai ctx.a2ai = a2ai myreq.req.wait() @@ -376,12 +380,13 @@ def forward(ctx, *output): emb_split_lengths = a2ai.emb_split_lengths if a2ai.emb_split_lengths else a2ai.lS * a2ai.lN * a2ai.E outputs = output[0].split(emb_split_lengths) outputs = tuple([out.view([a2ai.lN, -1]) for out in outputs]) + # print("All2All_Wait:forward done") return outputs @staticmethod def backward(ctx, *grad_outputs): global myreq - #print("All2All_Wait:backward") + # print("All2All_Wait:backward") a2ai = ctx.a2ai grad_outputs = [gout.contiguous().view([-1]) for gout in grad_outputs] grad_output = torch.cat(grad_outputs) @@ -389,6 +394,7 @@ def backward(ctx, *grad_outputs): req = dist.all_to_all_single(grad_input, grad_output, a2ai.mb_split_lengths, a2ai.emb_split_lengths, async_op=True) myreq.req = req myreq.tensor = grad_input + # print("All2All_Wait:backward done") return (grad_output,) class AllGather(Function): diff --git a/job.all.sh b/job.all.sh new file mode 100644 index 00000000..38f04355 --- /dev/null +++ b/job.all.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +#SBATCH --job-name=testdlrm #The name you want the job to have +#SBATCH --output=/private/home/hongzhang/tmp/dlrm/output-%j +#SBATCH --error=/private/home/hongzhang/tmp/dlrm/error-%j +#SBATCH --nodes=1 # -C volta32gb #The number of compute nodes to use +#SBATCH --ntasks=8 #The total number of cpu tasks to run +#SBATCH --time=00:40:00 # max time +#SBATCH --exclusive # exclusive nodes +#SBATCH --gres=gpu:volta:8 -C volta32gb +#SBATCH --mem-per-cpu=60GB + +# for mpirun host file +echo $SLURM_NODELIST +echo $SLURM_NODELIST > hostfile1 + +source /private/home/hongzhang/.zshrc +#module purge +#module load anaconda3/2019.07 +#module load cuda/10.1 +#module load cudnn/v7.6.5.32-cuda.10.1 +#module load openmpi/4.0.2/gcc.7.4.0-cuda.10.1 + +#export NCCL_ROOT_DIR=/private/home/hongzhang/codes/nccl/build +#export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$NCCL_ROOT_DIR/lib +#export CUDA_PATH=$CUDA_HOME +#export CUDNN_PATH=$CUDNN_ROOT_DIR +#export MPI_PATH=$MPI_HOME +#export NCCL_PATH=$NCCL_ROOT_DIR + +conda activate mytorch + +which python3 + +# large_arch_emb="2600-2600-2600-2600-2600-2600-2600-2600" +# large_arch_emb="26000000-26000000-26000000-26000000-26000000-26000000-26000000-26000000" +large_arch_emb_usr=$(printf '26000%.0s' {1..815}) +large_arch_emb_usr=${large_arch_emb_usr//"02"/"0-2"} +large_arch_emb_ads=$(printf '14000%.0s' {1..544}) +large_arch_emb_ads=${large_arch_emb_ads//"01"/"0-1"} +large_arch_emb="$large_arch_emb_usr-$large_arch_emb_ads" + +# --hostfile hostfile1 +# random +/public/apps/openmpi/4.0.2/gcc.7.4.0/bin/mpirun -prefix /public/apps/openmpi/4.0.2/gcc.7.4.0/ -v -np 8 python3 dlrm_s_pytorch.py --arch-sparse-feature-size=64 --arch-mlp-bot="2000-1024-1024-1024-1024-1024-1024-1024-1024-1024-1024-512-64" --arch-mlp-top="4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-1" --arch-embedding-size=$large_arch_emb --data-generation=random --loss-function=bce --round-targets=True --learning-rate=0.1 --mini-batch-size=2048 --print-freq=1 --print-time --test-mini-batch-size=10240 --test-num-workers=16 --use-gpu --dist-backend='nccl' --num-indices-per-lookup-fixed=1 --num-indices-per-lookup=30 --num-batches=4 --arch-project-size=30 + +# fb_synthetic +# /public/apps/openmpi/4.0.2/gcc.7.4.0/bin/mpirun -prefix /public/apps/openmpi/4.0.2/gcc.7.4.0/ -v -np 8 python3 dlrm_s_pytorch.py --arch-sparse-feature-size=64 --arch-mlp-bot="2000-1024-1024-1024-1024-1024-1024-1024-1024-1024-1024-512-64" --arch-mlp-top="4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-1" --arch-embedding-size=$large_arch_emb --data-generation=fb_synthetic --loss-function=bce --round-targets=True --learning-rate=0.1 --mini-batch-size=2048 --print-freq=1 --print-time --test-mini-batch-size=10240 --test-num-workers=16 --use-gpu --dist-backend='nccl' --num-indices-per-lookup-fixed=1 --num-indices-per-lookup=30 --num-batches=4 --arch-project-size=30 + +# srun --label /private/home/hongzhang/.conda/envs/mytorch/bin/python3 dlrm_s_pytorch.py --arch-sparse-feature-size=64 --arch-mlp-bot="2000-1024-1024-1024-1024-1024-1024-1024-1024-1024-1024-512-64" --arch-mlp-top="4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-1" --arch-embedding-size=$large_arch_emb --data-generation=random --loss-function=bce --round-targets=True --learning-rate=0.1 --mini-batch-size=128 --print-freq=1 --print-time --test-mini-batch-size=10240 --test-num-workers=16 --use-gpu --dist-backend='nccl' --num-indices-per-lookup-fixed=1 --num-indices-per-lookup=30 --num-batches=4 diff --git a/synthetic_data_loader.py b/synthetic_data_loader.py new file mode 100644 index 00000000..6bb538d6 --- /dev/null +++ b/synthetic_data_loader.py @@ -0,0 +1,72 @@ +import sys +import torch +from torch.utils.data import Dataset + + +def collate_wrapper_random(list_of_tuples): + # where each tuple is (X, lS_o, lS_i, T) + (X, lS_o, lS_i, T) = list_of_tuples[0] + return (X, + torch.stack(lS_o), + lS_i, + T) + + +class RandomDataset(Dataset): + + def __init__( + self, + mini_batch_size, + nbatches=1, + write_data_folder="./synthetic_data/syn_data_bs65536/", + ): + self.write_data_folder = write_data_folder + self.num_batches = nbatches + self.mini_batch_size = mini_batch_size + + self.X = torch.load(f"{self.write_data_folder}/X_0.pt") + self.lS_o = torch.load(f"{self.write_data_folder}/lS_o_0.pt") + self.lS_i = torch.load(f"{self.write_data_folder}/lS_i_0.pt") + self.T = torch.load(f"{self.write_data_folder}/T_0.pt") + # print('data loader initiated ...') + + def __getitem__(self, index): + sInd = index * self.mini_batch_size + eInd = sInd + self.mini_batch_size + if sInd >= len(self.X): + sys.exit(f' mini_batch_size({self.mini_batch_size}) * ' + f'num_batches({self.num_batches}) has to be less' + f' than size of data({len(self.X)})' + ) + X = self.X[sInd:eInd] + lS_o = [i[:][sInd:eInd] - i[:][sInd] for i in self.lS_o] + + if eInd < len(self.lS_o[0]): + lS_i = [val[self.lS_o[ind][sInd]:self.lS_o[ind][eInd]] for ind, val in enumerate(self.lS_i)] + elif sInd < len(self.lS_o[0]): + lS_i = [val[self.lS_o[ind][sInd]:] for ind, val in enumerate(self.lS_i)] + + T = self.T[sInd:eInd] + return (X, lS_o, lS_i, T) + + def __len__(self): + return self.num_batches + + +def make_random_data_and_loader(args, ln_emb, m_den): + + train_data = RandomDataset( + args.mini_batch_size, + nbatches=args.num_batches + ) + train_loader = torch.utils.data.DataLoader( + train_data, + batch_size=1, + shuffle=False, + num_workers=args.num_workers, + collate_fn=collate_wrapper_random, + pin_memory=False, + drop_last=False, + ) + return train_data, train_loader + From ebef57544545676069f88fc3965581f1a5ca511e Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Fri, 25 Sep 2020 13:17:12 -0700 Subject: [PATCH 42/57] Fix hang problem and add README --- README.params | 43 --------------------------------- README.params.md | 51 ++++++++++++++++++++++++++++++++++++++++ dlrm_s_pytorch.py | 24 +++++++++---------- extend_distributed.py | 14 ++++++----- synthetic_data_loader.py | 3 ++- 5 files changed, 73 insertions(+), 62 deletions(-) create mode 100644 README.params.md diff --git a/README.params b/README.params index 6abca3db..b28b04f6 100644 --- a/README.params +++ b/README.params @@ -1,46 +1,3 @@ -This note is intended for branch params_dlrm. The purpose of this branch is to test DLRM with large data models. - -The data is configured with total 1359 embedding tables, including user related 815 tables with hash size 26000000 and -ads related tables with hash size 14000000. You could set up the parameters in the shell script as following: - -large_arch_emb_usr=$(printf '26000000%.0s' {1..815}) -large_arch_emb_usr=${large_arch_emb_usr//"03"/"0-3"} - -large_arch_emb_ads=$(printf '14000000%.0s' {1..544}) -large_arch_emb_ads=${large_arch_emb_ads//"04"/"0-4"} - -large_arch_emb="$large_arch_emb_usr-$large_arch_emb_ads" - -There are a new parameter which do not exist in master branch: ---arch-project-size : reduces the number of interaction features for the dot operation. - This is mainly due to the memory concern. It reduces the memory size needed for top MLP. - -Also, the data-generation supports one more data model "fb_synthetic". - -The model parameter size depends on the mini-batch-size and data types. -Suppose, we set mini-batch-size=2048, data type=float32, -The model size for embedding tables are about 14TB. -The MLP model size are about 8.5GB per copy. - -Here are the command line to run the large model dlrm: -python dlrm_s_pytorch.py - --arch-sparse-feature-size=128 - --arch-mlp-bot="2000-1024-1024-1024-1024-1024-1024-1024-1024-1024-1024-512-128" - --arch-mlp-top="4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-1" - --arch-embedding-size=$large_arch_emb - --data-generation=rando - --loss-function=bce - --round-targets=True - --learning-rate=0.1 - --mini-batch-size=2048 - --print-freq=10240 - --print-time - --test-mini-batch-size=16384 - --test-num-workers=16 - --num-indices-per-lookup-fixed=1 - --num-indices-per-lookup=28 - --arch-projection-size 30 - --use-gpu diff --git a/README.params.md b/README.params.md new file mode 100644 index 00000000..86a43f71 --- /dev/null +++ b/README.params.md @@ -0,0 +1,51 @@ + +# DLRM Distributed Branch + +Extend the PyTorch implementation to run DLRM on multi nodes on distributed platforms. +The distributed version will be needed when data model becomes large. + +It inherents all the parameters from master DLRM implementation. +The distributed version add one more parameter: + +**--dist-backend**: + The backend support for the distributed version. As in torch.distributed package, + it can be "nccl", "mpi", and "gloo". + +In addition, it introduces the following new parameter:: +**--arch-project-size** : + Reducing the number of interaction features for the dot operation. + A project operation is applied to the dotted features to reduce its dimension size. + This is mainly due to the memory concern. It reduces the memory size needed for top MLP. + A side effect is that it may also imrpove the model accuracy. + +## Usage + +Currently, it is launched with mpirun on multi-nodes. The hostfile need to be created or +a host list should be given. The DLRM parameters should be given in the same way as single +node master branch. +```bash +mpirun -np 128 -hostfile hostfile python dlrm_s_pytorch.py ... +``` + +## Example +```bash +python dlrm_s_pytorch.py + --arch-sparse-feature-size=128 + --arch-mlp-bot="2000-1024-1024-128" + --arch-mlp-top="4096-4096-4096-1" + --arch-embedding-size=$large_arch_emb + --data-generation=random + --loss-function=bce + --round-targets=True + --learning-rate=0.1 + --mini-batch-size=2048 + --print-freq=10240 + --print-time + --test-mini-batch-size=16384 + --test-num-workers=16 + --num-indices-per-lookup-fixed=1 + --num-indices-per-lookup=100 + --arch-projection-size 30 + --use-gpu +``` + diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index f8483165..81dd7f65 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -737,19 +737,13 @@ def dash_separated_floats(value): args = parser.parse_args() - print("=== Get Env ===") print(socket.gethostname()) -# myenv = os.environ -# for e in myenv: -# print(e, "=", myenv[e]) - print("=== Done ===") ext_dist.init_distributed(backend=args.dist_backend) - print("success size= ", ext_dist.my_size, ext_dist.my_rank) + # print("success size= ", ext_dist.my_size, ext_dist.my_rank) ext_dist.barrier() - print("passed barrier") if args.mlperf_logging: print('command line args: ', json.dumps(vars(args))) @@ -1139,6 +1133,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): ) ext_dist.barrier() + startTime = time.time() print("time/loss/accuracy (if enabled):") with torch.autograd.profiler.profile(args.enable_profiling, use_gpu, record_shapes=True) as prof: while k < args.nepochs: @@ -1215,7 +1210,6 @@ def loss_fn_wrap(Z, T, use_gpu, device): # print(l.weight.grad.norm().item()) # optimizer - ### ext_dist.barrier() optimizer.step() ### lr_scheduler.step() @@ -1252,7 +1246,8 @@ def loss_fn_wrap(Z, T, use_gpu, device): "Finished {} it {}/{} of epoch {}, {:.2f} ms/it, ".format( str_run_type, j + 1, nbatches, k, gT ) - + "loss {:.6f}, accuracy {:3.3f} %".format(gL, gA * 100) + + "loss {:.6f}, accuracy {:3.3f} % it {} for task {} ".format(gL, + gA * 100, total_iter, ext_dist.my_rank) ) # Uncomment the line below to print out the total time with overhead # print("Accumulated time so far: {}" \ @@ -1445,17 +1440,22 @@ def loss_fn_wrap(Z, T, use_gpu, device): break if (ext_dist.my_rank == 0 and should_print): - print("ITER : ", j) + print("ITER : ", j, " from nvidia-smi") os.system("nvidia-smi") k += 1 # nepochs if (ext_dist.my_rank == 0): - print("MEMORY INFO") - print(torch.cuda.memory_allocated(0)) + # print(torch.cuda.memory_allocated(0)) print(torch.cuda.memory_summary(0)) + # print("from nvidia-smi") os.system("nvidia-smi") + endTime = time.time() + ext_dist.barrier() + print("Process {} Done with time {:.6f} {:.6f}!".format(ext_dist.my_rank, + time.time() - startTime, endTime - startTime), flush=True) + file_prefix = "%s/dlrm_s_pytorch_r%d" % (args.out_dir, ext_dist.my_rank) # profiling if args.enable_profiling: diff --git a/extend_distributed.py b/extend_distributed.py index 43ec4578..fa9985f7 100644 --- a/extend_distributed.py +++ b/extend_distributed.py @@ -160,14 +160,16 @@ def init_distributed(rank = -1, size = -1, backend=''): b = b.to(dev) c = c.to(dev) dist.all_to_all_single(b, a) - print("alltoall on rank :", my_rank, "a = ", a, " b = ", b) + if my_rank == 0: + print("alltoall on rank :", my_rank, "a = ", a, " b = ", b) else: dist.all_to_all_single(b, a) t2 = time.time() if torch.equal(b, c): alltoall_supported = True - print("All to all single test passed for rank ", my_rank, " time ", t2 - t1) + if my_rank == 0: + print("All to all single test passed for rank ", my_rank, " time ", t2 - t1) else: print("Failed alltoall single test! for rank= ", my_rank, " time ", t2 - t1) except RuntimeError: @@ -334,7 +336,7 @@ class All2All_Req(Function): @staticmethod def forward(ctx, a2ai, *inputs): global myreq - #print("All2All_Req:forward") + # print("All2All_Req:forward ", my_rank) mb_split_lengths = a2ai.gNS if mb_split_lengths: mb_split_lengths = [m * a2ai.E for m in mb_split_lengths] emb_split_lengths = a2ai.gSS @@ -356,7 +358,7 @@ def forward(ctx, a2ai, *inputs): @staticmethod def backward(ctx, *grad_output): global myreq - #print("All2All_Req:backward") + # print("All2All_Req:backward ", my_rank) a2ai = ctx.a2ai myreq.req.wait() myreq.req = None @@ -371,7 +373,7 @@ class All2All_Wait(Function): @staticmethod def forward(ctx, *output): global myreq - # print("All2All_Wait:forward") + # print("All2All_Wait:forward ", my_rank) a2ai = myreq.a2ai ctx.a2ai = a2ai myreq.req.wait() @@ -386,7 +388,7 @@ def forward(ctx, *output): @staticmethod def backward(ctx, *grad_outputs): global myreq - # print("All2All_Wait:backward") + # print("All2All_Wait:backward ", my_rank) a2ai = ctx.a2ai grad_outputs = [gout.contiguous().view([-1]) for gout in grad_outputs] grad_output = torch.cat(grad_outputs) diff --git a/synthetic_data_loader.py b/synthetic_data_loader.py index 6bb538d6..58068227 100644 --- a/synthetic_data_loader.py +++ b/synthetic_data_loader.py @@ -18,7 +18,8 @@ def __init__( self, mini_batch_size, nbatches=1, - write_data_folder="./synthetic_data/syn_data_bs65536/", +# write_data_folder="./synthetic_data/syn_data_bs65536/", + write_data_folder="./synthetic_data/syn_data_bs65536_3M_emb_size/", ): self.write_data_folder = write_data_folder self.num_batches = nbatches From b0420b0eff6807e230210bf9be218254fb7f3beb Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Fri, 25 Sep 2020 13:18:35 -0700 Subject: [PATCH 43/57] remove README.param --- README.params | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 README.params diff --git a/README.params b/README.params deleted file mode 100644 index b28b04f6..00000000 --- a/README.params +++ /dev/null @@ -1,3 +0,0 @@ - - - From 38ecd56cbf504848ad82b53756637d2fe91ff8f2 Mon Sep 17 00:00:00 2001 From: "aghaderi@fb.com" Date: Mon, 28 Sep 2020 14:40:37 -0700 Subject: [PATCH 44/57] data module cean-up --- dlrm_data.py | 347 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 347 insertions(+) create mode 100644 dlrm_data.py diff --git a/dlrm_data.py b/dlrm_data.py new file mode 100644 index 00000000..27dfa3d7 --- /dev/null +++ b/dlrm_data.py @@ -0,0 +1,347 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +# +# Description: delivering inputs and targets for the dlrm benchmark +# The inpts and outputs are used according to the following two option(s) +# 1) random distribution, generated and loaded based on uniform distribution +# 2) FB distribution, the synthetic data which already generated based on +# the distribution of the Facabook data would be loaded. + +from __future__ import absolute_import, division, print_function, unicode_literals + +# numpy +import numpy as np +from numpy import random as ra + + +# pytorch +import torch +from torch.utils.data import Dataset # , RandomSampler + + +class RandomDataset(Dataset): + """ Uniform distribution """ + def __init__( + self, + m_den, + ln_emb, + data_size, + num_batches, + mini_batch_size, + num_indices_per_lookup, + num_indices_per_lookup_fixed, + num_targets=1, + round_targets=False, + data_generation="random", + trace_file="", + enable_padding=False, + reset_seed_on_access=False, + rand_seed=0 + ): + # compute batch size + nbatches = int(np.ceil((data_size * 1.0) / mini_batch_size)) + if num_batches != 0: + nbatches = num_batches + data_size = nbatches * mini_batch_size + # print("Total number of batches %d" % nbatches) + + # save args (recompute data_size if needed) + self.m_den = m_den + self.ln_emb = ln_emb + self.data_size = data_size + self.num_batches = nbatches + self.mini_batch_size = mini_batch_size + self.num_indices_per_lookup = num_indices_per_lookup + self.num_indices_per_lookup_fixed = num_indices_per_lookup_fixed + self.num_targets = num_targets + self.round_targets = round_targets + self.data_generation = data_generation + self.trace_file = trace_file + self.enable_padding = enable_padding + self.reset_seed_on_access = reset_seed_on_access + self.rand_seed = rand_seed + + def reset_numpy_seed(self, numpy_rand_seed): + np.random.seed(numpy_rand_seed) + # torch.manual_seed(numpy_rand_seed) + + def __getitem__(self, index): + + if isinstance(index, slice): + return [ + self[idx] for idx in range( + index.start or 0, index.stop or len(self), index.step or 1 + ) + ] + + # WARNING: reset seed on access to first element + # (e.g. if same random samples needed across epochs) + if self.reset_seed_on_access and index == 0: + self.reset_numpy_seed(self.rand_seed) + + # number of data points in a batch + n = min(self.mini_batch_size, self.data_size - (index * self.mini_batch_size)) + + # generate a batch of dense and sparse features + if self.data_generation == "random": + (X, lS_o, lS_i) = generate_uniform_input_batch( + self.m_den, + self.ln_emb, + n, + self.num_indices_per_lookup, + self.num_indices_per_lookup_fixed + ) + elif self.data_generation == "synthetic": + (X, lS_o, lS_i) = generate_synthetic_input_batch( + self.m_den, + self.ln_emb, + n, + self.num_indices_per_lookup, + self.num_indices_per_lookup_fixed, + self.trace_file, + self.enable_padding + ) + else: + sys.exit( + "ERROR: --data-generation=" + self.data_generation + " is not supported" + ) + + # generate a batch of target (probability of a click) + T = generate_random_output_batch(n, self.num_targets, self.round_targets) + + return (X, lS_o, lS_i, T) + + def __len__(self): + # WARNING: note that we produce bacthes of outputs in __getitem__ + # therefore we should use num_batches rather than data_size below + return self.num_batches + + +def collate_wrapper_random(list_of_tuples): + # where each tuple is (X, lS_o, lS_i, T) + (X, lS_o, lS_i, T) = list_of_tuples[0] + return (X, + torch.stack(lS_o), + lS_i, + T) + + +def make_random_data_and_loader(args, ln_emb, m_den): + + train_data = RandomDataset( + m_den, + ln_emb, + args.data_size, + args.num_batches, + args.mini_batch_size, + args.num_indices_per_lookup, + args.num_indices_per_lookup_fixed, + 1, # num_targets + args.round_targets, + args.data_generation, + args.data_trace_file, + args.data_trace_enable_padding, + reset_seed_on_access=True, + rand_seed=args.numpy_rand_seed + ) # WARNING: generates a batch of lookups at once + train_loader = torch.utils.data.DataLoader( + train_data, + batch_size=1, + shuffle=False, + num_workers=args.num_workers, + collate_fn=collate_wrapper_random, + pin_memory=False, + drop_last=False, # True + ) + return train_data, train_loader + + +def generate_random_data( + m_den, + ln_emb, + data_size, + num_batches, + mini_batch_size, + num_indices_per_lookup, + num_indices_per_lookup_fixed, + num_targets=1, + round_targets=False, + data_generation="random", + trace_file="", + enable_padding=False, +): + nbatches = int(np.ceil((data_size * 1.0) / mini_batch_size)) + if num_batches != 0: + nbatches = num_batches + data_size = nbatches * mini_batch_size + # print("Total number of batches %d" % nbatches) + + # inputs + lT = [] + lX = [] + lS_offsets = [] + lS_indices = [] + for j in range(0, nbatches): + # number of data points in a batch + n = min(mini_batch_size, data_size - (j * mini_batch_size)) + + # generate a batch of dense and sparse features + if data_generation == "random": + (Xt, lS_emb_offsets, lS_emb_indices) = generate_uniform_input_batch( + m_den, + ln_emb, + n, + num_indices_per_lookup, + num_indices_per_lookup_fixed + ) + elif data_generation == "synthetic": + (Xt, lS_emb_offsets, lS_emb_indices) = generate_synthetic_input_batch( + m_den, + ln_emb, + n, + num_indices_per_lookup, + num_indices_per_lookup_fixed, + trace_file, + enable_padding + ) + else: + sys.exit( + "ERROR: --data-generation=" + data_generation + " is not supported" + ) + # dense feature + lX.append(Xt) + # sparse feature (sparse indices) + lS_offsets.append(lS_emb_offsets) + lS_indices.append(lS_emb_indices) + + # generate a batch of target (probability of a click) + P = generate_random_output_batch(n, num_targets, round_targets) + lT.append(P) + + return (nbatches, lX, lS_offsets, lS_indices, lT) + + +def generate_random_output_batch(n, num_targets, round_targets=False): + # target (probability of a click) + if round_targets: + P = np.round(ra.rand(n, num_targets).astype(np.float32)).astype(np.float32) + else: + P = ra.rand(n, num_targets).astype(np.float32) + + return torch.tensor(P) + + +# uniform ditribution (input data) +def generate_uniform_input_batch( + m_den, + ln_emb, + n, + num_indices_per_lookup, + num_indices_per_lookup_fixed, +): + # dense feature + #Xt = torch.tensor(ra.rand(n, m_den).astype(np.float32)) + Xt = torch.tensor(ra.rand(1, m_den).astype(np.float32)) + + # sparse feature (sparse indices) + lS_emb_offsets = [] + lS_emb_indices = [] + # for each embedding generate a list of n lookups, + # where each lookup is composed of multiple sparse indices + for size in ln_emb: + lS_batch_offsets = [] + lS_batch_indices = [] + offset = 0 + for _ in range(n): + # num of sparse indices to be used per embedding (between + if num_indices_per_lookup_fixed: + sparse_group_size = np.int64(num_indices_per_lookup) + else: + # random between [1,num_indices_per_lookup]) + r = ra.random(1) + sparse_group_size = np.int64( + np.round(max([1.0], r * min(size, num_indices_per_lookup))) + ) + # sparse indices to be used per embedding + r = ra.random(sparse_group_size) + sparse_group = np.unique(np.round(r * (size - 1)).astype(np.int64)) + # reset sparse_group_size in case some index duplicates were removed + sparse_group_size = np.int64(sparse_group.size) + # store lengths and indices + lS_batch_offsets += [offset] + lS_batch_indices += sparse_group.tolist() + # update offset for next iteration + offset += sparse_group_size + lS_emb_offsets.append(torch.tensor(lS_batch_offsets)) + lS_emb_indices.append(torch.tensor(lS_batch_indices)) + + return (Xt, lS_emb_offsets, lS_emb_indices) + +class SyntheticDataLoader(Dataset): + + def __init__( + self, + mini_batch_size, + nbatches=1, + write_data_folder="./synthetic_data/syn_data_bs65536/", + ): + self.write_data_folder = write_data_folder + self.num_batches = nbatches + self.mini_batch_size = mini_batch_size + + self.X = torch.load(f"{self.write_data_folder}/X_0.pt") + self.lS_o = torch.load(f"{self.write_data_folder}/lS_o_0.pt") + self.lS_i = torch.load(f"{self.write_data_folder}/lS_i_0.pt") + self.T = torch.load(f"{self.write_data_folder}/T_0.pt") + # print('data loader initiated ...') + + def __getitem__(self, index): + sInd = index * self.mini_batch_size + eInd = sInd + self.mini_batch_size + if sInd >= len(self.X): + sys.exit(f' mini_batch_size({self.mini_batch_size}) * ' + f'num_batches({self.num_batches}) has to be less' + f' than size of data({len(self.X)})' + ) + X = self.X[sInd:eInd] + lS_o = [i[:][sInd:eInd] - i[:][sInd] for i in self.lS_o] + + if eInd < len(self.lS_o[0]): + lS_i = [val[self.lS_o[ind][sInd]:self.lS_o[ind][eInd]] for ind, val in enumerate(self.lS_i)] + elif sInd < len(self.lS_o[0]): + lS_i = [val[self.lS_o[ind][sInd]:] for ind, val in enumerate(self.lS_i)] + + T = self.T[sInd:eInd] + return (X, lS_o, lS_i, T) + + def __len__(self): + return self.num_batches + + +def synthetic_data_loader(args, ln_emb, m_den): + + train_data = RandomDataset( + args.mini_batch_size, + nbatches=args.num_batches + ) + train_loader = torch.utils.data.DataLoader( + train_data, + batch_size=1, + shuffle=False, + num_workers=args.num_workers, + collate_fn=collate_wrapper_random, + pin_memory=False, + drop_last=False, + ) + return train_data, train_loader + + +def data_loader(args, ln_emb, m_den): + data_gens = {"random": make_random_data_and_loader, + "fb_synthetic": synthetic_data_loader, + } + train_data, train_ld = data_gens[args.data_generation](args, ln_emb, m_den) + + return train_data, train_ld From 323a593ee87eeca3efed46ffd7407f21b107af0d Mon Sep 17 00:00:00 2001 From: "aghaderi@fb.com" Date: Mon, 28 Sep 2020 14:41:36 -0700 Subject: [PATCH 45/57] data module cean-up --- synthetic_data_loader.py | 73 ---------------------------------------- 1 file changed, 73 deletions(-) delete mode 100644 synthetic_data_loader.py diff --git a/synthetic_data_loader.py b/synthetic_data_loader.py deleted file mode 100644 index 58068227..00000000 --- a/synthetic_data_loader.py +++ /dev/null @@ -1,73 +0,0 @@ -import sys -import torch -from torch.utils.data import Dataset - - -def collate_wrapper_random(list_of_tuples): - # where each tuple is (X, lS_o, lS_i, T) - (X, lS_o, lS_i, T) = list_of_tuples[0] - return (X, - torch.stack(lS_o), - lS_i, - T) - - -class RandomDataset(Dataset): - - def __init__( - self, - mini_batch_size, - nbatches=1, -# write_data_folder="./synthetic_data/syn_data_bs65536/", - write_data_folder="./synthetic_data/syn_data_bs65536_3M_emb_size/", - ): - self.write_data_folder = write_data_folder - self.num_batches = nbatches - self.mini_batch_size = mini_batch_size - - self.X = torch.load(f"{self.write_data_folder}/X_0.pt") - self.lS_o = torch.load(f"{self.write_data_folder}/lS_o_0.pt") - self.lS_i = torch.load(f"{self.write_data_folder}/lS_i_0.pt") - self.T = torch.load(f"{self.write_data_folder}/T_0.pt") - # print('data loader initiated ...') - - def __getitem__(self, index): - sInd = index * self.mini_batch_size - eInd = sInd + self.mini_batch_size - if sInd >= len(self.X): - sys.exit(f' mini_batch_size({self.mini_batch_size}) * ' - f'num_batches({self.num_batches}) has to be less' - f' than size of data({len(self.X)})' - ) - X = self.X[sInd:eInd] - lS_o = [i[:][sInd:eInd] - i[:][sInd] for i in self.lS_o] - - if eInd < len(self.lS_o[0]): - lS_i = [val[self.lS_o[ind][sInd]:self.lS_o[ind][eInd]] for ind, val in enumerate(self.lS_i)] - elif sInd < len(self.lS_o[0]): - lS_i = [val[self.lS_o[ind][sInd]:] for ind, val in enumerate(self.lS_i)] - - T = self.T[sInd:eInd] - return (X, lS_o, lS_i, T) - - def __len__(self): - return self.num_batches - - -def make_random_data_and_loader(args, ln_emb, m_den): - - train_data = RandomDataset( - args.mini_batch_size, - nbatches=args.num_batches - ) - train_loader = torch.utils.data.DataLoader( - train_data, - batch_size=1, - shuffle=False, - num_workers=args.num_workers, - collate_fn=collate_wrapper_random, - pin_memory=False, - drop_last=False, - ) - return train_data, train_loader - From b016326fce55c589e1077750b8f5e6ff0056ea07 Mon Sep 17 00:00:00 2001 From: "aghaderi@fb.com" Date: Tue, 29 Sep 2020 00:15:32 -0700 Subject: [PATCH 46/57] copy dlrm_data.py from PARAM-Bench --- dlrm_data.py | 110 +++++++-------------------------------------------- 1 file changed, 14 insertions(+), 96 deletions(-) diff --git a/dlrm_data.py b/dlrm_data.py index 27dfa3d7..c7c1f140 100644 --- a/dlrm_data.py +++ b/dlrm_data.py @@ -6,17 +6,12 @@ # Description: delivering inputs and targets for the dlrm benchmark # The inpts and outputs are used according to the following two option(s) # 1) random distribution, generated and loaded based on uniform distribution -# 2) FB distribution, the synthetic data which already generated based on -# the distribution of the Facabook data would be loaded. +# 2) synthetic data, the synthetic pre-generated data would be loaded. from __future__ import absolute_import, division, print_function, unicode_literals - -# numpy +import sys import numpy as np from numpy import random as ra - - -# pytorch import torch from torch.utils.data import Dataset # , RandomSampler @@ -93,20 +88,6 @@ def __getitem__(self, index): self.num_indices_per_lookup, self.num_indices_per_lookup_fixed ) - elif self.data_generation == "synthetic": - (X, lS_o, lS_i) = generate_synthetic_input_batch( - self.m_den, - self.ln_emb, - n, - self.num_indices_per_lookup, - self.num_indices_per_lookup_fixed, - self.trace_file, - self.enable_padding - ) - else: - sys.exit( - "ERROR: --data-generation=" + self.data_generation + " is not supported" - ) # generate a batch of target (probability of a click) T = generate_random_output_batch(n, self.num_targets, self.round_targets) @@ -158,71 +139,6 @@ def make_random_data_and_loader(args, ln_emb, m_den): return train_data, train_loader -def generate_random_data( - m_den, - ln_emb, - data_size, - num_batches, - mini_batch_size, - num_indices_per_lookup, - num_indices_per_lookup_fixed, - num_targets=1, - round_targets=False, - data_generation="random", - trace_file="", - enable_padding=False, -): - nbatches = int(np.ceil((data_size * 1.0) / mini_batch_size)) - if num_batches != 0: - nbatches = num_batches - data_size = nbatches * mini_batch_size - # print("Total number of batches %d" % nbatches) - - # inputs - lT = [] - lX = [] - lS_offsets = [] - lS_indices = [] - for j in range(0, nbatches): - # number of data points in a batch - n = min(mini_batch_size, data_size - (j * mini_batch_size)) - - # generate a batch of dense and sparse features - if data_generation == "random": - (Xt, lS_emb_offsets, lS_emb_indices) = generate_uniform_input_batch( - m_den, - ln_emb, - n, - num_indices_per_lookup, - num_indices_per_lookup_fixed - ) - elif data_generation == "synthetic": - (Xt, lS_emb_offsets, lS_emb_indices) = generate_synthetic_input_batch( - m_den, - ln_emb, - n, - num_indices_per_lookup, - num_indices_per_lookup_fixed, - trace_file, - enable_padding - ) - else: - sys.exit( - "ERROR: --data-generation=" + data_generation + " is not supported" - ) - # dense feature - lX.append(Xt) - # sparse feature (sparse indices) - lS_offsets.append(lS_emb_offsets) - lS_indices.append(lS_emb_indices) - - # generate a batch of target (probability of a click) - P = generate_random_output_batch(n, num_targets, round_targets) - lT.append(P) - - return (nbatches, lX, lS_offsets, lS_indices, lT) - - def generate_random_output_batch(n, num_targets, round_targets=False): # target (probability of a click) if round_targets: @@ -279,22 +195,23 @@ def generate_uniform_input_batch( return (Xt, lS_emb_offsets, lS_emb_indices) -class SyntheticDataLoader(Dataset): + +class SyntheticDataset(Dataset): def __init__( self, mini_batch_size, nbatches=1, - write_data_folder="./synthetic_data/syn_data_bs65536/", + synthetic_data_folder="./synthetic_data/syn_data_bs65536/", ): - self.write_data_folder = write_data_folder + self.synthetic_data_folder = synthetic_data_folder self.num_batches = nbatches self.mini_batch_size = mini_batch_size - self.X = torch.load(f"{self.write_data_folder}/X_0.pt") - self.lS_o = torch.load(f"{self.write_data_folder}/lS_o_0.pt") - self.lS_i = torch.load(f"{self.write_data_folder}/lS_i_0.pt") - self.T = torch.load(f"{self.write_data_folder}/T_0.pt") + self.X = torch.load(f"{self.synthetic_data_folder}/X_0.pt") + self.lS_o = torch.load(f"{self.synthetic_data_folder}/lS_o_0.pt") + self.lS_i = torch.load(f"{self.synthetic_data_folder}/lS_i_0.pt") + self.T = torch.load(f"{self.synthetic_data_folder}/T_0.pt") # print('data loader initiated ...') def __getitem__(self, index): @@ -322,9 +239,10 @@ def __len__(self): def synthetic_data_loader(args, ln_emb, m_den): - train_data = RandomDataset( + train_data = SyntheticDataset( args.mini_batch_size, - nbatches=args.num_batches + nbatches=args.num_batches, + synthetic_data_folder=args.synthetic_data_folder, ) train_loader = torch.utils.data.DataLoader( train_data, @@ -340,7 +258,7 @@ def synthetic_data_loader(args, ln_emb, m_den): def data_loader(args, ln_emb, m_den): data_gens = {"random": make_random_data_and_loader, - "fb_synthetic": synthetic_data_loader, + "synthetic": synthetic_data_loader, } train_data, train_ld = data_gens[args.data_generation](args, ln_emb, m_den) From 601ad2e6b6451e26681818bfde9974364ccbe22c Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Tue, 27 Oct 2020 13:41:34 -0700 Subject: [PATCH 47/57] update project file --- dlrm_s_pytorch.py | 5 +++-- project.py | 37 ++++++++++++++++++++++++++++++++----- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 81dd7f65..a9cee1db 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -97,7 +97,7 @@ import project # import fb_synthetic_data_pytorch as fb_syn_data -import synthetic_data_loader as fb_syn_data +# import synthetic_data_loader as fb_syn_data # from torchviz import make_dot # import torch.nn.functional as Functional @@ -356,7 +356,7 @@ def interact_features(self, x, ly): T = torch.cat([x] + ly, dim=1).view((batch_size, -1, d)) # perform a dot product if (self.proj_size > 0): - R = project.project(T, self.proj_size, batch_size, d, x, self.proj_l) + R = project.project(T, x, self.proj_l) #TT = torch.transpose(T, 1, 2) #TS = torch.reshape(TT, (-1, TT.size(2))) #TC = self.apply_mlp(TS, self.proj_l) @@ -778,6 +778,7 @@ def dash_separated_floats(value): else: device = torch.device("cuda", 0) ngpus = torch.cuda.device_count() # 1 + ngpus=1 print("Using {} GPU(s)...".format(ngpus)) else: device = torch.device("cpu") diff --git a/project.py b/project.py index cc0ad0a8..b83fa85e 100644 --- a/project.py +++ b/project.py @@ -1,21 +1,48 @@ +# This feature can be used to reduce the memory size consumed by the feature layer of the top MLP. +# Suppose we have n sparse features, each sparse features is represented by an embedding of size d, +# then, we can represent the sparse embeddings by a matrix X = (n, d). The dot product between sparse +# features is X(X^T), which is a symmetric matrix of (n, n) and will be fed into the top MLP. +# Actually We only need the upper or lower traingles to eliminate duplication. If n is large, +# such as, n = 1000, then the number of dot features fed into the MLP will be n^2/2 = 50,000. +# Considering the layer size 4096, the weight parameters will be a matrix (n^2/2, 4096), which +# may consume a large amount of precious memory resources. + +# To reduce the number of dot features, we introduce a parameter called arch-projec-size (k) to compress +# the embeddings. We introduce a parameter matrix Y = (n, k) to compute the weighted sum of the +# dot features. The compressed embeddings is represented by (X^T)Y. Then, we compute the compressed dot +# features by X(X^T)Y = (n, k). Therefore, we can reduce the dot features fed into MLP from n*n/2 +# to n*k. + import sys import torch import torch.nn as nn import numpy as np -def project(T, project_size, batch_size, d, x, layer): +""" +Compute the projected dot features +T: (batch_size, n, d), batched raw embeddings +x: dense features +proj_layer: the projection layer created by create_proj +""" +def project(T, x, proj_layer): TT = torch.transpose(T, 1, 2) - TS = torch.reshape(TT, (-1, TT.size(2))) - TC = layer(TS) - TR = torch.reshape(TC, (-1, d, project_size)) + # TS = torch.reshape(TT, (-1, TT.size(2))) + # TC = proj_layer(TS) + # TR = torch.reshape(TC, (-1, T.shape[2], k)) + TR = proj_layer(TT) Z = torch.bmm(T, TR) - Zflat = Z.view((batch_size, -1)) + Zflat = Z.view((T.shape[0], -1)) R = torch.cat([x] + [Zflat], dim=1) return R +""" +Create the project layer +n: number of sparse features +m: projection size +""" def create_proj(n, m): # build MLP layer by layer layers = nn.ModuleList() From e1e2ca509b7e4d9528f8b9591aea07d8b357e3da Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Tue, 10 Nov 2020 17:52:43 -0800 Subject: [PATCH 48/57] add boundary check for dlrm_data --- dlrm_data.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dlrm_data.py b/dlrm_data.py index c7c1f140..317d2c7d 100644 --- a/dlrm_data.py +++ b/dlrm_data.py @@ -201,12 +201,14 @@ class SyntheticDataset(Dataset): def __init__( self, mini_batch_size, + ln_emb, nbatches=1, synthetic_data_folder="./synthetic_data/syn_data_bs65536/", ): self.synthetic_data_folder = synthetic_data_folder self.num_batches = nbatches self.mini_batch_size = mini_batch_size + self.ln_emb = ln_emb self.X = torch.load(f"{self.synthetic_data_folder}/X_0.pt") self.lS_o = torch.load(f"{self.synthetic_data_folder}/lS_o_0.pt") @@ -229,7 +231,11 @@ def __getitem__(self, index): lS_i = [val[self.lS_o[ind][sInd]:self.lS_o[ind][eInd]] for ind, val in enumerate(self.lS_i)] elif sInd < len(self.lS_o[0]): lS_i = [val[self.lS_o[ind][sInd]:] for ind, val in enumerate(self.lS_i)] - + for i in range(len(lS_i)): + bound = self.ln_emb[i] + if not bound == 26000000: + lS_i[i] %= bound + T = self.T[sInd:eInd] return (X, lS_o, lS_i, T) @@ -241,6 +247,7 @@ def synthetic_data_loader(args, ln_emb, m_den): train_data = SyntheticDataset( args.mini_batch_size, + ln_emb, nbatches=args.num_batches, synthetic_data_folder=args.synthetic_data_folder, ) From 0eb05fde5ade3501dcb460fae9e65799e01971f9 Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Thu, 19 Nov 2020 13:50:19 -0800 Subject: [PATCH 49/57] usse synthetic data --- dlrm_data.py | 2 +- dlrm_s_pytorch.py | 11 ++++++----- extend_distributed.py | 2 +- job.all.sh | 8 ++++---- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/dlrm_data.py b/dlrm_data.py index 317d2c7d..22cc3a1b 100644 --- a/dlrm_data.py +++ b/dlrm_data.py @@ -159,7 +159,7 @@ def generate_uniform_input_batch( ): # dense feature #Xt = torch.tensor(ra.rand(n, m_den).astype(np.float32)) - Xt = torch.tensor(ra.rand(1, m_den).astype(np.float32)) + Xt = torch.tensor(ra.rand(n, m_den).astype(np.float32)) # sparse feature (sparse indices) lS_emb_offsets = [] diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index a9cee1db..0727a6c6 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -96,7 +96,7 @@ import uuid import project -# import fb_synthetic_data_pytorch as fb_syn_data +import dlrm_data as dd # import synthetic_data_loader as fb_syn_data # from torchviz import make_dot @@ -673,7 +673,8 @@ def dash_separated_floats(value): parser.add_argument( "--data-generation", type=str, default="random" ) # synthetic or dataset - + parser.add_argument("--synthetic-data-folder", type=str, + default="./synthetic_data/syn_data_bs65536") # add Gaussian distribution parser.add_argument("--rand-data-dist", type=str, default="uniform") # uniform or gaussian parser.add_argument("--rand-data-min", type=float, default=0) @@ -804,11 +805,11 @@ def dash_separated_floats(value): m_den = train_data.m_den ln_bot[0] = m_den - elif args.data_generation == "fb_synthetic": + elif args.data_generation == "synthetic": # input and target at random ln_emb = np.fromstring(args.arch_embedding_size, dtype=int, sep="-") m_den = ln_bot[0] - train_data, train_ld = fb_syn_data.make_random_data_and_loader(args, ln_emb, m_den) + train_data, train_ld = dd.data_loader(args, ln_emb, m_den) nbatches = args.num_batches if args.num_batches > 0 else len(train_ld) table_feature_map = None # {idx : idx for idx in range(len(ln_emb))} @@ -816,7 +817,7 @@ def dash_separated_floats(value): # input and target at random ln_emb = np.fromstring(args.arch_embedding_size, dtype=int, sep="-") m_den = ln_bot[0] - train_data, train_ld = dp.make_random_data_and_loader(args, ln_emb, m_den) + train_data, train_ld = dd.make_random_data_and_loader(args, ln_emb, m_den) nbatches = args.num_batches if args.num_batches > 0 else len(train_ld) ### parse command line arguments ### diff --git a/extend_distributed.py b/extend_distributed.py index fa9985f7..312521f8 100644 --- a/extend_distributed.py +++ b/extend_distributed.py @@ -138,7 +138,7 @@ def init_distributed(rank = -1, size = -1, backend=''): assert(my_local_rank >= 0) assert(my_local_size >= 0) - print("Check local rank ", my_local_rank, " size ", my_local_size, os.environ["MASTER_ADDR"], os.environ["MASTER_PORT"]) + print("Check local rank ", my_local_rank, " size ", my_local_size, " global rank ", rank, " global size ", size, os.environ["MASTER_ADDR"], os.environ["MASTER_PORT"]) dist.init_process_group(backend, rank=rank, world_size=size) my_rank = dist.get_rank() diff --git a/job.all.sh b/job.all.sh index 38f04355..cf56aaca 100644 --- a/job.all.sh +++ b/job.all.sh @@ -34,17 +34,17 @@ which python3 # large_arch_emb="2600-2600-2600-2600-2600-2600-2600-2600" # large_arch_emb="26000000-26000000-26000000-26000000-26000000-26000000-26000000-26000000" -large_arch_emb_usr=$(printf '26000%.0s' {1..815}) +large_arch_emb_usr=$(printf '260%.0s' {1..815}) large_arch_emb_usr=${large_arch_emb_usr//"02"/"0-2"} -large_arch_emb_ads=$(printf '14000%.0s' {1..544}) +large_arch_emb_ads=$(printf '140%.0s' {1..544}) large_arch_emb_ads=${large_arch_emb_ads//"01"/"0-1"} large_arch_emb="$large_arch_emb_usr-$large_arch_emb_ads" # --hostfile hostfile1 # random -/public/apps/openmpi/4.0.2/gcc.7.4.0/bin/mpirun -prefix /public/apps/openmpi/4.0.2/gcc.7.4.0/ -v -np 8 python3 dlrm_s_pytorch.py --arch-sparse-feature-size=64 --arch-mlp-bot="2000-1024-1024-1024-1024-1024-1024-1024-1024-1024-1024-512-64" --arch-mlp-top="4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-1" --arch-embedding-size=$large_arch_emb --data-generation=random --loss-function=bce --round-targets=True --learning-rate=0.1 --mini-batch-size=2048 --print-freq=1 --print-time --test-mini-batch-size=10240 --test-num-workers=16 --use-gpu --dist-backend='nccl' --num-indices-per-lookup-fixed=1 --num-indices-per-lookup=30 --num-batches=4 --arch-project-size=30 +# /public/apps/openmpi/4.0.2/gcc.7.4.0/bin/mpirun -prefix /public/apps/openmpi/4.0.2/gcc.7.4.0/ -v -np 8 python3 dlrm_s_pytorch.py --arch-sparse-feature-size=64 --arch-mlp-bot="2000-1024-1024-1024-1024-1024-1024-1024-1024-1024-1024-512-64" --arch-mlp-top="4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-1" --arch-embedding-size=$large_arch_emb --data-generation=random --loss-function=bce --round-targets=True --learning-rate=0.1 --mini-batch-size=2048 --print-freq=1 --print-time --test-mini-batch-size=10240 --test-num-workers=16 --use-gpu --dist-backend='nccl' --num-indices-per-lookup-fixed=1 --num-indices-per-lookup=30 --num-batches=4 --arch-project-size=30 # fb_synthetic -# /public/apps/openmpi/4.0.2/gcc.7.4.0/bin/mpirun -prefix /public/apps/openmpi/4.0.2/gcc.7.4.0/ -v -np 8 python3 dlrm_s_pytorch.py --arch-sparse-feature-size=64 --arch-mlp-bot="2000-1024-1024-1024-1024-1024-1024-1024-1024-1024-1024-512-64" --arch-mlp-top="4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-1" --arch-embedding-size=$large_arch_emb --data-generation=fb_synthetic --loss-function=bce --round-targets=True --learning-rate=0.1 --mini-batch-size=2048 --print-freq=1 --print-time --test-mini-batch-size=10240 --test-num-workers=16 --use-gpu --dist-backend='nccl' --num-indices-per-lookup-fixed=1 --num-indices-per-lookup=30 --num-batches=4 --arch-project-size=30 +/public/apps/openmpi/4.0.2/gcc.7.4.0/bin/mpirun -prefix /public/apps/openmpi/4.0.2/gcc.7.4.0/ -v -np 8 python3 dlrm_s_pytorch.py --arch-sparse-feature-size=64 --arch-mlp-bot="2000-1024-1024-1024-1024-1024-1024-1024-1024-1024-1024-512-64" --arch-mlp-top="4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-1" --arch-embedding-size=$large_arch_emb --data-generation=synthetic --loss-function=bce --round-targets=True --learning-rate=0.1 --mini-batch-size=2048 --print-freq=1 --print-time --test-mini-batch-size=10240 --test-num-workers=16 --use-gpu --dist-backend='nccl' --num-indices-per-lookup-fixed=1 --num-indices-per-lookup=28 --num-batches=4 --arch-project-size=30 # srun --label /private/home/hongzhang/.conda/envs/mytorch/bin/python3 dlrm_s_pytorch.py --arch-sparse-feature-size=64 --arch-mlp-bot="2000-1024-1024-1024-1024-1024-1024-1024-1024-1024-1024-512-64" --arch-mlp-top="4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-4096-1" --arch-embedding-size=$large_arch_emb --data-generation=random --loss-function=bce --round-targets=True --learning-rate=0.1 --mini-batch-size=128 --print-freq=1 --print-time --test-mini-batch-size=10240 --test-num-workers=16 --use-gpu --dist-backend='nccl' --num-indices-per-lookup-fixed=1 --num-indices-per-lookup=30 --num-batches=4 From a755b01786083edac1b78b114dc3059ebb3760bb Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Mon, 30 Nov 2020 03:03:39 -0800 Subject: [PATCH 50/57] modify time computation method --- dlrm_s_pytorch.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 0727a6c6..1f70aa71 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -1136,6 +1136,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): ext_dist.barrier() startTime = time.time() + skipped = 0 print("time/loss/accuracy (if enabled):") with torch.autograd.profiler.profile(args.enable_profiling, use_gpu, record_shapes=True) as prof: while k < args.nepochs: @@ -1154,6 +1155,11 @@ def loss_fn_wrap(Z, T, use_gpu, device): if j < skip_upto_batch: continue + if (skipped == 2): + ext_dist.barrier() + startTime = time.time() + skipped = skipped + 1 + if args.mlperf_logging: current_time = time_wrap(use_gpu) if previous_iteration_time: @@ -1453,10 +1459,13 @@ def loss_fn_wrap(Z, T, use_gpu, device): # print("from nvidia-smi") os.system("nvidia-smi") - endTime = time.time() + endTime = time.time() - startTime ext_dist.barrier() - print("Process {} Done with time {:.6f} {:.6f}!".format(ext_dist.my_rank, - time.time() - startTime, endTime - startTime), flush=True) + finalTime = time.time() - startTime + if (skipped > 2): + skipped -= 2 + ext_dist.orig_print("Process {} Done with time {:.6f}s {:.6f}s, iter {:.1f}ms {:.1f}ms steps {}".format(ext_dist.my_rank, + finalTime, endTime, finalTime*1000.0/skipped, endTime*1000.0/skipped, skipped), flush=True) file_prefix = "%s/dlrm_s_pytorch_r%d" % (args.out_dir, ext_dist.my_rank) # profiling From 40dddb7aaa2000ea02e3c5912dec2038ac94b81d Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Tue, 1 Dec 2020 16:23:37 -0800 Subject: [PATCH 51/57] change output + turn off nvidia-smi + reuse syn data --- dlrm_data.py | 8 ++++--- dlrm_s_pytorch.py | 59 +++++++++++++++++++++++++---------------------- 2 files changed, 37 insertions(+), 30 deletions(-) diff --git a/dlrm_data.py b/dlrm_data.py index 22cc3a1b..00b37679 100644 --- a/dlrm_data.py +++ b/dlrm_data.py @@ -217,12 +217,14 @@ def __init__( # print('data loader initiated ...') def __getitem__(self, index): + # module out index for reuse + index = index % (len(self.X) // self.mini_batch_size) sInd = index * self.mini_batch_size eInd = sInd + self.mini_batch_size if sInd >= len(self.X): sys.exit(f' mini_batch_size({self.mini_batch_size}) * ' - f'num_batches({self.num_batches}) has to be less' - f' than size of data({len(self.X)})' + f'num_batches({self.num_batches}) has to be less' + f' than size of data({len(self.X)})' ) X = self.X[sInd:eInd] lS_o = [i[:][sInd:eInd] - i[:][sInd] for i in self.lS_o] @@ -235,7 +237,7 @@ def __getitem__(self, index): bound = self.ln_emb[i] if not bound == 26000000: lS_i[i] %= bound - + T = self.T[sInd:eInd] return (X, lS_o, lS_i, T) diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index 1f70aa71..abe7cc38 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -95,6 +95,7 @@ import uuid import project +from torch.nn.parallel import DistributedDataParallel as DDP import dlrm_data as dd # import synthetic_data_loader as fb_syn_data @@ -484,7 +485,7 @@ def distributed_forward(self, dense_x, lS_o, lS_i): z = p ### gather the distributed results on each rank ### - # For some reason it requires explicit sync before all_gather call if + # For some reason it requires explicit sync before all_gather call if # tensor is on GPU memory if z.is_cuda: torch.cuda.synchronize() (_, batch_split_lengths) = ext_dist.get_split_lengths(batch_size) @@ -492,7 +493,7 @@ def distributed_forward(self, dense_x, lS_o, lS_i): #print("Z: %s" % z) return z - + def parallel_forward(self, dense_x, lS_o, lS_i): ### prepare model (overwrite) ### # WARNING: # of devices must be >= batch size in parallel_forward call @@ -674,7 +675,7 @@ def dash_separated_floats(value): "--data-generation", type=str, default="random" ) # synthetic or dataset parser.add_argument("--synthetic-data-folder", type=str, - default="./synthetic_data/syn_data_bs65536") + default="./synthetic_data/syn_data_bs65536") # add Gaussian distribution parser.add_argument("--rand-data-dist", type=str, default="uniform") # uniform or gaussian parser.add_argument("--rand-data-min", type=float, default=0) @@ -804,7 +805,7 @@ def dash_separated_floats(value): ))) m_den = train_data.m_den ln_bot[0] = m_den - + elif args.data_generation == "synthetic": # input and target at random ln_emb = np.fromstring(args.arch_embedding_size, dtype=int, sep="-") @@ -987,15 +988,15 @@ def dash_separated_floats(value): dlrm = dlrm.to(device) # .cuda() if dlrm.ndevices > 1: dlrm.emb_l = dlrm.create_emb(m_spa, ln_emb) - + if ext_dist.my_size > 1: if use_gpu: device_ids = [ext_dist.my_local_rank] - dlrm.bot_l = ext_dist.DDP(dlrm.bot_l, device_ids=device_ids) - dlrm.top_l = ext_dist.DDP(dlrm.top_l, device_ids=device_ids) + dlrm.bot_l = DDP(dlrm.bot_l, device_ids=device_ids) + dlrm.top_l = DDP(dlrm.top_l, device_ids=device_ids) else: - dlrm.bot_l = ext_dist.DDP(dlrm.bot_l) - dlrm.top_l = ext_dist.DDP(dlrm.top_l) + dlrm.bot_l = DDP(dlrm.bot_l) + dlrm.top_l = DDP(dlrm.top_l) # specify the loss function if args.loss_function == "mse": @@ -1158,6 +1159,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): if (skipped == 2): ext_dist.barrier() startTime = time.time() + ext_dist.orig_print("ORIG TIME: ", startTime, accum_time_begin, startTime - accum_time_begin, " for process ", ext_dist.my_rank) skipped = skipped + 1 if args.mlperf_logging: @@ -1254,12 +1256,14 @@ def loss_fn_wrap(Z, T, use_gpu, device): "Finished {} it {}/{} of epoch {}, {:.2f} ms/it, ".format( str_run_type, j + 1, nbatches, k, gT ) - + "loss {:.6f}, accuracy {:3.3f} % it {} for task {} ".format(gL, + + "loss {:.6f}, accuracy {:3.3f} % it {} for task {} ".format(gL, gA * 100, total_iter, ext_dist.my_rank) ) # Uncomment the line below to print out the total time with overhead - # print("Accumulated time so far: {}" \ - # .format(time_wrap(use_gpu) - accum_time_begin)) + if ext_dist.my_rank < 0: + tt1 = time_wrap(use_gpu) + ext_dist.orig_print("Accumulated time so far: {} for process {} for step {} at {}" \ + .format(tt1 - accum_time_begin, ext_dist.my_rank, skipped, tt1)) total_iter = 0 total_samp = 0 @@ -1447,25 +1451,26 @@ def loss_fn_wrap(Z, T, use_gpu, device): + " reached, stop training") break - if (ext_dist.my_rank == 0 and should_print): - print("ITER : ", j, " from nvidia-smi") - os.system("nvidia-smi") - + #if (ext_dist.my_rank == 0 and should_print): + # print("ITER : ", j, " from nvidia-smi") + # os.system("nvidia-smi") + k += 1 # nepochs - if (ext_dist.my_rank == 0): - # print(torch.cuda.memory_allocated(0)) - print(torch.cuda.memory_summary(0)) - # print("from nvidia-smi") - os.system("nvidia-smi") - - endTime = time.time() - startTime + #if (ext_dist.my_rank == 0): + # # print(torch.cuda.memory_allocated(0)) + # print(torch.cuda.memory_summary(0)) + # # print("from nvidia-smi") + # os.system("nvidia-smi") + + tt2 = time.time() + endTime = tt2 - startTime ext_dist.barrier() finalTime = time.time() - startTime if (skipped > 2): skipped -= 2 - ext_dist.orig_print("Process {} Done with time {:.6f}s {:.6f}s, iter {:.1f}ms {:.1f}ms steps {}".format(ext_dist.my_rank, - finalTime, endTime, finalTime*1000.0/skipped, endTime*1000.0/skipped, skipped), flush=True) + ext_dist.orig_print("Process {} Done with time {:.6f}s {:.6f}s, iter {:.1f}ms {:.1f}ms steps {} {}".format(ext_dist.my_rank, + finalTime, endTime, finalTime*1000.0/skipped, endTime*1000.0/skipped, skipped, tt2), flush=True) file_prefix = "%s/dlrm_s_pytorch_r%d" % (args.out_dir, ext_dist.my_rank) # profiling @@ -1473,8 +1478,8 @@ def loss_fn_wrap(Z, T, use_gpu, device): os.makedirs(args.out_dir, exist_ok=True) with open("TT"+str(uuid.uuid4().hex), "w") as prof_f: prof_f.write(prof.key_averages(group_by_input_shape=True).table( - sort_by="self_cpu_time_total", - )) + sort_by="self_cpu_time_total", + )) # with open("%s.prof" % file_prefix, "w") as prof_f: # prof_f.write(prof.key_averages().table(sort_by="cpu_time_total")) From 64b73557ac974642d9661e682483bab3d27dfa93 Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Thu, 10 Dec 2020 03:40:37 -0800 Subject: [PATCH 52/57] start to change input --- dlrm_data.py | 1 + dlrm_s_pytorch.py | 11 +++- extend_distributed.py | 11 ++++ profile.py | 127 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 profile.py diff --git a/dlrm_data.py b/dlrm_data.py index 00b37679..7e8264a4 100644 --- a/dlrm_data.py +++ b/dlrm_data.py @@ -135,6 +135,7 @@ def make_random_data_and_loader(args, ln_emb, m_den): collate_fn=collate_wrapper_random, pin_memory=False, drop_last=False, # True + # persistent_workers=True, ) return train_data, train_loader diff --git a/dlrm_s_pytorch.py b/dlrm_s_pytorch.py index abe7cc38..5129a39b 100644 --- a/dlrm_s_pytorch.py +++ b/dlrm_s_pytorch.py @@ -98,6 +98,7 @@ from torch.nn.parallel import DistributedDataParallel as DDP import dlrm_data as dd + # import synthetic_data_loader as fb_syn_data # from torchviz import make_dot @@ -1137,7 +1138,9 @@ def loss_fn_wrap(Z, T, use_gpu, device): ext_dist.barrier() startTime = time.time() + startTime0 = startTime skipped = 0 + print("time/loss/accuracy (if enabled):") with torch.autograd.profiler.profile(args.enable_profiling, use_gpu, record_shapes=True) as prof: while k < args.nepochs: @@ -1260,7 +1263,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): gA * 100, total_iter, ext_dist.my_rank) ) # Uncomment the line below to print out the total time with overhead - if ext_dist.my_rank < 0: + if ext_dist.my_rank < 2: tt1 = time_wrap(use_gpu) ext_dist.orig_print("Accumulated time so far: {} for process {} for step {} at {}" \ .format(tt1 - accum_time_begin, ext_dist.my_rank, skipped, tt1)) @@ -1466,10 +1469,12 @@ def loss_fn_wrap(Z, T, use_gpu, device): tt2 = time.time() endTime = tt2 - startTime ext_dist.barrier() - finalTime = time.time() - startTime + tt3 = time.time() + finalTime = tt3 - startTime if (skipped > 2): skipped -= 2 - ext_dist.orig_print("Process {} Done with time {:.6f}s {:.6f}s, iter {:.1f}ms {:.1f}ms steps {} {}".format(ext_dist.my_rank, + ext_dist.orig_print("Process {} Done with total time {:.6f} measure time {:.6f}s {:.6f}s, \ + iter {:.1f}ms {:.1f}ms steps {} {}".format(ext_dist.my_rank, tt3 - startTime0, finalTime, endTime, finalTime*1000.0/skipped, endTime*1000.0/skipped, skipped, tt2), flush=True) file_prefix = "%s/dlrm_s_pytorch_r%d" % (args.out_dir, ext_dist.my_rank) diff --git a/extend_distributed.py b/extend_distributed.py index 312521f8..e816654d 100644 --- a/extend_distributed.py +++ b/extend_distributed.py @@ -4,6 +4,9 @@ from torch.autograd import Function from torch.nn.parallel import DistributedDataParallel as DDP import torch.distributed as dist + +import profile as tm + try: import torch_ccl except ImportError as e: @@ -337,6 +340,7 @@ class All2All_Req(Function): def forward(ctx, a2ai, *inputs): global myreq # print("All2All_Req:forward ", my_rank) + tm.tmA2A10.start() mb_split_lengths = a2ai.gNS if mb_split_lengths: mb_split_lengths = [m * a2ai.E for m in mb_split_lengths] emb_split_lengths = a2ai.gSS @@ -353,12 +357,14 @@ def forward(ctx, a2ai, *inputs): a2ai.emb_split_lengths = emb_split_lengths myreq.a2ai = a2ai ctx.a2ai = a2ai + tm.tmA2A10.stop() return myreq.tensor @staticmethod def backward(ctx, *grad_output): global myreq # print("All2All_Req:backward ", my_rank) + tm.tmA2A12.start() a2ai = ctx.a2ai myreq.req.wait() myreq.req = None @@ -366,6 +372,7 @@ def backward(ctx, *grad_output): grad_inputs = grad_input.view([a2ai.N, -1]).split(a2ai.E, dim=1) grad_inputs = [gin.contiguous() for gin in grad_inputs] myreq.tensor = None + tm.tmA2A12.stop() return (None, *grad_inputs) @@ -374,6 +381,7 @@ class All2All_Wait(Function): def forward(ctx, *output): global myreq # print("All2All_Wait:forward ", my_rank) + tm.tmA2A11.start() a2ai = myreq.a2ai ctx.a2ai = a2ai myreq.req.wait() @@ -382,6 +390,7 @@ def forward(ctx, *output): emb_split_lengths = a2ai.emb_split_lengths if a2ai.emb_split_lengths else a2ai.lS * a2ai.lN * a2ai.E outputs = output[0].split(emb_split_lengths) outputs = tuple([out.view([a2ai.lN, -1]) for out in outputs]) + tm.tmA2A11.stop() # print("All2All_Wait:forward done") return outputs @@ -389,6 +398,7 @@ def forward(ctx, *output): def backward(ctx, *grad_outputs): global myreq # print("All2All_Wait:backward ", my_rank) + tm.tmA2A13.start() a2ai = ctx.a2ai grad_outputs = [gout.contiguous().view([-1]) for gout in grad_outputs] grad_output = torch.cat(grad_outputs) @@ -396,6 +406,7 @@ def backward(ctx, *grad_outputs): req = dist.all_to_all_single(grad_input, grad_output, a2ai.mb_split_lengths, a2ai.emb_split_lengths, async_op=True) myreq.req = req myreq.tensor = grad_input + tm.tmA2A13.stop() # print("All2All_Wait:backward done") return (grad_output,) diff --git a/profile.py b/profile.py new file mode 100644 index 00000000..0df3eff6 --- /dev/null +++ b/profile.py @@ -0,0 +1,127 @@ +# Add some self profiling information +# Allow nested timer exists + +import time + +class TimerError(Exception): + """Exception in ProfTimer class""" + +class ProfTimer: + def __init__(self, timername="Timer for DLRM Activity"): + self._name = timername + self._start = 0.0 + self._count = 0 + self._elapsed = 0.0 + + def start(self): + """Start a new timer""" + self._start = time.perf_counter() + + def stop(self): + if self._start == 0.0: + raise TimerError(f"Timer is not running.") + self._elapsed += time.perf_counter() - self._start + self._count += 1 + self._start = 0.0 + + def count(self): + return _self._count + + def reset(self): + self._elapsed = 0.0 + self._count = 0 + + def elapsed(self): + return self._elapsed + + def output(self, level): + if level == 0: + print(f"{self._name }: {self._elapsed:0.6f} seconds with counts {self._count}") + else: + print(f" {self._name }: {self._elapsed:0.6f} seconds with counts {self._count}") + +alltimers = [] +tmGetData = ProfTimer("GetData") +tmFwd = ProfTimer("Forword") +tmLoss = ProfTimer("Loss ") +tmZero = ProfTimer("Zero ") +tmBwd = ProfTimer("Backwrd") +tmOpt = ProfTimer("Opt ") + +tmH2D = ProfTimer("CopyH2D") +tmEmb = ProfTimer("EMB ") +tmA2A = ProfTimer("All2All") +tmA2A1 = ProfTimer("All2All1") +tmBot = ProfTimer("Bottom ") +tmInt = ProfTimer("Inter ") +tmTop = ProfTimer("Top MLP") +tmAllGa = ProfTimer("Allgath") + +tmA2A10 = ProfTimer("All2All10") +tmA2A11 = ProfTimer("All2All11") +tmA2A12 = ProfTimer("All2All12") +tmA2A13 = ProfTimer("All2All13") + +def tmClear(): + + tmGetData.reset() + tmFwd.reset() + tmLoss.reset() + tmZero.reset() + tmBwd.reset() + tmOpt.reset() + + tmH2D.reset() + tmEmb.reset() + tmA2A.reset() + tmA2A1.reset() + tmBot.reset() + tmInt.reset() + tmTop.reset() + tmAllGa.reset() + + tmA2A10.reset() + tmA2A11.reset() + tmA2A12.reset() + tmA2A13.reset() + +def tmSummary(pid): + + print("Summary of the tm timers:") + print("---------{:6d}----------------".format(pid)) + tmGetData.output(0) + tmFwd.output(0) + tmH2D.output(1) + tmEmb.output(1) + tmA2A.output(1) + tmA2A1.output(1) + tmBot.output(1) + tmInt.output(1) + tmTop.output(1) + tmAllGa.output(1) + tmLoss.output(0) + tmZero.output(0) + tmBwd.output(0) + tmOpt.output(0) + + tmA2A10.output(1) + tmA2A11.output(1) + tmA2A12.output(1) + tmA2A13.output(1) + print("========={:6d}================".format(pid)) + +if __name__ == "__main__": + + t1 = ProfTimer("Test1") + t1.start() + time.sleep(3) + t1.stop() + t1.elapsed() + t1.output() + + t1.start() + time.sleep(5) + t1.stop() + t1.output() + + From c0fba8635e098dbdb2f5a49eab47506f7a47730b Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Thu, 10 Dec 2020 03:41:01 -0800 Subject: [PATCH 53/57] add tt.py --- tt.py | 1572 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1572 insertions(+) create mode 100644 tt.py diff --git a/tt.py b/tt.py new file mode 100644 index 00000000..1bb52f52 --- /dev/null +++ b/tt.py @@ -0,0 +1,1572 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +# +# Description: an implementation of a deep learning recommendation model (DLRM) +# The model input consists of dense and sparse features. The former is a vector +# of floating point values. The latter is a list of sparse indices into +# embedding tables, which consist of vectors of floating point values. +# The selected vectors are passed to mlp networks denoted by triangles, +# in some cases the vectors are interacted through operators (Ops). +# +# output: +# vector of values +# model: | +# /\ +# /__\ +# | +# _____________________> Op <___________________ +# / | \ +# /\ /\ /\ +# /__\ /__\ ... /__\ +# | | | +# | Op Op +# | ____/__\_____ ____/__\____ +# | |_Emb_|____|__| ... |_Emb_|__|___| +# input: +# [ dense features ] [sparse indices] , ..., [sparse indices] +# +# More precise definition of model layers: +# 1) fully connected layers of an mlp +# z = f(y) +# y = Wx + b +# +# 2) embedding lookup (for a list of sparse indices p=[p1,...,pk]) +# z = Op(e1,...,ek) +# obtain vectors e1=E[:,p1], ..., ek=E[:,pk] +# +# 3) Operator Op can be one of the following +# Sum(e1,...,ek) = e1 + ... + ek +# Dot(e1,...,ek) = [e1'e1, ..., e1'ek, ..., ek'e1, ..., ek'ek] +# Cat(e1,...,ek) = [e1', ..., ek']' +# where ' denotes transpose operation +# +# References: +# [1] Maxim Naumov, Dheevatsa Mudigere, Hao-Jun Michael Shi, Jianyu Huang, +# Narayanan Sundaram, Jongsoo Park, Xiaodong Wang, Udit Gupta, Carole-Jean Wu, +# Alisson G. Azzolini, Dmytro Dzhulgakov, Andrey Mallevich, Ilia Cherniavskii, +# Yinghai Lu, Raghuraman Krishnamoorthi, Ansha Yu, Volodymyr Kondratenko, +# Stephanie Pereira, Xianjie Chen, Wenlin Chen, Vijay Rao, Bill Jia, Liang Xiong, +# Misha Smelyanskiy, "Deep Learning Recommendation Model for Personalization and +# Recommendation Systems", CoRR, arXiv:1906.00091, 2019 + +from __future__ import absolute_import, division, print_function, unicode_literals + +# miscellaneous +import builtins +import functools +# import bisect +# import shutil +import time +import json +# data generation +import dlrm_data_pytorch as dp + +# numpy +import numpy as np +import socket + +# onnx +# The onnx import causes deprecation warnings every time workers +# are spawned during testing. So, we filter out those warnings. +import warnings +with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) +## import onnx + +# pytorch +import torch +from torch import onnx +import torch.nn as nn +from torch.nn.parallel.parallel_apply import parallel_apply +from torch.nn.parallel.replicate import replicate +from torch.nn.parallel.scatter_gather import gather, scatter + +# For distributed run +import extend_distributed as ext_dist + +# quotient-remainder trick +from tricks.qr_embedding_bag import QREmbeddingBag +# mixed-dimension trick +from tricks.md_embedding_bag import PrEmbeddingBag, md_solver + +import sklearn.metrics + +import uuid +import project +from torch.nn.parallel import DistributedDataParallel as DDP + +import dlrm_data as dd + +# Add dlrm self profiling timers +import profile as tm + +# import synthetic_data_loader as fb_syn_data + +# from torchviz import make_dot +# import torch.nn.functional as Functional +# from torch.nn.parameter import Parameter + +from torch.optim.lr_scheduler import _LRScheduler + +exc = getattr(builtins, "IOError", "FileNotFoundError") + +class LRPolicyScheduler(_LRScheduler): + def __init__(self, optimizer, num_warmup_steps, decay_start_step, num_decay_steps): + self.num_warmup_steps = num_warmup_steps + self.decay_start_step = decay_start_step + self.decay_end_step = decay_start_step + num_decay_steps + self.num_decay_steps = num_decay_steps + + if self.decay_start_step < self.num_warmup_steps: + sys.exit("Learning rate warmup must finish before the decay starts") + + super(LRPolicyScheduler, self).__init__(optimizer) + + def get_lr(self): + step_count = self._step_count + if step_count < self.num_warmup_steps: + # warmup + scale = 1.0 - (self.num_warmup_steps - step_count) / self.num_warmup_steps + lr = [base_lr * scale for base_lr in self.base_lrs] + self.last_lr = lr + elif self.decay_start_step <= step_count and step_count < self.decay_end_step: + # decay + decayed_steps = step_count - self.decay_start_step + scale = ((self.num_decay_steps - decayed_steps) / self.num_decay_steps) ** 2 + min_lr = 0.0000001 + lr = [max(min_lr, base_lr * scale) for base_lr in self.base_lrs] + self.last_lr = lr + else: + if self.num_decay_steps > 0: + # freeze at last, either because we're after decay + # or because we're between warmup and decay + lr = self.last_lr + else: + # do not adjust + lr = self.base_lrs + return lr + +### define dlrm in PyTorch ### +class DLRM_Net(nn.Module): + def create_mlp(self, ln, sigmoid_layer): + # build MLP layer by layer + layers = nn.ModuleList() + for i in range(0, ln.size - 1): + n = ln[i] + m = ln[i + 1] + + # construct fully connected operator + LL = nn.Linear(int(n), int(m), bias=True) + + # initialize the weights + # with torch.no_grad(): + # custom Xavier input, output or two-sided fill + mean = 0.0 # std_dev = np.sqrt(variance) + std_dev = np.sqrt(2 / (m + n)) # np.sqrt(1 / m) # np.sqrt(1 / n) + W = np.random.normal(mean, std_dev, size=(m, n)).astype(np.float32) + std_dev = np.sqrt(1 / m) # np.sqrt(2 / (m + 1)) + bt = np.random.normal(mean, std_dev, size=m).astype(np.float32) + # approach 1 + LL.weight.data = torch.tensor(W, requires_grad=True) + LL.bias.data = torch.tensor(bt, requires_grad=True) + # approach 2 + # LL.weight.data.copy_(torch.tensor(W)) + # LL.bias.data.copy_(torch.tensor(bt)) + # approach 3 + # LL.weight = Parameter(torch.tensor(W),requires_grad=True) + # LL.bias = Parameter(torch.tensor(bt),requires_grad=True) + layers.append(LL) + + # construct sigmoid or relu operator + if i == sigmoid_layer: + layers.append(nn.Sigmoid()) + else: + layers.append(nn.ReLU()) + + # approach 1: use ModuleList + # return layers + # approach 2: use Sequential container to wrap all layers + return torch.nn.Sequential(*layers) + + def create_emb(self, m, ln): + emb_l = nn.ModuleList() + # save the numpy random state + np_rand_state = np.random.get_state() + for i in range(0, ln.size): + if ext_dist.my_size > 1: + if not i in self.local_emb_indices: continue + # Use per table random seed for Embedding initialization + np.random.seed(self.l_emb_seeds[i]) + n = ln[i] + # construct embedding operator + if self.qr_flag and n > self.qr_threshold: + EE = QREmbeddingBag(n, m, self.qr_collisions, + operation=self.qr_operation, mode="sum", sparse=True) + elif self.md_flag: + base = max(m) + _m = m[i] if n > self.md_threshold else base + EE = PrEmbeddingBag(n, _m, base) + # use np initialization as below for consistency... + W = np.random.uniform( + low=-np.sqrt(1 / n), high=np.sqrt(1 / n), size=(n, _m) + ).astype(np.float32) + EE.embs.weight.data = torch.tensor(W, requires_grad=True) + + else: + #_weight = torch.empty([n, m]).uniform_(-np.sqrt(1 / n), np.sqrt(1 / n)) + #EE = nn.EmbeddingBag(n, m, mode="sum", sparse=True, _weight= _weight) + #EE = nn.EmbeddingBag(n, m, mode="sum", sparse=True) + + # initialize embeddings + # nn.init.uniform_(EE.weight, a=-np.sqrt(1 / n), b=np.sqrt(1 / n)) + W = np.random.uniform( + low=-np.sqrt(1 / n), high=np.sqrt(1 / n), size=(n, m) + ).astype(np.float32) + # approach 1 + EE = nn.EmbeddingBag(n, m, mode="sum", sparse=True, _weight=torch.tensor(W, requires_grad=True)) + #EE.weight.data = torch.tensor(W, requires_grad=True) + # approach 2 + # EE.weight.data.copy_(torch.tensor(W)) + # approach 3 + # EE.weight = Parameter(torch.tensor(W),requires_grad=True) + + if ext_dist.my_size > 1: + if i in self.local_emb_indices: + emb_l.append(EE) + else: + emb_l.append(EE) + + # Restore the numpy random state + np.random.set_state(np_rand_state) + return emb_l + + def __init__( + self, + m_spa=None, + ln_emb=None, + ln_bot=None, + ln_top=None, + proj_size = 0, + arch_interaction_op=None, + arch_interaction_itself=False, + sigmoid_bot=-1, + sigmoid_top=-1, + sync_dense_params=True, + loss_threshold=0.0, + ndevices=-1, + qr_flag=False, + qr_operation="mult", + qr_collisions=0, + qr_threshold=200, + md_flag=False, + md_threshold=200, + ): + super(DLRM_Net, self).__init__() + + if ( + (m_spa is not None) + and (ln_emb is not None) + and (ln_bot is not None) + and (ln_top is not None) + and (arch_interaction_op is not None) + ): + + # save arguments + self.proj_size = proj_size + self.ndevices = ndevices + self.output_d = 0 + self.parallel_model_batch_size = -1 + self.parallel_model_is_not_prepared = True + self.arch_interaction_op = arch_interaction_op + self.arch_interaction_itself = arch_interaction_itself + self.sync_dense_params = sync_dense_params + self.loss_threshold = loss_threshold + # create variables for QR embedding if applicable + self.qr_flag = qr_flag + if self.qr_flag: + self.qr_collisions = qr_collisions + self.qr_operation = qr_operation + self.qr_threshold = qr_threshold + # create variables for MD embedding if applicable + self.md_flag = md_flag + if self.md_flag: + self.md_threshold = md_threshold + + # generate np seeds for Emb table initialization + self.l_emb_seeds = np.random.randint(low=0, high=100000, size=len(ln_emb)) + + #If running distributed, get local slice of embedding tables + if ext_dist.my_size > 1: + n_emb = len(ln_emb) + self.n_global_emb = n_emb + self.n_local_emb, self.n_emb_per_rank = ext_dist.get_split_lengths(n_emb) + self.local_emb_slice = ext_dist.get_my_slice(n_emb) + self.local_emb_indices = list(range(n_emb))[self.local_emb_slice] + #ln_emb = ln_emb[self.local_emb_slice] + + # create operators + if ndevices <= 1: + self.emb_l = self.create_emb(m_spa, ln_emb) + self.bot_l = self.create_mlp(ln_bot, sigmoid_bot) + self.top_l = self.create_mlp(ln_top, sigmoid_top) + if (proj_size > 0): + self.proj_l = project.create_proj(len(ln_emb)+1, proj_size) + + def apply_mlp(self, x, layers): + # approach 1: use ModuleList + # for layer in layers: + # x = layer(x) + # return x + # approach 2: use Sequential container to wrap all layers + return layers(x) + + def apply_proj(self, x, layers): + # approach 1: use ModuleList + # for layer in layers: + # x = layer(x) + # return x + # approach 2: use Sequential container to wrap all layers + return layers(x) + + def apply_emb(self, lS_o, lS_i, emb_l): + # WARNING: notice that we are processing the batch at once. We implicitly + # assume that the data is laid out such that: + # 1. each embedding is indexed with a group of sparse indices, + # corresponding to a single lookup + # 2. for each embedding the lookups are further organized into a batch + # 3. for a list of embedding tables there is a list of batched lookups + + ly = [] + for k, sparse_index_group_batch in enumerate(lS_i): + sparse_offset_group_batch = lS_o[k] + + # embedding lookup + # We are using EmbeddingBag, which implicitly uses sum operator. + # The embeddings are represented as tall matrices, with sum + # happening vertically across 0 axis, resulting in a row vector + E = emb_l[k] + V = E(sparse_index_group_batch, sparse_offset_group_batch) + + ly.append(V) + + # print(ly) + return ly + + def interact_features(self, x, ly): + if self.arch_interaction_op == "dot": + # concatenate dense and sparse features + (batch_size, d) = x.shape + T = torch.cat([x] + ly, dim=1).view((batch_size, -1, d)) + # perform a dot product + if (self.proj_size > 0): + R = project.project(T, x, self.proj_l) + #TT = torch.transpose(T, 1, 2) + #TS = torch.reshape(TT, (-1, TT.size(2))) + #TC = self.apply_mlp(TS, self.proj_l) + #TR = torch.reshape(TC, (-1, d ,self.proj_size)) + #Z = torch.bmm(T, TR) + #Zflat = Z.view((batch_size, -1)) + #R = torch.cat([x] + [Zflat], dim=1) + else: + Z = torch.bmm(T, torch.transpose(T, 1, 2)) + # append dense feature with the interactions (into a row vector) + # approach 1: all + # Zflat = Z.view((batch_size, -1)) + # approach 2: unique + _, ni, nj = Z.shape + # approach 1: tril_indices + # offset = 0 if self.arch_interaction_itself else -1 + # li, lj = torch.tril_indices(ni, nj, offset=offset) + # approach 2: custom + offset = 1 if self.arch_interaction_itself else 0 + li = torch.tensor([i for i in range(ni) for j in range(i + offset)]) + lj = torch.tensor([j for i in range(nj) for j in range(i + offset)]) + Zflat = Z[:, li, lj] + # concatenate dense features and interactions + R = torch.cat([x] + [Zflat], dim=1) + elif self.arch_interaction_op == "cat": + # concatenation features (into a row vector) + R = torch.cat([x] + ly, dim=1) + else: + sys.exit( + "ERROR: --arch-interaction-op=" + + self.arch_interaction_op + + " is not supported" + ) + + return R + + def forward(self, dense_x, lS_o, lS_i): + if ext_dist.my_size > 1: + return self.distributed_forward(dense_x, lS_o, lS_i) + elif self.ndevices <= 1: + return self.sequential_forward(dense_x, lS_o, lS_i) + else: + return self.parallel_forward(dense_x, lS_o, lS_i) + + def sequential_forward(self, dense_x, lS_o, lS_i): + # process dense features (using bottom mlp), resulting in a row vector + x = self.apply_mlp(dense_x, self.bot_l) + # debug prints + # print("intermediate") + # print(x.detach().cpu().numpy()) + + # process sparse features(using embeddings), resulting in a list of row vectors + ly = self.apply_emb(lS_o, lS_i, self.emb_l) + # for y in ly: + # print(y.detach().cpu().numpy()) + + # interact features (dense and sparse) + z = self.interact_features(x, ly) + # print(z.detach().cpu().numpy()) + + # obtain probability of a click (using top mlp) + p = self.apply_mlp(z, self.top_l) + + # clamp output if needed + if 0.0 < self.loss_threshold and self.loss_threshold < 1.0: + z = torch.clamp(p, min=self.loss_threshold, max=(1.0 - self.loss_threshold)) + else: + z = p + + return z + + def distributed_forward(self, dense_x, lS_o, lS_i): + batch_size = dense_x.size()[0] + # WARNING: # of ranks must be <= batch size in distributed_forward call + if batch_size < ext_dist.my_size: + sys.exit("ERROR: batch_size (%d) must be larger than number of ranks (%d)" % (batch_size, ext_dist.my_size)) + if batch_size % ext_dist.my_size != 0: + sys.exit("ERROR: batch_size %d can not split across %d ranks evenly" % (batch_size, ext_dist.my_size)) + + dense_x = dense_x[ext_dist.get_my_slice(batch_size)] + lS_o = lS_o[self.local_emb_slice] + lS_i = lS_i[self.local_emb_slice] + + if (len(self.emb_l) != len(lS_o)) or (len(self.emb_l) != len(lS_i)): + sys.exit("ERROR: corrupted model input detected in distributed_forward call") + + # embeddings + tm.tmEmb.start() + ly = self.apply_emb(lS_o, lS_i, self.emb_l) + tm.tmEmb.stop() + + # print("ly: ", ly) + # debug prints + # print(ly) + + # WARNING: Note that at this point we have the result of the embedding lookup + # for the entire batch on each rank. We would like to obtain partial results + # corresponding to all embedding lookups, but part of the batch on each rank. + # Therefore, matching the distribution of output of bottom mlp, so that both + # could be used for subsequent interactions on each device. + if len(self.emb_l) != len(ly): + sys.exit("ERROR: corrupted intermediate result in distributed_forward call") + + tm.tmA2A.start() + a2a_req = ext_dist.alltoall(ly, self.n_emb_per_rank) + tm.tmA2A.stop() + + tm.tmBot.start() + x = self.apply_mlp(dense_x, self.bot_l) + tm.tmBot.stop() + + # debug prints + # print(x) + + tm.tmA2A1.start() + ly = a2a_req.wait() + tm.tmA2A1.stop() + # print("ly: ", ly) + ly = list(ly) + + # interactions + tm.tmInt.start() + z = self.interact_features(x, ly) + tm.tmInt.stop() + # debug prints + # print(z) + + # top mlp + tm.tmTop.start() + p = self.apply_mlp(z, self.top_l) + tm.tmTop.stop() + + # clamp output if needed + if 0.0 < self.loss_threshold and self.loss_threshold < 1.0: + z = torch.clamp( + p, min=self.loss_threshold, max=(1.0 - self.loss_threshold) + ) + else: + z = p + + ### gather the distributed results on each rank ### + # For some reason it requires explicit sync before all_gather call if + # tensor is on GPU memory + tm.tmAllGa.start() + if z.is_cuda: torch.cuda.synchronize() + (_, batch_split_lengths) = ext_dist.get_split_lengths(batch_size) + z = ext_dist.all_gather(z, batch_split_lengths) + tm.tmAllGa.stop() + #print("Z: %s" % z) + + return z + + def parallel_forward(self, dense_x, lS_o, lS_i): + ### prepare model (overwrite) ### + # WARNING: # of devices must be >= batch size in parallel_forward call + batch_size = dense_x.size()[0] + ndevices = min(self.ndevices, batch_size, len(self.emb_l)) + device_ids = range(ndevices) + # WARNING: must redistribute the model if mini-batch size changes(this is common + # for last mini-batch, when # of elements in the dataset/batch size is not even + if self.parallel_model_batch_size != batch_size: + self.parallel_model_is_not_prepared = True + + if self.parallel_model_is_not_prepared or self.sync_dense_params: + # replicate mlp (data parallelism) + self.bot_l_replicas = replicate(self.bot_l, device_ids) + self.top_l_replicas = replicate(self.top_l, device_ids) + self.parallel_model_batch_size = batch_size + + if self.parallel_model_is_not_prepared: + # distribute embeddings (model parallelism) + t_list = [] + for k, emb in enumerate(self.emb_l): + d = torch.device("cuda:" + str(k % ndevices)) + emb.to(d) + t_list.append(emb.to(d)) + self.emb_l = nn.ModuleList(t_list) + self.parallel_model_is_not_prepared = False + + ### prepare input (overwrite) ### + # scatter dense features (data parallelism) + # print(dense_x.device) + dense_x = scatter(dense_x, device_ids, dim=0) + # distribute sparse features (model parallelism) + if (len(self.emb_l) != len(lS_o)) or (len(self.emb_l) != len(lS_i)): + sys.exit("ERROR: corrupted model input detected in parallel_forward call") + + t_list = [] + i_list = [] + for k, _ in enumerate(self.emb_l): + d = torch.device("cuda:" + str(k % ndevices)) + t_list.append(lS_o[k].to(d)) + i_list.append(lS_i[k].to(d)) + lS_o = t_list + lS_i = i_list + + ### compute results in parallel ### + # bottom mlp + # WARNING: Note that the self.bot_l is a list of bottom mlp modules + # that have been replicated across devices, while dense_x is a tuple of dense + # inputs that has been scattered across devices on the first (batch) dimension. + # The output is a list of tensors scattered across devices according to the + # distribution of dense_x. + x = parallel_apply(self.bot_l_replicas, dense_x, None, device_ids) + # debug prints + # print(x) + + # embeddings + ly = self.apply_emb(lS_o, lS_i, self.emb_l) + # debug prints + # print(ly) + + # butterfly shuffle (implemented inefficiently for now) + # WARNING: Note that at this point we have the result of the embedding lookup + # for the entire batch on each device. We would like to obtain partial results + # corresponding to all embedding lookups, but part of the batch on each device. + # Therefore, matching the distribution of output of bottom mlp, so that both + # could be used for subsequent interactions on each device. + if len(self.emb_l) != len(ly): + sys.exit("ERROR: corrupted intermediate result in parallel_forward call") + + t_list = [] + for k, _ in enumerate(self.emb_l): + d = torch.device("cuda:" + str(k % ndevices)) + y = scatter(ly[k], device_ids, dim=0) + t_list.append(y) + # adjust the list to be ordered per device + ly = list(map(lambda y: list(y), zip(*t_list))) + # debug prints + # print(ly) + + # interactions + z = [] + for k in range(ndevices): + zk = self.interact_features(x[k], ly[k]) + z.append(zk) + # debug prints + # print(z) + + # top mlp + # WARNING: Note that the self.top_l is a list of top mlp modules that + # have been replicated across devices, while z is a list of interaction results + # that by construction are scattered across devices on the first (batch) dim. + # The output is a list of tensors scattered across devices according to the + # distribution of z. + p = parallel_apply(self.top_l_replicas, z, None, device_ids) + + ### gather the distributed results ### + p0 = gather(p, self.output_d, dim=0) + + # clamp output if needed + if 0.0 < self.loss_threshold and self.loss_threshold < 1.0: + z0 = torch.clamp( + p0, min=self.loss_threshold, max=(1.0 - self.loss_threshold) + ) + else: + z0 = p0 + + return z0 + + +def dash_separated_ints(value): + vals = value.split('-') + for val in vals: + try: + int(val) + except ValueError: + raise argparse.ArgumentTypeError( + "%s is not a valid dash separated list of ints" % value) + + return value + + +def dash_separated_floats(value): + vals = value.split('-') + for val in vals: + try: + float(val) + except ValueError: + raise argparse.ArgumentTypeError( + "%s is not a valid dash separated list of floats" % value) + + return value + + +if __name__ == "__main__": + ### import packages ### + import sys + import os + import argparse + + ### parse arguments ### + parser = argparse.ArgumentParser( + description="Train Deep Learning Recommendation Model (DLRM)" + ) + # model related parameters + parser.add_argument("--arch-sparse-feature-size", type=int, default=2) + + parser.add_argument( + "--arch-embedding-size", type=dash_separated_ints, default="4-3-2") + parser.add_argument("--arch-project-size", type=int, default=0) + + # j will be replaced with the table number + parser.add_argument( + "--arch-mlp-bot", type=dash_separated_ints, default="4-3-2") + parser.add_argument( + "--arch-mlp-top", type=dash_separated_ints, default="4-2-1") + parser.add_argument( + "--arch-interaction-op", type=str, choices=['dot', 'cat'], default="dot") + parser.add_argument("--arch-interaction-itself", action="store_true", default=False) + # embedding table options + parser.add_argument("--md-flag", action="store_true", default=False) + parser.add_argument("--md-threshold", type=int, default=200) + parser.add_argument("--md-temperature", type=float, default=0.3) + parser.add_argument("--md-round-dims", action="store_true", default=False) + parser.add_argument("--qr-flag", action="store_true", default=False) + parser.add_argument("--qr-threshold", type=int, default=200) + parser.add_argument("--qr-operation", type=str, default="mult") + parser.add_argument("--qr-collisions", type=int, default=4) + # activations and loss + parser.add_argument("--activation-function", type=str, default="relu") + parser.add_argument("--loss-function", type=str, default="mse") # or bce or wbce + parser.add_argument( + "--loss-weights", type=dash_separated_floats, default="1.0-1.0") # for wbce + parser.add_argument("--loss-threshold", type=float, default=0.0) # 1.0e-7 + parser.add_argument("--round-targets", type=bool, default=False) + # data + parser.add_argument("--data-size", type=int, default=1) + parser.add_argument("--num-batches", type=int, default=0) + parser.add_argument( + "--data-generation", type=str, default="random" + ) # synthetic or dataset + parser.add_argument("--synthetic-data-folder", type=str, + default="./synthetic_data/syn_data_bs65536") + # add Gaussian distribution + parser.add_argument("--rand-data-dist", type=str, default="uniform") # uniform or gaussian + parser.add_argument("--rand-data-min", type=float, default=0) + parser.add_argument("--rand-data-max", type=float, default=1) + parser.add_argument("--rand-data-mu", type=float, default=-1) + parser.add_argument("--rand-data-sigma", type=float, default=1) + + parser.add_argument("--data-trace-file", type=str, default="./input/dist_emb_j.log") + parser.add_argument("--data-set", type=str, default="kaggle") # or terabyte + parser.add_argument("--raw-data-file", type=str, default="") + parser.add_argument("--processed-data-file", type=str, default="") + parser.add_argument("--data-randomize", type=str, default="total") # or day or none + parser.add_argument("--data-trace-enable-padding", type=bool, default=False) + parser.add_argument("--max-ind-range", type=int, default=-1) + parser.add_argument("--data-sub-sample-rate", type=float, default=0.0) # in [0, 1] + parser.add_argument("--num-indices-per-lookup", type=int, default=10) + parser.add_argument("--num-indices-per-lookup-fixed", type=bool, default=False) + parser.add_argument("--num-workers", type=int, default=0) + parser.add_argument("--memory-map", action="store_true", default=False) + # training + parser.add_argument("--mini-batch-size", type=int, default=1) + parser.add_argument("--nepochs", type=int, default=1) + parser.add_argument("--learning-rate", type=float, default=0.01) + parser.add_argument("--print-precision", type=int, default=5) + parser.add_argument("--numpy-rand-seed", type=int, default=123) + parser.add_argument("--sync-dense-params", type=bool, default=True) + # inference + parser.add_argument("--inference-only", action="store_true", default=False) + # onnx + parser.add_argument("--save-onnx", action="store_true", default=False) + # gpu + parser.add_argument("--use-gpu", action="store_true", default=False) + # distributed run + parser.add_argument("--dist-backend", type=str, default="") + # debugging and profiling + parser.add_argument("--print-freq", type=int, default=1) + parser.add_argument("--test-freq", type=int, default=-1) + parser.add_argument("--test-mini-batch-size", type=int, default=-1) + parser.add_argument("--test-num-workers", type=int, default=-1) + parser.add_argument("--print-time", action="store_true", default=False) + parser.add_argument("--debug-mode", action="store_true", default=False) + parser.add_argument("--enable-profiling", action="store_true", default=False) + parser.add_argument("--plot-compute-graph", action="store_true", default=False) + # store/load model + parser.add_argument("--out-dir", type=str, default=".") + parser.add_argument("--save-model", type=str, default="") + parser.add_argument("--load-model", type=str, default="") + # mlperf logging (disables other output and stops early) + parser.add_argument("--mlperf-logging", action="store_true", default=False) + # stop at target accuracy Kaggle 0.789, Terabyte (sub-sampled=0.875) 0.8107 + parser.add_argument("--mlperf-acc-threshold", type=float, default=0.0) + # stop at target AUC Terabyte (no subsampling) 0.8025 + parser.add_argument("--mlperf-auc-threshold", type=float, default=0.0) + parser.add_argument("--mlperf-bin-loader", action='store_true', default=False) + parser.add_argument("--mlperf-bin-shuffle", action='store_true', default=False) + + # LR policy + parser.add_argument("--lr-num-warmup-steps", type=int, default=0) + parser.add_argument("--lr-decay-start-step", type=int, default=0) + parser.add_argument("--lr-num-decay-steps", type=int, default=0) + + args = parser.parse_args() + + print(socket.gethostname()) + + ext_dist.init_distributed(backend=args.dist_backend) + + # print("success size= ", ext_dist.my_size, ext_dist.my_rank) + + ext_dist.barrier() + + if args.mlperf_logging: + print('command line args: ', json.dumps(vars(args))) + + ### some basic setup ### + np.random.seed(args.numpy_rand_seed) + np.set_printoptions(precision=args.print_precision) + torch.set_printoptions(precision=args.print_precision) + torch.manual_seed(args.numpy_rand_seed) + + if (args.test_mini_batch_size < 0): + # if the parameter is not set, use the training batch size + args.test_mini_batch_size = args.mini_batch_size + if (args.test_num_workers < 0): + # if the parameter is not set, use the same parameter for training + args.test_num_workers = args.num_workers + if args.mini_batch_size % ext_dist.my_size !=0 or args.test_mini_batch_size % ext_dist.my_size != 0: + print("Either test minibatch (%d) or train minibatch (%d) does not split across %d ranks" % (args.test_mini_batch_size, args.mini_batch_size, ext_dist.my_size)) + sys.exit(1) + + use_gpu = args.use_gpu and torch.cuda.is_available() + if use_gpu: + torch.cuda.manual_seed_all(args.numpy_rand_seed) + torch.backends.cudnn.deterministic = True + if ext_dist.my_size > 1: + ngpus = torch.cuda.device_count() # 1 + if ext_dist.my_local_size > torch.cuda.device_count(): + print("Not sufficient GPUs available... local_size = %d, ngpus = %d" % (ext_dist.my_local_size, ngpus)) + sys.exit(1) + ngpus = 1 + device = torch.device("cuda", ext_dist.my_local_rank) + else: + device = torch.device("cuda", 0) + ngpus = torch.cuda.device_count() # 1 + ngpus=1 + print("Using {} GPU(s)...".format(ngpus)) + else: + device = torch.device("cpu") + print("Using CPU...") + + ### prepare training data ### + ln_bot = np.fromstring(args.arch_mlp_bot, dtype=int, sep="-") + # input data + if (args.data_generation == "dataset"): + + train_data, train_ld, test_data, test_ld = \ + dp.make_criteo_data_and_loaders(args) + nbatches = args.num_batches if args.num_batches > 0 else len(train_ld) + nbatches_test = len(test_ld) + + ln_emb = train_data.counts + # enforce maximum limit on number of vectors per embedding + if args.max_ind_range > 0: + ln_emb = np.array(list(map( + lambda x: x if x < args.max_ind_range else args.max_ind_range, + ln_emb + ))) + m_den = train_data.m_den + ln_bot[0] = m_den + + elif args.data_generation == "synthetic": + # input and target at random + ln_emb = np.fromstring(args.arch_embedding_size, dtype=int, sep="-") + m_den = ln_bot[0] + train_data, train_ld = dd.data_loader(args, ln_emb, m_den) + nbatches = args.num_batches if args.num_batches > 0 else len(train_ld) + table_feature_map = None # {idx : idx for idx in range(len(ln_emb))} + + else: + # input and target at random + ln_emb = np.fromstring(args.arch_embedding_size, dtype=int, sep="-") + m_den = ln_bot[0] + train_data, train_ld = dd.make_random_data_and_loader(args, ln_emb, m_den) + nbatches = args.num_batches if args.num_batches > 0 else len(train_ld) + + ### parse command line arguments ### + m_spa = args.arch_sparse_feature_size + num_fea = ln_emb.size + 1 # num sparse + num dense features + m_den_out = ln_bot[ln_bot.size - 1] + if args.arch_interaction_op == "dot": + # approach 1: all + # num_int = num_fea * num_fea + m_den_out + # approach 2: unique + if (args.arch_project_size > 0): + num_int = num_fea * args.arch_project_size + m_den_out + else: + if args.arch_interaction_itself: + num_int = (num_fea * (num_fea + 1)) // 2 + m_den_out + else: + num_int = (num_fea * (num_fea - 1)) // 2 + m_den_out + elif args.arch_interaction_op == "cat": + num_int = num_fea * m_den_out + else: + sys.exit( + "ERROR: --arch-interaction-op=" + + args.arch_interaction_op + + " is not supported" + ) + arch_mlp_top_adjusted = str(num_int) + "-" + args.arch_mlp_top + ln_top = np.fromstring(arch_mlp_top_adjusted, dtype=int, sep="-") + + # sanity check: feature sizes and mlp dimensions must match + if m_den != ln_bot[0]: + sys.exit( + "ERROR: arch-dense-feature-size " + + str(m_den) + + " does not match first dim of bottom mlp " + + str(ln_bot[0]) + ) + if args.qr_flag: + if args.qr_operation == "concat" and 2 * m_spa != m_den_out: + sys.exit( + "ERROR: 2 arch-sparse-feature-size " + + str(2 * m_spa) + + " does not match last dim of bottom mlp " + + str(m_den_out) + + " (note that the last dim of bottom mlp must be 2x the embedding dim)" + ) + if args.qr_operation != "concat" and m_spa != m_den_out: + sys.exit( + "ERROR: arch-sparse-feature-size " + + str(m_spa) + + " does not match last dim of bottom mlp " + + str(m_den_out) + ) + else: + if m_spa != m_den_out: + sys.exit( + "ERROR: arch-sparse-feature-size " + + str(m_spa) + + " does not match last dim of bottom mlp " + + str(m_den_out) + ) + if num_int != ln_top[0]: + sys.exit( + "ERROR: # of feature interactions " + + str(num_int) + + " does not match first dimension of top mlp " + + str(ln_top[0]) + ) + + # assign mixed dimensions if applicable + if args.md_flag: + m_spa = md_solver( + torch.tensor(ln_emb), + args.md_temperature, # alpha + d0=m_spa, + round_dim=args.md_round_dims + ).tolist() + + # test prints (model arch) + if args.debug_mode: + print("model arch:") + print( + "mlp top arch " + + str(ln_top.size - 1) + + " layers, with input to output dimensions:" + ) + print(ln_top) + print("# of interactions") + print(num_int) + print( + "mlp bot arch " + + str(ln_bot.size - 1) + + " layers, with input to output dimensions:" + ) + print(ln_bot) + print("# of features (sparse and dense)") + print(num_fea) + print("dense feature size") + print(m_den) + print("sparse feature size") + print(m_spa) + print( + "# of embeddings (= # of sparse features) " + + str(ln_emb.size) + + ", with dimensions " + + str(m_spa) + + "x:" + ) + print(ln_emb) + + print("data (inputs and targets):") + for j, (X, lS_o, lS_i, T) in enumerate(train_ld): + # early exit if nbatches was set by the user and has been exceeded + if nbatches > 0 and j >= nbatches: + break + + print("mini-batch: %d" % j) + print(X.detach().cpu().numpy()) + # transform offsets to lengths when printing + print( + [ + np.diff( + S_o.detach().cpu().tolist() + list(lS_i[i].shape) + ).tolist() + for i, S_o in enumerate(lS_o) + ] + ) + print([S_i.detach().cpu().tolist() for S_i in lS_i]) + print(T.detach().cpu().numpy()) + + ndevices = min(ngpus, args.mini_batch_size, num_fea - 1) if use_gpu else -1 + + ### construct the neural network specified above ### + # WARNING: to obtain exactly the same initialization for + # the weights we need to start from the same random seed. + # np.random.seed(args.numpy_rand_seed) + dlrm = DLRM_Net( + m_spa, + ln_emb, + ln_bot, + ln_top, + args.arch_project_size, + arch_interaction_op=args.arch_interaction_op, + arch_interaction_itself=args.arch_interaction_itself, + sigmoid_bot=-1, + sigmoid_top=ln_top.size - 2, + sync_dense_params=args.sync_dense_params, + loss_threshold=args.loss_threshold, + ndevices=ndevices, + qr_flag=args.qr_flag, + qr_operation=args.qr_operation, + qr_collisions=args.qr_collisions, + qr_threshold=args.qr_threshold, + md_flag=args.md_flag, + md_threshold=args.md_threshold, + ) + # test prints + if args.debug_mode: + print("initial parameters (weights and bias):") + for param in dlrm.parameters(): + print(param.detach().cpu().numpy()) + # print(dlrm) + + if use_gpu: + # Custom Model-Data Parallel + # the mlps are replicated and use data parallelism, while + # the embeddings are distributed and use model parallelism + dlrm = dlrm.to(device) # .cuda() + if dlrm.ndevices > 1: + dlrm.emb_l = dlrm.create_emb(m_spa, ln_emb) + + if ext_dist.my_size > 1: + if use_gpu: + device_ids = [ext_dist.my_local_rank] + dlrm.bot_l = DDP(dlrm.bot_l, device_ids=device_ids) + dlrm.top_l = DDP(dlrm.top_l, device_ids=device_ids) + else: + dlrm.bot_l = DDP(dlrm.bot_l) + dlrm.top_l = DDP(dlrm.top_l) + + # specify the loss function + if args.loss_function == "mse": + loss_fn = torch.nn.MSELoss(reduction="mean") + elif args.loss_function == "bce": + loss_fn = torch.nn.BCELoss(reduction="mean") + elif args.loss_function == "wbce": + loss_ws = torch.tensor(np.fromstring(args.loss_weights, dtype=float, sep="-")) + loss_fn = torch.nn.BCELoss(reduction="none") + else: + sys.exit("ERROR: --loss-function=" + args.loss_function + " is not supported") + + if not args.inference_only: + # specify the optimizer algorithm + + if ext_dist.my_size == 1: + optimizer = torch.optim.SGD(dlrm.parameters(), lr=args.learning_rate) + #lr_scheduler = LRPolicyScheduler(optimizer, args.lr_num_warmup_steps, args.lr_decay_start_step, + # args.lr_num_decay_steps) + else: + optimizer = torch.optim.SGD([ + {"params": [p for emb in dlrm.emb_l for p in emb.parameters()], "lr" : args.learning_rate}, + {"params": dlrm.bot_l.parameters(), "lr" : args.learning_rate * ext_dist.my_size}, + {"params": dlrm.top_l.parameters(), "lr" : args.learning_rate * ext_dist.my_size} + ], lr=args.learning_rate) + + ### main loop ### + def time_wrap(use_gpu): + if use_gpu: + torch.cuda.synchronize() + return time.time() + + def dlrm_wrap(X, lS_o, lS_i, use_gpu, device): + if use_gpu: # .cuda() + # lS_i can be either a list of tensors or a stacked tensor. + # Handle each case below: + tm.tmH2D.start() + lS_i = [S_i.to(device) for S_i in lS_i] if isinstance(lS_i, list) \ + else lS_i.to(device) + lS_o = [S_o.to(device) for S_o in lS_o] if isinstance(lS_o, list) \ + else lS_o.to(device) + X = X.to(device) + tm.tmH2D.stop() + + return dlrm( + X, + lS_o, + lS_i + ) + else: + return dlrm(X, lS_o, lS_i) + + def loss_fn_wrap(Z, T, use_gpu, device): + if args.loss_function == "mse" or args.loss_function == "bce": + if use_gpu: + return loss_fn(Z, T.to(device)) + else: + return loss_fn(Z, T) + elif args.loss_function == "wbce": + if use_gpu: + loss_ws_ = loss_ws[T.data.view(-1).long()].view_as(T).to(device) + loss_fn_ = loss_fn(Z, T.to(device)) + else: + loss_ws_ = loss_ws[T.data.view(-1).long()].view_as(T) + loss_fn_ = loss_fn(Z, T.to(device)) + loss_sc_ = loss_ws_ * loss_fn_ + # debug prints + # print(loss_ws_) + # print(loss_fn_) + return loss_sc_.mean() + + # training or inference + best_gA_test = 0 + best_auc_test = 0 + skip_upto_epoch = 0 + skip_upto_batch = 0 + total_time = 0 + total_loss = 0 + total_accu = 0 + total_iter = 0 + total_samp = 0 + k = 0 + + # Load model is specified + if not (args.load_model == ""): + print("Loading saved model {}".format(args.load_model)) + if use_gpu: + if dlrm.ndevices > 1: + # NOTE: when targeting inference on multiple GPUs, + # load the model as is on CPU or GPU, with the move + # to multiple GPUs to be done in parallel_forward + ld_model = torch.load(args.load_model) + else: + # NOTE: when targeting inference on single GPU, + # note that the call to .to(device) has already happened + ld_model = torch.load( + args.load_model, + map_location=torch.device('cuda') + # map_location=lambda storage, loc: storage.cuda(0) + ) + else: + # when targeting inference on CPU + ld_model = torch.load(args.load_model, map_location=torch.device('cpu')) + dlrm.load_state_dict(ld_model["state_dict"]) + ld_j = ld_model["iter"] + ld_k = ld_model["epoch"] + ld_nepochs = ld_model["nepochs"] + ld_nbatches = ld_model["nbatches"] + ld_nbatches_test = ld_model["nbatches_test"] + ld_gA = ld_model["train_acc"] + ld_gL = ld_model["train_loss"] + ld_total_loss = ld_model["total_loss"] + ld_total_accu = ld_model["total_accu"] + ld_gA_test = ld_model["test_acc"] + ld_gL_test = ld_model["test_loss"] + if not args.inference_only: + optimizer.load_state_dict(ld_model["opt_state_dict"]) + best_gA_test = ld_gA_test + total_loss = ld_total_loss + total_accu = ld_total_accu + skip_upto_epoch = ld_k # epochs + skip_upto_batch = ld_j # batches + else: + args.print_freq = ld_nbatches + args.test_freq = 0 + + print( + "Saved at: epoch = {:d}/{:d}, batch = {:d}/{:d}, ntbatch = {:d}".format( + ld_k, ld_nepochs, ld_j, ld_nbatches, ld_nbatches_test + ) + ) + print( + "Training state: loss = {:.6f}, accuracy = {:3.3f} %".format( + ld_gL, ld_gA * 100 + ) + ) + print( + "Testing state: loss = {:.6f}, accuracy = {:3.3f} %".format( + ld_gL_test, ld_gA_test * 100 + ) + ) + + ext_dist.barrier() + startTime = time.time() + startTime0 = startTime + skipped = 0 + + print("Processing data") + t1 = time.time() + myobj = list(enumerate(train_ld)) + t2 = time.time() + print("Processing data takes {} seconds with len={}".format(t2-t1, len(myobj))) + print("time/loss/accuracy (if enabled):") + with torch.autograd.profiler.profile(args.enable_profiling, use_gpu, record_shapes=True) as prof: + while k < args.nepochs: + if k < skip_upto_epoch: + continue + + accum_time_begin = time_wrap(use_gpu) + + if args.mlperf_logging: + previous_iteration_time = None + + # for j, (X, lS_o, lS_i, T) in enumerate(train_ld): + for j in range(nbatches): + tm.tmGetData.start() + X, lS_o, lS_i, T = myobj[j][1] + tm.tmGetData.stop() + + if j == 0 and args.save_onnx: + (X_onnx, lS_o_onnx, lS_i_onnx) = (X, lS_o, lS_i) + + if j < skip_upto_batch: + continue + + if (skipped == 2): + ext_dist.barrier() + startTime = time.time() + ext_dist.orig_print("ORIG TIME: ", startTime, accum_time_begin, startTime - accum_time_begin, " for process ", ext_dist.my_rank) + tm.tmClear() + skipped = skipped + 1 + + if args.mlperf_logging: + current_time = time_wrap(use_gpu) + if previous_iteration_time: + iteration_time = current_time - previous_iteration_time + else: + iteration_time = 0 + previous_iteration_time = current_time + else: + t1 = time_wrap(use_gpu) + + # early exit if nbatches was set by the user and has been exceeded + if nbatches > 0 and j >= nbatches: + break + ''' + # debug prints + print("input and targets") + print(X.detach().cpu().numpy()) + print([np.diff(S_o.detach().cpu().tolist() + + list(lS_i[i].shape)).tolist() for i, S_o in enumerate(lS_o)]) + print([S_i.detach().cpu().numpy().tolist() for S_i in lS_i]) + print(T.detach().cpu().numpy()) + ''' + # Skip the batch if batch size not multiple of total ranks + if ext_dist.my_size > 1 and X.size(0) % ext_dist.my_size != 0: + print("Warning: Skiping the batch %d with size %d" % (j, X.size(0))) + continue + + + # forward pass + tm.tmFwd.start() + Z = dlrm_wrap(X, lS_o, lS_i, use_gpu, device) + tm.tmFwd.stop() + + # loss + tm.tmLoss.start() + E = loss_fn_wrap(Z, T, use_gpu, device) + ''' + # debug prints + print("output and loss") + print(Z.detach().cpu().numpy()) + print(E.detach().cpu().numpy()) + ''' + # compute loss and accuracy + L = E.detach().cpu().numpy() # numpy array + S = Z.detach().cpu().numpy() # numpy array + T = T.detach().cpu().numpy() # numpy array + mbs = T.shape[0] # = args.mini_batch_size except maybe for last + A = np.sum((np.round(S, 0) == T).astype(np.uint8)) + tm.tmLoss.stop() + + if not args.inference_only: + # scaled error gradient propagation + # (where we do not accumulate gradients across mini-batches) + tm.tmZero.start() + optimizer.zero_grad() + tm.tmZero.stop() + + # backward pass + tm.tmBwd.start() + E.backward() + tm.tmBwd.stop() + + # debug prints (check gradient norm) + # for l in mlp.layers: + # if hasattr(l, 'weight'): + # print(l.weight.grad.norm().item()) + + # optimizer + tm.tmOpt.start() + optimizer.step() + tm.tmOpt.stop() + + ### lr_scheduler.step() + + if args.mlperf_logging: + total_time += iteration_time + else: + t2 = time_wrap(use_gpu) + total_time += t2 - t1 + total_accu += A + total_loss += L * mbs + total_iter += 1 + total_samp += mbs + + should_print = ((j + 1) % args.print_freq == 0) or (j + 1 == nbatches) + should_test = ( + (args.test_freq > 0) + and (args.data_generation == "dataset") + and (((j + 1) % args.test_freq == 0) or (j + 1 == nbatches)) + ) + + # print time, loss and accuracy + if should_print or should_test: + gT = 1000.0 * total_time / total_iter if args.print_time else -1 + total_time = 0 + + gA = total_accu / total_samp + total_accu = 0 + + gL = total_loss / total_samp + total_loss = 0 + + str_run_type = "inference" if args.inference_only else "training" + print( + "Finished {} it {}/{} of epoch {}, {:.2f} ms/it, ".format( + str_run_type, j + 1, nbatches, k, gT + ) + + "loss {:.6f}, accuracy {:3.3f} % it {} for task {} ".format(gL, + gA * 100, total_iter, ext_dist.my_rank) + ) + # Uncomment the line below to print out the total time with overhead + if ext_dist.my_rank < 2: + tt1 = time_wrap(use_gpu) + ext_dist.orig_print("Accumulated time so far: {} for process {} for step {} at {}" \ + .format(tt1 - accum_time_begin, ext_dist.my_rank, skipped, tt1)) + total_iter = 0 + total_samp = 0 + + # testing + if should_test and not args.inference_only: + # don't measure training iter time in a test iteration + if args.mlperf_logging: + previous_iteration_time = None + + test_accu = 0 + test_loss = 0 + test_samp = 0 + + accum_test_time_begin = time_wrap(use_gpu) + if args.mlperf_logging: + scores = [] + targets = [] + + for i, (X_test, lS_o_test, lS_i_test, T_test) in enumerate(test_ld): + # early exit if nbatches was set by the user and was exceeded + if nbatches > 0 and i >= nbatches: + break + + # Skip the batch if batch size not multiple of total ranks + if ext_dist.my_size > 1 and X_test.size(0) % ext_dist.my_size != 0: + print("Warning: Skiping the batch %d with size %d" % (i, X_test.size(0))) + continue + + t1_test = time_wrap(use_gpu) + + # forward pass + Z_test = dlrm_wrap( + X_test, lS_o_test, lS_i_test, use_gpu, device + ) + if args.mlperf_logging: + S_test = Z_test.detach().cpu().numpy() # numpy array + T_test = T_test.detach().cpu().numpy() # numpy array + scores.append(S_test) + targets.append(T_test) + else: + # loss + E_test = loss_fn_wrap(Z_test, T_test, use_gpu, device) + + # compute loss and accuracy + L_test = E_test.detach().cpu().numpy() # numpy array + S_test = Z_test.detach().cpu().numpy() # numpy array + T_test = T_test.detach().cpu().numpy() # numpy array + mbs_test = T_test.shape[0] # = mini_batch_size except last + A_test = np.sum((np.round(S_test, 0) == T_test).astype(np.uint8)) + test_accu += A_test + test_loss += L_test * mbs_test + test_samp += mbs_test + + t2_test = time_wrap(use_gpu) + + if args.mlperf_logging: + scores = np.concatenate(scores, axis=0) + targets = np.concatenate(targets, axis=0) + + metrics = { + 'loss' : sklearn.metrics.log_loss, + 'recall' : lambda y_true, y_score: + sklearn.metrics.recall_score( + y_true=y_true, + y_pred=np.round(y_score) + ), + 'precision' : lambda y_true, y_score: + sklearn.metrics.precision_score( + y_true=y_true, + y_pred=np.round(y_score) + ), + 'f1' : lambda y_true, y_score: + sklearn.metrics.f1_score( + y_true=y_true, + y_pred=np.round(y_score) + ), + 'ap' : sklearn.metrics.average_precision_score, + 'roc_auc' : sklearn.metrics.roc_auc_score, + 'accuracy' : lambda y_true, y_score: + sklearn.metrics.accuracy_score( + y_true=y_true, + y_pred=np.round(y_score) + ), + # 'pre_curve' : sklearn.metrics.precision_recall_curve, + # 'roc_curve' : sklearn.metrics.roc_curve, + } + + # print("Compute time for validation metric : ", end="") + # first_it = True + validation_results = {} + for metric_name, metric_function in metrics.items(): + # if first_it: + # first_it = False + # else: + # print(", ", end="") + # metric_compute_start = time_wrap(False) + validation_results[metric_name] = metric_function( + targets, + scores + ) + # metric_compute_end = time_wrap(False) + # met_time = metric_compute_end - metric_compute_start + # print("{} {:.4f}".format(metric_name, 1000 * (met_time)), + # end="") + # print(" ms") + gA_test = validation_results['accuracy'] + gL_test = validation_results['loss'] + else: + gA_test = test_accu / test_samp + gL_test = test_loss / test_samp + + is_best = gA_test > best_gA_test + if is_best: + best_gA_test = gA_test + if not (args.save_model == ""): + print("Saving model to {}".format(args.save_model)) + torch.save( + { + "epoch": k, + "nepochs": args.nepochs, + "nbatches": nbatches, + "nbatches_test": nbatches_test, + "iter": j + 1, + "state_dict": dlrm.state_dict(), + "train_acc": gA, + "train_loss": gL, + "test_acc": gA_test, + "test_loss": gL_test, + "total_loss": total_loss, + "total_accu": total_accu, + "opt_state_dict": optimizer.state_dict(), + }, + args.save_model, + ) + + if args.mlperf_logging: + is_best = validation_results['roc_auc'] > best_auc_test + if is_best: + best_auc_test = validation_results['roc_auc'] + + print( + "Testing at - {}/{} of epoch {},".format(j + 1, nbatches, k) + + " loss {:.6f}, recall {:.4f}, precision {:.4f},".format( + validation_results['loss'], + validation_results['recall'], + validation_results['precision'] + ) + + " f1 {:.4f}, ap {:.4f},".format( + validation_results['f1'], + validation_results['ap'], + ) + + " auc {:.4f}, best auc {:.4f},".format( + validation_results['roc_auc'], + best_auc_test + ) + + " accuracy {:3.3f} %, best accuracy {:3.3f} %".format( + validation_results['accuracy'] * 100, + best_gA_test * 100 + ) + ) + else: + print( + "Testing at - {}/{} of epoch {},".format(j + 1, nbatches, 0) + + " loss {:.6f}, accuracy {:3.3f} %, best {:3.3f} %".format( + gL_test, gA_test * 100, best_gA_test * 100 + ) + ) + # Uncomment the line below to print out the total time with overhead + # print("Total test time for this group: {}" \ + # .format(time_wrap(use_gpu) - accum_test_time_begin)) + + if (args.mlperf_logging + and (args.mlperf_acc_threshold > 0) + and (best_gA_test > args.mlperf_acc_threshold)): + print("MLPerf testing accuracy threshold " + + str(args.mlperf_acc_threshold) + + " reached, stop training") + break + + if (args.mlperf_logging + and (args.mlperf_auc_threshold > 0) + and (best_auc_test > args.mlperf_auc_threshold)): + print("MLPerf testing auc threshold " + + str(args.mlperf_auc_threshold) + + " reached, stop training") + break + + #if (ext_dist.my_rank == 0 and should_print): + # print("ITER : ", j, " from nvidia-smi") + # os.system("nvidia-smi") + + k += 1 # nepochs + + #if (ext_dist.my_rank == 0): + # # print(torch.cuda.memory_allocated(0)) + # print(torch.cuda.memory_summary(0)) + # # print("from nvidia-smi") + # os.system("nvidia-smi") + + tt2 = time.time() + endTime = tt2 - startTime + ext_dist.barrier() + tt3 = time.time() + finalTime = tt3 - startTime + if (skipped > 2): + skipped -= 2 + ext_dist.orig_print("Process {} Done with total time {:.6f} measure time {:.6f}s {:.6f}s, \ + iter {:.1f}ms {:.1f}ms steps {} {}".format(ext_dist.my_rank, tt3 - startTime0, + finalTime, endTime, finalTime*1000.0/skipped, endTime*1000.0/skipped, skipped, tt2), flush=True) + if (ext_dist.my_rank < 2): + tm.tmSummary(ext_dist.my_rank) + + file_prefix = "%s/dlrm_s_pytorch_r%d" % (args.out_dir, ext_dist.my_rank) + # profiling + if args.enable_profiling: + os.makedirs(args.out_dir, exist_ok=True) + with open("TT"+str(uuid.uuid4().hex), "w") as prof_f: + prof_f.write(prof.key_averages(group_by_input_shape=True).table( + sort_by="self_cpu_time_total", + )) + +# with open("%s.prof" % file_prefix, "w") as prof_f: +# prof_f.write(prof.key_averages().table(sort_by="cpu_time_total")) +# prof.export_chrome_trace("./%s.json" % file_prefix) +# print(prof.key_averages().table(sort_by="cpu_time_total")) + + # plot compute graph + if args.plot_compute_graph: + sys.exit( + "ERROR: Please install pytorchviz package in order to use the" + + " visualization. Then, uncomment its import above as well as" + + " three lines below and run the code again." + ) + # os.makedirs(args.out_dir, exist_ok=True) + # V = Z.mean() if args.inference_only else E + # dot = make_dot(V, params=dict(dlrm.named_parameters())) + # dot.render('%s_graph' % file_prefix) # write .pdf file + + # test prints + if not args.inference_only and args.debug_mode: + print("updated parameters (weights and bias):") + for param in dlrm.parameters(): + print(param.detach().cpu().numpy()) + + # export the model in onnx + if args.save_onnx: + + dlrm_pytorch_onnx_file = "dlrm_s_pytorch.onnx" + torch.onnx.export( + dlrm, (X_onnx, lS_o_onnx, lS_i_onnx), dlrm_pytorch_onnx_file, verbose=True, use_external_data_format=True + ) + + # recover the model back + dlrm_pytorch_onnx = onnx.load("%s.onnx" % file_prefix) + # check the onnx model + onnx.checker.check_model(dlrm_pytorch_onnx) From 77541e5bf8ec079a2e632cf609952a831eed8b88 Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Fri, 11 Dec 2020 09:21:40 -0800 Subject: [PATCH 54/57] tested version on FAIR --- dlrm_data.py | 10 ++++++---- profile.py | 12 ++++++++++++ tt.py | 43 +++++++++++++++++++++++++++++++++---------- 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/dlrm_data.py b/dlrm_data.py index 7e8264a4..28afd8da 100644 --- a/dlrm_data.py +++ b/dlrm_data.py @@ -234,10 +234,10 @@ def __getitem__(self, index): lS_i = [val[self.lS_o[ind][sInd]:self.lS_o[ind][eInd]] for ind, val in enumerate(self.lS_i)] elif sInd < len(self.lS_o[0]): lS_i = [val[self.lS_o[ind][sInd]:] for ind, val in enumerate(self.lS_i)] - for i in range(len(lS_i)): - bound = self.ln_emb[i] - if not bound == 26000000: - lS_i[i] %= bound +# for i in range(len(lS_i)): +# bound = self.ln_emb[i] +# if not bound == 26000000: +# lS_i[i] %= bound T = self.T[sInd:eInd] return (X, lS_o, lS_i, T) @@ -251,6 +251,8 @@ def synthetic_data_loader(args, ln_emb, m_den): train_data = SyntheticDataset( args.mini_batch_size, ln_emb, + # how to repeat ? + # nbatches=min(args.num_batches, 65536 // args.mini_batch_size), nbatches=args.num_batches, synthetic_data_folder=args.synthetic_data_folder, ) diff --git a/profile.py b/profile.py index 0df3eff6..bb29033a 100644 --- a/profile.py +++ b/profile.py @@ -47,6 +47,10 @@ def output(self, level): tmZero = ProfTimer("Zero ") tmBwd = ProfTimer("Backwrd") tmOpt = ProfTimer("Opt ") +tmSync = ProfTimer("CudaSyn") +tmSync1 = ProfTimer("CudaSy1") +tmSync2 = ProfTimer("CudaSy2") +tmSync3 = ProfTimer("CudaSy3") tmH2D = ProfTimer("CopyH2D") tmEmb = ProfTimer("EMB ") @@ -70,6 +74,10 @@ def tmClear(): tmZero.reset() tmBwd.reset() tmOpt.reset() + tmSync.reset() + tmSync1.reset() + tmSync2.reset() + tmSync3.reset() tmH2D.reset() tmEmb.reset() @@ -103,6 +111,10 @@ def tmSummary(pid): tmZero.output(0) tmBwd.output(0) tmOpt.output(0) +# tmSync.output(0) + tmSync1.output(0) + tmSync2.output(0) + tmSync3.output(0) tmA2A10.output(1) tmA2A11.output(1) diff --git a/tt.py b/tt.py index 1bb52f52..152dce83 100644 --- a/tt.py +++ b/tt.py @@ -101,6 +101,8 @@ # Add dlrm self profiling timers import profile as tm +# import pyprof +# pyprof.init() # causing errors, some symbols not found # import synthetic_data_loader as fb_syn_data @@ -1164,18 +1166,25 @@ def loss_fn_wrap(Z, T, use_gpu, device): startTime0 = startTime skipped = 0 - print("Processing data") - t1 = time.time() - myobj = list(enumerate(train_ld)) - t2 = time.time() - print("Processing data takes {} seconds with len={}".format(t2-t1, len(myobj))) + #print("Processing data") + #t1 = time.time() + syndatasetlen = min(65536 // args.mini_batch_size, nbatches) + #myobj = list(enumerate(train_ld)) + #t2 = time.time() + #print("Processing data takes {} seconds with len={} {} {} {}".format(t2-t1, len(myobj), nbatches, args.mini_batch_size, syndatasetlen)) print("time/loss/accuracy (if enabled):") with torch.autograd.profiler.profile(args.enable_profiling, use_gpu, record_shapes=True) as prof: + # with torch.autograd.profiler.emit_nvtx(): + while k < args.nepochs: if k < skip_upto_epoch: continue - accum_time_begin = time_wrap(use_gpu) + if use_gpu: + tm.tmSync1.start() + torch.cuda.synchronize() + tm.tmSync1.stop() + accum_time_begin = time.time() if args.mlperf_logging: previous_iteration_time = None @@ -1183,7 +1192,8 @@ def loss_fn_wrap(Z, T, use_gpu, device): # for j, (X, lS_o, lS_i, T) in enumerate(train_ld): for j in range(nbatches): tm.tmGetData.start() - X, lS_o, lS_i, T = myobj[j][1] + # X, lS_o, lS_i, T = myobj[j%syndatasetlen][1] + X, lS_o, lS_i, T = train_data.__getitem__(j%syndatasetlen) tm.tmGetData.stop() if j == 0 and args.save_onnx: @@ -1196,6 +1206,8 @@ def loss_fn_wrap(Z, T, use_gpu, device): ext_dist.barrier() startTime = time.time() ext_dist.orig_print("ORIG TIME: ", startTime, accum_time_begin, startTime - accum_time_begin, " for process ", ext_dist.my_rank) + # torch.cuda.profiler.cudart().cudaProfilerStart() + torch.cuda.profiler.start() tm.tmClear() skipped = skipped + 1 @@ -1207,7 +1219,11 @@ def loss_fn_wrap(Z, T, use_gpu, device): iteration_time = 0 previous_iteration_time = current_time else: - t1 = time_wrap(use_gpu) + if use_gpu: + tm.tmSync2.start() + torch.cuda.synchronize() + tm.tmSync2.stop() + t1 = time.time() # early exit if nbatches was set by the user and has been exceeded if nbatches > 0 and j >= nbatches: @@ -1276,8 +1292,13 @@ def loss_fn_wrap(Z, T, use_gpu, device): if args.mlperf_logging: total_time += iteration_time else: - t2 = time_wrap(use_gpu) + if use_gpu: + tm.tmSync3.start() + torch.cuda.synchronize() + tm.tmSync3.stop() + t2 = time.time() total_time += t2 - t1 + total_accu += A total_loss += L * mbs total_iter += 1 @@ -1311,7 +1332,7 @@ def loss_fn_wrap(Z, T, use_gpu, device): ) # Uncomment the line below to print out the total time with overhead if ext_dist.my_rank < 2: - tt1 = time_wrap(use_gpu) + tt1 = time.time() ext_dist.orig_print("Accumulated time so far: {} for process {} for step {} at {}" \ .format(tt1 - accum_time_begin, ext_dist.my_rank, skipped, tt1)) total_iter = 0 @@ -1518,6 +1539,8 @@ def loss_fn_wrap(Z, T, use_gpu, device): ext_dist.barrier() tt3 = time.time() finalTime = tt3 - startTime + # torch.cuda.profiler.cudart().cudaProfilerStop() + torch.cuda.profiler.stop() if (skipped > 2): skipped -= 2 ext_dist.orig_print("Process {} Done with total time {:.6f} measure time {:.6f}s {:.6f}s, \ From 61875e4798dfb6d8332cd8fdfc96d51016c5f06d Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Sun, 13 Dec 2020 19:13:29 -0800 Subject: [PATCH 55/57] hack data access --- dlrm_profile.py | 1595 +++++++++++++++++++++++++++++++++++++++++++++++ tt.py | 29 +- 2 files changed, 1614 insertions(+), 10 deletions(-) create mode 100644 dlrm_profile.py diff --git a/dlrm_profile.py b/dlrm_profile.py new file mode 100644 index 00000000..152dce83 --- /dev/null +++ b/dlrm_profile.py @@ -0,0 +1,1595 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. +# +# Description: an implementation of a deep learning recommendation model (DLRM) +# The model input consists of dense and sparse features. The former is a vector +# of floating point values. The latter is a list of sparse indices into +# embedding tables, which consist of vectors of floating point values. +# The selected vectors are passed to mlp networks denoted by triangles, +# in some cases the vectors are interacted through operators (Ops). +# +# output: +# vector of values +# model: | +# /\ +# /__\ +# | +# _____________________> Op <___________________ +# / | \ +# /\ /\ /\ +# /__\ /__\ ... /__\ +# | | | +# | Op Op +# | ____/__\_____ ____/__\____ +# | |_Emb_|____|__| ... |_Emb_|__|___| +# input: +# [ dense features ] [sparse indices] , ..., [sparse indices] +# +# More precise definition of model layers: +# 1) fully connected layers of an mlp +# z = f(y) +# y = Wx + b +# +# 2) embedding lookup (for a list of sparse indices p=[p1,...,pk]) +# z = Op(e1,...,ek) +# obtain vectors e1=E[:,p1], ..., ek=E[:,pk] +# +# 3) Operator Op can be one of the following +# Sum(e1,...,ek) = e1 + ... + ek +# Dot(e1,...,ek) = [e1'e1, ..., e1'ek, ..., ek'e1, ..., ek'ek] +# Cat(e1,...,ek) = [e1', ..., ek']' +# where ' denotes transpose operation +# +# References: +# [1] Maxim Naumov, Dheevatsa Mudigere, Hao-Jun Michael Shi, Jianyu Huang, +# Narayanan Sundaram, Jongsoo Park, Xiaodong Wang, Udit Gupta, Carole-Jean Wu, +# Alisson G. Azzolini, Dmytro Dzhulgakov, Andrey Mallevich, Ilia Cherniavskii, +# Yinghai Lu, Raghuraman Krishnamoorthi, Ansha Yu, Volodymyr Kondratenko, +# Stephanie Pereira, Xianjie Chen, Wenlin Chen, Vijay Rao, Bill Jia, Liang Xiong, +# Misha Smelyanskiy, "Deep Learning Recommendation Model for Personalization and +# Recommendation Systems", CoRR, arXiv:1906.00091, 2019 + +from __future__ import absolute_import, division, print_function, unicode_literals + +# miscellaneous +import builtins +import functools +# import bisect +# import shutil +import time +import json +# data generation +import dlrm_data_pytorch as dp + +# numpy +import numpy as np +import socket + +# onnx +# The onnx import causes deprecation warnings every time workers +# are spawned during testing. So, we filter out those warnings. +import warnings +with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) +## import onnx + +# pytorch +import torch +from torch import onnx +import torch.nn as nn +from torch.nn.parallel.parallel_apply import parallel_apply +from torch.nn.parallel.replicate import replicate +from torch.nn.parallel.scatter_gather import gather, scatter + +# For distributed run +import extend_distributed as ext_dist + +# quotient-remainder trick +from tricks.qr_embedding_bag import QREmbeddingBag +# mixed-dimension trick +from tricks.md_embedding_bag import PrEmbeddingBag, md_solver + +import sklearn.metrics + +import uuid +import project +from torch.nn.parallel import DistributedDataParallel as DDP + +import dlrm_data as dd + +# Add dlrm self profiling timers +import profile as tm +# import pyprof +# pyprof.init() # causing errors, some symbols not found + +# import synthetic_data_loader as fb_syn_data + +# from torchviz import make_dot +# import torch.nn.functional as Functional +# from torch.nn.parameter import Parameter + +from torch.optim.lr_scheduler import _LRScheduler + +exc = getattr(builtins, "IOError", "FileNotFoundError") + +class LRPolicyScheduler(_LRScheduler): + def __init__(self, optimizer, num_warmup_steps, decay_start_step, num_decay_steps): + self.num_warmup_steps = num_warmup_steps + self.decay_start_step = decay_start_step + self.decay_end_step = decay_start_step + num_decay_steps + self.num_decay_steps = num_decay_steps + + if self.decay_start_step < self.num_warmup_steps: + sys.exit("Learning rate warmup must finish before the decay starts") + + super(LRPolicyScheduler, self).__init__(optimizer) + + def get_lr(self): + step_count = self._step_count + if step_count < self.num_warmup_steps: + # warmup + scale = 1.0 - (self.num_warmup_steps - step_count) / self.num_warmup_steps + lr = [base_lr * scale for base_lr in self.base_lrs] + self.last_lr = lr + elif self.decay_start_step <= step_count and step_count < self.decay_end_step: + # decay + decayed_steps = step_count - self.decay_start_step + scale = ((self.num_decay_steps - decayed_steps) / self.num_decay_steps) ** 2 + min_lr = 0.0000001 + lr = [max(min_lr, base_lr * scale) for base_lr in self.base_lrs] + self.last_lr = lr + else: + if self.num_decay_steps > 0: + # freeze at last, either because we're after decay + # or because we're between warmup and decay + lr = self.last_lr + else: + # do not adjust + lr = self.base_lrs + return lr + +### define dlrm in PyTorch ### +class DLRM_Net(nn.Module): + def create_mlp(self, ln, sigmoid_layer): + # build MLP layer by layer + layers = nn.ModuleList() + for i in range(0, ln.size - 1): + n = ln[i] + m = ln[i + 1] + + # construct fully connected operator + LL = nn.Linear(int(n), int(m), bias=True) + + # initialize the weights + # with torch.no_grad(): + # custom Xavier input, output or two-sided fill + mean = 0.0 # std_dev = np.sqrt(variance) + std_dev = np.sqrt(2 / (m + n)) # np.sqrt(1 / m) # np.sqrt(1 / n) + W = np.random.normal(mean, std_dev, size=(m, n)).astype(np.float32) + std_dev = np.sqrt(1 / m) # np.sqrt(2 / (m + 1)) + bt = np.random.normal(mean, std_dev, size=m).astype(np.float32) + # approach 1 + LL.weight.data = torch.tensor(W, requires_grad=True) + LL.bias.data = torch.tensor(bt, requires_grad=True) + # approach 2 + # LL.weight.data.copy_(torch.tensor(W)) + # LL.bias.data.copy_(torch.tensor(bt)) + # approach 3 + # LL.weight = Parameter(torch.tensor(W),requires_grad=True) + # LL.bias = Parameter(torch.tensor(bt),requires_grad=True) + layers.append(LL) + + # construct sigmoid or relu operator + if i == sigmoid_layer: + layers.append(nn.Sigmoid()) + else: + layers.append(nn.ReLU()) + + # approach 1: use ModuleList + # return layers + # approach 2: use Sequential container to wrap all layers + return torch.nn.Sequential(*layers) + + def create_emb(self, m, ln): + emb_l = nn.ModuleList() + # save the numpy random state + np_rand_state = np.random.get_state() + for i in range(0, ln.size): + if ext_dist.my_size > 1: + if not i in self.local_emb_indices: continue + # Use per table random seed for Embedding initialization + np.random.seed(self.l_emb_seeds[i]) + n = ln[i] + # construct embedding operator + if self.qr_flag and n > self.qr_threshold: + EE = QREmbeddingBag(n, m, self.qr_collisions, + operation=self.qr_operation, mode="sum", sparse=True) + elif self.md_flag: + base = max(m) + _m = m[i] if n > self.md_threshold else base + EE = PrEmbeddingBag(n, _m, base) + # use np initialization as below for consistency... + W = np.random.uniform( + low=-np.sqrt(1 / n), high=np.sqrt(1 / n), size=(n, _m) + ).astype(np.float32) + EE.embs.weight.data = torch.tensor(W, requires_grad=True) + + else: + #_weight = torch.empty([n, m]).uniform_(-np.sqrt(1 / n), np.sqrt(1 / n)) + #EE = nn.EmbeddingBag(n, m, mode="sum", sparse=True, _weight= _weight) + #EE = nn.EmbeddingBag(n, m, mode="sum", sparse=True) + + # initialize embeddings + # nn.init.uniform_(EE.weight, a=-np.sqrt(1 / n), b=np.sqrt(1 / n)) + W = np.random.uniform( + low=-np.sqrt(1 / n), high=np.sqrt(1 / n), size=(n, m) + ).astype(np.float32) + # approach 1 + EE = nn.EmbeddingBag(n, m, mode="sum", sparse=True, _weight=torch.tensor(W, requires_grad=True)) + #EE.weight.data = torch.tensor(W, requires_grad=True) + # approach 2 + # EE.weight.data.copy_(torch.tensor(W)) + # approach 3 + # EE.weight = Parameter(torch.tensor(W),requires_grad=True) + + if ext_dist.my_size > 1: + if i in self.local_emb_indices: + emb_l.append(EE) + else: + emb_l.append(EE) + + # Restore the numpy random state + np.random.set_state(np_rand_state) + return emb_l + + def __init__( + self, + m_spa=None, + ln_emb=None, + ln_bot=None, + ln_top=None, + proj_size = 0, + arch_interaction_op=None, + arch_interaction_itself=False, + sigmoid_bot=-1, + sigmoid_top=-1, + sync_dense_params=True, + loss_threshold=0.0, + ndevices=-1, + qr_flag=False, + qr_operation="mult", + qr_collisions=0, + qr_threshold=200, + md_flag=False, + md_threshold=200, + ): + super(DLRM_Net, self).__init__() + + if ( + (m_spa is not None) + and (ln_emb is not None) + and (ln_bot is not None) + and (ln_top is not None) + and (arch_interaction_op is not None) + ): + + # save arguments + self.proj_size = proj_size + self.ndevices = ndevices + self.output_d = 0 + self.parallel_model_batch_size = -1 + self.parallel_model_is_not_prepared = True + self.arch_interaction_op = arch_interaction_op + self.arch_interaction_itself = arch_interaction_itself + self.sync_dense_params = sync_dense_params + self.loss_threshold = loss_threshold + # create variables for QR embedding if applicable + self.qr_flag = qr_flag + if self.qr_flag: + self.qr_collisions = qr_collisions + self.qr_operation = qr_operation + self.qr_threshold = qr_threshold + # create variables for MD embedding if applicable + self.md_flag = md_flag + if self.md_flag: + self.md_threshold = md_threshold + + # generate np seeds for Emb table initialization + self.l_emb_seeds = np.random.randint(low=0, high=100000, size=len(ln_emb)) + + #If running distributed, get local slice of embedding tables + if ext_dist.my_size > 1: + n_emb = len(ln_emb) + self.n_global_emb = n_emb + self.n_local_emb, self.n_emb_per_rank = ext_dist.get_split_lengths(n_emb) + self.local_emb_slice = ext_dist.get_my_slice(n_emb) + self.local_emb_indices = list(range(n_emb))[self.local_emb_slice] + #ln_emb = ln_emb[self.local_emb_slice] + + # create operators + if ndevices <= 1: + self.emb_l = self.create_emb(m_spa, ln_emb) + self.bot_l = self.create_mlp(ln_bot, sigmoid_bot) + self.top_l = self.create_mlp(ln_top, sigmoid_top) + if (proj_size > 0): + self.proj_l = project.create_proj(len(ln_emb)+1, proj_size) + + def apply_mlp(self, x, layers): + # approach 1: use ModuleList + # for layer in layers: + # x = layer(x) + # return x + # approach 2: use Sequential container to wrap all layers + return layers(x) + + def apply_proj(self, x, layers): + # approach 1: use ModuleList + # for layer in layers: + # x = layer(x) + # return x + # approach 2: use Sequential container to wrap all layers + return layers(x) + + def apply_emb(self, lS_o, lS_i, emb_l): + # WARNING: notice that we are processing the batch at once. We implicitly + # assume that the data is laid out such that: + # 1. each embedding is indexed with a group of sparse indices, + # corresponding to a single lookup + # 2. for each embedding the lookups are further organized into a batch + # 3. for a list of embedding tables there is a list of batched lookups + + ly = [] + for k, sparse_index_group_batch in enumerate(lS_i): + sparse_offset_group_batch = lS_o[k] + + # embedding lookup + # We are using EmbeddingBag, which implicitly uses sum operator. + # The embeddings are represented as tall matrices, with sum + # happening vertically across 0 axis, resulting in a row vector + E = emb_l[k] + V = E(sparse_index_group_batch, sparse_offset_group_batch) + + ly.append(V) + + # print(ly) + return ly + + def interact_features(self, x, ly): + if self.arch_interaction_op == "dot": + # concatenate dense and sparse features + (batch_size, d) = x.shape + T = torch.cat([x] + ly, dim=1).view((batch_size, -1, d)) + # perform a dot product + if (self.proj_size > 0): + R = project.project(T, x, self.proj_l) + #TT = torch.transpose(T, 1, 2) + #TS = torch.reshape(TT, (-1, TT.size(2))) + #TC = self.apply_mlp(TS, self.proj_l) + #TR = torch.reshape(TC, (-1, d ,self.proj_size)) + #Z = torch.bmm(T, TR) + #Zflat = Z.view((batch_size, -1)) + #R = torch.cat([x] + [Zflat], dim=1) + else: + Z = torch.bmm(T, torch.transpose(T, 1, 2)) + # append dense feature with the interactions (into a row vector) + # approach 1: all + # Zflat = Z.view((batch_size, -1)) + # approach 2: unique + _, ni, nj = Z.shape + # approach 1: tril_indices + # offset = 0 if self.arch_interaction_itself else -1 + # li, lj = torch.tril_indices(ni, nj, offset=offset) + # approach 2: custom + offset = 1 if self.arch_interaction_itself else 0 + li = torch.tensor([i for i in range(ni) for j in range(i + offset)]) + lj = torch.tensor([j for i in range(nj) for j in range(i + offset)]) + Zflat = Z[:, li, lj] + # concatenate dense features and interactions + R = torch.cat([x] + [Zflat], dim=1) + elif self.arch_interaction_op == "cat": + # concatenation features (into a row vector) + R = torch.cat([x] + ly, dim=1) + else: + sys.exit( + "ERROR: --arch-interaction-op=" + + self.arch_interaction_op + + " is not supported" + ) + + return R + + def forward(self, dense_x, lS_o, lS_i): + if ext_dist.my_size > 1: + return self.distributed_forward(dense_x, lS_o, lS_i) + elif self.ndevices <= 1: + return self.sequential_forward(dense_x, lS_o, lS_i) + else: + return self.parallel_forward(dense_x, lS_o, lS_i) + + def sequential_forward(self, dense_x, lS_o, lS_i): + # process dense features (using bottom mlp), resulting in a row vector + x = self.apply_mlp(dense_x, self.bot_l) + # debug prints + # print("intermediate") + # print(x.detach().cpu().numpy()) + + # process sparse features(using embeddings), resulting in a list of row vectors + ly = self.apply_emb(lS_o, lS_i, self.emb_l) + # for y in ly: + # print(y.detach().cpu().numpy()) + + # interact features (dense and sparse) + z = self.interact_features(x, ly) + # print(z.detach().cpu().numpy()) + + # obtain probability of a click (using top mlp) + p = self.apply_mlp(z, self.top_l) + + # clamp output if needed + if 0.0 < self.loss_threshold and self.loss_threshold < 1.0: + z = torch.clamp(p, min=self.loss_threshold, max=(1.0 - self.loss_threshold)) + else: + z = p + + return z + + def distributed_forward(self, dense_x, lS_o, lS_i): + batch_size = dense_x.size()[0] + # WARNING: # of ranks must be <= batch size in distributed_forward call + if batch_size < ext_dist.my_size: + sys.exit("ERROR: batch_size (%d) must be larger than number of ranks (%d)" % (batch_size, ext_dist.my_size)) + if batch_size % ext_dist.my_size != 0: + sys.exit("ERROR: batch_size %d can not split across %d ranks evenly" % (batch_size, ext_dist.my_size)) + + dense_x = dense_x[ext_dist.get_my_slice(batch_size)] + lS_o = lS_o[self.local_emb_slice] + lS_i = lS_i[self.local_emb_slice] + + if (len(self.emb_l) != len(lS_o)) or (len(self.emb_l) != len(lS_i)): + sys.exit("ERROR: corrupted model input detected in distributed_forward call") + + # embeddings + tm.tmEmb.start() + ly = self.apply_emb(lS_o, lS_i, self.emb_l) + tm.tmEmb.stop() + + # print("ly: ", ly) + # debug prints + # print(ly) + + # WARNING: Note that at this point we have the result of the embedding lookup + # for the entire batch on each rank. We would like to obtain partial results + # corresponding to all embedding lookups, but part of the batch on each rank. + # Therefore, matching the distribution of output of bottom mlp, so that both + # could be used for subsequent interactions on each device. + if len(self.emb_l) != len(ly): + sys.exit("ERROR: corrupted intermediate result in distributed_forward call") + + tm.tmA2A.start() + a2a_req = ext_dist.alltoall(ly, self.n_emb_per_rank) + tm.tmA2A.stop() + + tm.tmBot.start() + x = self.apply_mlp(dense_x, self.bot_l) + tm.tmBot.stop() + + # debug prints + # print(x) + + tm.tmA2A1.start() + ly = a2a_req.wait() + tm.tmA2A1.stop() + # print("ly: ", ly) + ly = list(ly) + + # interactions + tm.tmInt.start() + z = self.interact_features(x, ly) + tm.tmInt.stop() + # debug prints + # print(z) + + # top mlp + tm.tmTop.start() + p = self.apply_mlp(z, self.top_l) + tm.tmTop.stop() + + # clamp output if needed + if 0.0 < self.loss_threshold and self.loss_threshold < 1.0: + z = torch.clamp( + p, min=self.loss_threshold, max=(1.0 - self.loss_threshold) + ) + else: + z = p + + ### gather the distributed results on each rank ### + # For some reason it requires explicit sync before all_gather call if + # tensor is on GPU memory + tm.tmAllGa.start() + if z.is_cuda: torch.cuda.synchronize() + (_, batch_split_lengths) = ext_dist.get_split_lengths(batch_size) + z = ext_dist.all_gather(z, batch_split_lengths) + tm.tmAllGa.stop() + #print("Z: %s" % z) + + return z + + def parallel_forward(self, dense_x, lS_o, lS_i): + ### prepare model (overwrite) ### + # WARNING: # of devices must be >= batch size in parallel_forward call + batch_size = dense_x.size()[0] + ndevices = min(self.ndevices, batch_size, len(self.emb_l)) + device_ids = range(ndevices) + # WARNING: must redistribute the model if mini-batch size changes(this is common + # for last mini-batch, when # of elements in the dataset/batch size is not even + if self.parallel_model_batch_size != batch_size: + self.parallel_model_is_not_prepared = True + + if self.parallel_model_is_not_prepared or self.sync_dense_params: + # replicate mlp (data parallelism) + self.bot_l_replicas = replicate(self.bot_l, device_ids) + self.top_l_replicas = replicate(self.top_l, device_ids) + self.parallel_model_batch_size = batch_size + + if self.parallel_model_is_not_prepared: + # distribute embeddings (model parallelism) + t_list = [] + for k, emb in enumerate(self.emb_l): + d = torch.device("cuda:" + str(k % ndevices)) + emb.to(d) + t_list.append(emb.to(d)) + self.emb_l = nn.ModuleList(t_list) + self.parallel_model_is_not_prepared = False + + ### prepare input (overwrite) ### + # scatter dense features (data parallelism) + # print(dense_x.device) + dense_x = scatter(dense_x, device_ids, dim=0) + # distribute sparse features (model parallelism) + if (len(self.emb_l) != len(lS_o)) or (len(self.emb_l) != len(lS_i)): + sys.exit("ERROR: corrupted model input detected in parallel_forward call") + + t_list = [] + i_list = [] + for k, _ in enumerate(self.emb_l): + d = torch.device("cuda:" + str(k % ndevices)) + t_list.append(lS_o[k].to(d)) + i_list.append(lS_i[k].to(d)) + lS_o = t_list + lS_i = i_list + + ### compute results in parallel ### + # bottom mlp + # WARNING: Note that the self.bot_l is a list of bottom mlp modules + # that have been replicated across devices, while dense_x is a tuple of dense + # inputs that has been scattered across devices on the first (batch) dimension. + # The output is a list of tensors scattered across devices according to the + # distribution of dense_x. + x = parallel_apply(self.bot_l_replicas, dense_x, None, device_ids) + # debug prints + # print(x) + + # embeddings + ly = self.apply_emb(lS_o, lS_i, self.emb_l) + # debug prints + # print(ly) + + # butterfly shuffle (implemented inefficiently for now) + # WARNING: Note that at this point we have the result of the embedding lookup + # for the entire batch on each device. We would like to obtain partial results + # corresponding to all embedding lookups, but part of the batch on each device. + # Therefore, matching the distribution of output of bottom mlp, so that both + # could be used for subsequent interactions on each device. + if len(self.emb_l) != len(ly): + sys.exit("ERROR: corrupted intermediate result in parallel_forward call") + + t_list = [] + for k, _ in enumerate(self.emb_l): + d = torch.device("cuda:" + str(k % ndevices)) + y = scatter(ly[k], device_ids, dim=0) + t_list.append(y) + # adjust the list to be ordered per device + ly = list(map(lambda y: list(y), zip(*t_list))) + # debug prints + # print(ly) + + # interactions + z = [] + for k in range(ndevices): + zk = self.interact_features(x[k], ly[k]) + z.append(zk) + # debug prints + # print(z) + + # top mlp + # WARNING: Note that the self.top_l is a list of top mlp modules that + # have been replicated across devices, while z is a list of interaction results + # that by construction are scattered across devices on the first (batch) dim. + # The output is a list of tensors scattered across devices according to the + # distribution of z. + p = parallel_apply(self.top_l_replicas, z, None, device_ids) + + ### gather the distributed results ### + p0 = gather(p, self.output_d, dim=0) + + # clamp output if needed + if 0.0 < self.loss_threshold and self.loss_threshold < 1.0: + z0 = torch.clamp( + p0, min=self.loss_threshold, max=(1.0 - self.loss_threshold) + ) + else: + z0 = p0 + + return z0 + + +def dash_separated_ints(value): + vals = value.split('-') + for val in vals: + try: + int(val) + except ValueError: + raise argparse.ArgumentTypeError( + "%s is not a valid dash separated list of ints" % value) + + return value + + +def dash_separated_floats(value): + vals = value.split('-') + for val in vals: + try: + float(val) + except ValueError: + raise argparse.ArgumentTypeError( + "%s is not a valid dash separated list of floats" % value) + + return value + + +if __name__ == "__main__": + ### import packages ### + import sys + import os + import argparse + + ### parse arguments ### + parser = argparse.ArgumentParser( + description="Train Deep Learning Recommendation Model (DLRM)" + ) + # model related parameters + parser.add_argument("--arch-sparse-feature-size", type=int, default=2) + + parser.add_argument( + "--arch-embedding-size", type=dash_separated_ints, default="4-3-2") + parser.add_argument("--arch-project-size", type=int, default=0) + + # j will be replaced with the table number + parser.add_argument( + "--arch-mlp-bot", type=dash_separated_ints, default="4-3-2") + parser.add_argument( + "--arch-mlp-top", type=dash_separated_ints, default="4-2-1") + parser.add_argument( + "--arch-interaction-op", type=str, choices=['dot', 'cat'], default="dot") + parser.add_argument("--arch-interaction-itself", action="store_true", default=False) + # embedding table options + parser.add_argument("--md-flag", action="store_true", default=False) + parser.add_argument("--md-threshold", type=int, default=200) + parser.add_argument("--md-temperature", type=float, default=0.3) + parser.add_argument("--md-round-dims", action="store_true", default=False) + parser.add_argument("--qr-flag", action="store_true", default=False) + parser.add_argument("--qr-threshold", type=int, default=200) + parser.add_argument("--qr-operation", type=str, default="mult") + parser.add_argument("--qr-collisions", type=int, default=4) + # activations and loss + parser.add_argument("--activation-function", type=str, default="relu") + parser.add_argument("--loss-function", type=str, default="mse") # or bce or wbce + parser.add_argument( + "--loss-weights", type=dash_separated_floats, default="1.0-1.0") # for wbce + parser.add_argument("--loss-threshold", type=float, default=0.0) # 1.0e-7 + parser.add_argument("--round-targets", type=bool, default=False) + # data + parser.add_argument("--data-size", type=int, default=1) + parser.add_argument("--num-batches", type=int, default=0) + parser.add_argument( + "--data-generation", type=str, default="random" + ) # synthetic or dataset + parser.add_argument("--synthetic-data-folder", type=str, + default="./synthetic_data/syn_data_bs65536") + # add Gaussian distribution + parser.add_argument("--rand-data-dist", type=str, default="uniform") # uniform or gaussian + parser.add_argument("--rand-data-min", type=float, default=0) + parser.add_argument("--rand-data-max", type=float, default=1) + parser.add_argument("--rand-data-mu", type=float, default=-1) + parser.add_argument("--rand-data-sigma", type=float, default=1) + + parser.add_argument("--data-trace-file", type=str, default="./input/dist_emb_j.log") + parser.add_argument("--data-set", type=str, default="kaggle") # or terabyte + parser.add_argument("--raw-data-file", type=str, default="") + parser.add_argument("--processed-data-file", type=str, default="") + parser.add_argument("--data-randomize", type=str, default="total") # or day or none + parser.add_argument("--data-trace-enable-padding", type=bool, default=False) + parser.add_argument("--max-ind-range", type=int, default=-1) + parser.add_argument("--data-sub-sample-rate", type=float, default=0.0) # in [0, 1] + parser.add_argument("--num-indices-per-lookup", type=int, default=10) + parser.add_argument("--num-indices-per-lookup-fixed", type=bool, default=False) + parser.add_argument("--num-workers", type=int, default=0) + parser.add_argument("--memory-map", action="store_true", default=False) + # training + parser.add_argument("--mini-batch-size", type=int, default=1) + parser.add_argument("--nepochs", type=int, default=1) + parser.add_argument("--learning-rate", type=float, default=0.01) + parser.add_argument("--print-precision", type=int, default=5) + parser.add_argument("--numpy-rand-seed", type=int, default=123) + parser.add_argument("--sync-dense-params", type=bool, default=True) + # inference + parser.add_argument("--inference-only", action="store_true", default=False) + # onnx + parser.add_argument("--save-onnx", action="store_true", default=False) + # gpu + parser.add_argument("--use-gpu", action="store_true", default=False) + # distributed run + parser.add_argument("--dist-backend", type=str, default="") + # debugging and profiling + parser.add_argument("--print-freq", type=int, default=1) + parser.add_argument("--test-freq", type=int, default=-1) + parser.add_argument("--test-mini-batch-size", type=int, default=-1) + parser.add_argument("--test-num-workers", type=int, default=-1) + parser.add_argument("--print-time", action="store_true", default=False) + parser.add_argument("--debug-mode", action="store_true", default=False) + parser.add_argument("--enable-profiling", action="store_true", default=False) + parser.add_argument("--plot-compute-graph", action="store_true", default=False) + # store/load model + parser.add_argument("--out-dir", type=str, default=".") + parser.add_argument("--save-model", type=str, default="") + parser.add_argument("--load-model", type=str, default="") + # mlperf logging (disables other output and stops early) + parser.add_argument("--mlperf-logging", action="store_true", default=False) + # stop at target accuracy Kaggle 0.789, Terabyte (sub-sampled=0.875) 0.8107 + parser.add_argument("--mlperf-acc-threshold", type=float, default=0.0) + # stop at target AUC Terabyte (no subsampling) 0.8025 + parser.add_argument("--mlperf-auc-threshold", type=float, default=0.0) + parser.add_argument("--mlperf-bin-loader", action='store_true', default=False) + parser.add_argument("--mlperf-bin-shuffle", action='store_true', default=False) + + # LR policy + parser.add_argument("--lr-num-warmup-steps", type=int, default=0) + parser.add_argument("--lr-decay-start-step", type=int, default=0) + parser.add_argument("--lr-num-decay-steps", type=int, default=0) + + args = parser.parse_args() + + print(socket.gethostname()) + + ext_dist.init_distributed(backend=args.dist_backend) + + # print("success size= ", ext_dist.my_size, ext_dist.my_rank) + + ext_dist.barrier() + + if args.mlperf_logging: + print('command line args: ', json.dumps(vars(args))) + + ### some basic setup ### + np.random.seed(args.numpy_rand_seed) + np.set_printoptions(precision=args.print_precision) + torch.set_printoptions(precision=args.print_precision) + torch.manual_seed(args.numpy_rand_seed) + + if (args.test_mini_batch_size < 0): + # if the parameter is not set, use the training batch size + args.test_mini_batch_size = args.mini_batch_size + if (args.test_num_workers < 0): + # if the parameter is not set, use the same parameter for training + args.test_num_workers = args.num_workers + if args.mini_batch_size % ext_dist.my_size !=0 or args.test_mini_batch_size % ext_dist.my_size != 0: + print("Either test minibatch (%d) or train minibatch (%d) does not split across %d ranks" % (args.test_mini_batch_size, args.mini_batch_size, ext_dist.my_size)) + sys.exit(1) + + use_gpu = args.use_gpu and torch.cuda.is_available() + if use_gpu: + torch.cuda.manual_seed_all(args.numpy_rand_seed) + torch.backends.cudnn.deterministic = True + if ext_dist.my_size > 1: + ngpus = torch.cuda.device_count() # 1 + if ext_dist.my_local_size > torch.cuda.device_count(): + print("Not sufficient GPUs available... local_size = %d, ngpus = %d" % (ext_dist.my_local_size, ngpus)) + sys.exit(1) + ngpus = 1 + device = torch.device("cuda", ext_dist.my_local_rank) + else: + device = torch.device("cuda", 0) + ngpus = torch.cuda.device_count() # 1 + ngpus=1 + print("Using {} GPU(s)...".format(ngpus)) + else: + device = torch.device("cpu") + print("Using CPU...") + + ### prepare training data ### + ln_bot = np.fromstring(args.arch_mlp_bot, dtype=int, sep="-") + # input data + if (args.data_generation == "dataset"): + + train_data, train_ld, test_data, test_ld = \ + dp.make_criteo_data_and_loaders(args) + nbatches = args.num_batches if args.num_batches > 0 else len(train_ld) + nbatches_test = len(test_ld) + + ln_emb = train_data.counts + # enforce maximum limit on number of vectors per embedding + if args.max_ind_range > 0: + ln_emb = np.array(list(map( + lambda x: x if x < args.max_ind_range else args.max_ind_range, + ln_emb + ))) + m_den = train_data.m_den + ln_bot[0] = m_den + + elif args.data_generation == "synthetic": + # input and target at random + ln_emb = np.fromstring(args.arch_embedding_size, dtype=int, sep="-") + m_den = ln_bot[0] + train_data, train_ld = dd.data_loader(args, ln_emb, m_den) + nbatches = args.num_batches if args.num_batches > 0 else len(train_ld) + table_feature_map = None # {idx : idx for idx in range(len(ln_emb))} + + else: + # input and target at random + ln_emb = np.fromstring(args.arch_embedding_size, dtype=int, sep="-") + m_den = ln_bot[0] + train_data, train_ld = dd.make_random_data_and_loader(args, ln_emb, m_den) + nbatches = args.num_batches if args.num_batches > 0 else len(train_ld) + + ### parse command line arguments ### + m_spa = args.arch_sparse_feature_size + num_fea = ln_emb.size + 1 # num sparse + num dense features + m_den_out = ln_bot[ln_bot.size - 1] + if args.arch_interaction_op == "dot": + # approach 1: all + # num_int = num_fea * num_fea + m_den_out + # approach 2: unique + if (args.arch_project_size > 0): + num_int = num_fea * args.arch_project_size + m_den_out + else: + if args.arch_interaction_itself: + num_int = (num_fea * (num_fea + 1)) // 2 + m_den_out + else: + num_int = (num_fea * (num_fea - 1)) // 2 + m_den_out + elif args.arch_interaction_op == "cat": + num_int = num_fea * m_den_out + else: + sys.exit( + "ERROR: --arch-interaction-op=" + + args.arch_interaction_op + + " is not supported" + ) + arch_mlp_top_adjusted = str(num_int) + "-" + args.arch_mlp_top + ln_top = np.fromstring(arch_mlp_top_adjusted, dtype=int, sep="-") + + # sanity check: feature sizes and mlp dimensions must match + if m_den != ln_bot[0]: + sys.exit( + "ERROR: arch-dense-feature-size " + + str(m_den) + + " does not match first dim of bottom mlp " + + str(ln_bot[0]) + ) + if args.qr_flag: + if args.qr_operation == "concat" and 2 * m_spa != m_den_out: + sys.exit( + "ERROR: 2 arch-sparse-feature-size " + + str(2 * m_spa) + + " does not match last dim of bottom mlp " + + str(m_den_out) + + " (note that the last dim of bottom mlp must be 2x the embedding dim)" + ) + if args.qr_operation != "concat" and m_spa != m_den_out: + sys.exit( + "ERROR: arch-sparse-feature-size " + + str(m_spa) + + " does not match last dim of bottom mlp " + + str(m_den_out) + ) + else: + if m_spa != m_den_out: + sys.exit( + "ERROR: arch-sparse-feature-size " + + str(m_spa) + + " does not match last dim of bottom mlp " + + str(m_den_out) + ) + if num_int != ln_top[0]: + sys.exit( + "ERROR: # of feature interactions " + + str(num_int) + + " does not match first dimension of top mlp " + + str(ln_top[0]) + ) + + # assign mixed dimensions if applicable + if args.md_flag: + m_spa = md_solver( + torch.tensor(ln_emb), + args.md_temperature, # alpha + d0=m_spa, + round_dim=args.md_round_dims + ).tolist() + + # test prints (model arch) + if args.debug_mode: + print("model arch:") + print( + "mlp top arch " + + str(ln_top.size - 1) + + " layers, with input to output dimensions:" + ) + print(ln_top) + print("# of interactions") + print(num_int) + print( + "mlp bot arch " + + str(ln_bot.size - 1) + + " layers, with input to output dimensions:" + ) + print(ln_bot) + print("# of features (sparse and dense)") + print(num_fea) + print("dense feature size") + print(m_den) + print("sparse feature size") + print(m_spa) + print( + "# of embeddings (= # of sparse features) " + + str(ln_emb.size) + + ", with dimensions " + + str(m_spa) + + "x:" + ) + print(ln_emb) + + print("data (inputs and targets):") + for j, (X, lS_o, lS_i, T) in enumerate(train_ld): + # early exit if nbatches was set by the user and has been exceeded + if nbatches > 0 and j >= nbatches: + break + + print("mini-batch: %d" % j) + print(X.detach().cpu().numpy()) + # transform offsets to lengths when printing + print( + [ + np.diff( + S_o.detach().cpu().tolist() + list(lS_i[i].shape) + ).tolist() + for i, S_o in enumerate(lS_o) + ] + ) + print([S_i.detach().cpu().tolist() for S_i in lS_i]) + print(T.detach().cpu().numpy()) + + ndevices = min(ngpus, args.mini_batch_size, num_fea - 1) if use_gpu else -1 + + ### construct the neural network specified above ### + # WARNING: to obtain exactly the same initialization for + # the weights we need to start from the same random seed. + # np.random.seed(args.numpy_rand_seed) + dlrm = DLRM_Net( + m_spa, + ln_emb, + ln_bot, + ln_top, + args.arch_project_size, + arch_interaction_op=args.arch_interaction_op, + arch_interaction_itself=args.arch_interaction_itself, + sigmoid_bot=-1, + sigmoid_top=ln_top.size - 2, + sync_dense_params=args.sync_dense_params, + loss_threshold=args.loss_threshold, + ndevices=ndevices, + qr_flag=args.qr_flag, + qr_operation=args.qr_operation, + qr_collisions=args.qr_collisions, + qr_threshold=args.qr_threshold, + md_flag=args.md_flag, + md_threshold=args.md_threshold, + ) + # test prints + if args.debug_mode: + print("initial parameters (weights and bias):") + for param in dlrm.parameters(): + print(param.detach().cpu().numpy()) + # print(dlrm) + + if use_gpu: + # Custom Model-Data Parallel + # the mlps are replicated and use data parallelism, while + # the embeddings are distributed and use model parallelism + dlrm = dlrm.to(device) # .cuda() + if dlrm.ndevices > 1: + dlrm.emb_l = dlrm.create_emb(m_spa, ln_emb) + + if ext_dist.my_size > 1: + if use_gpu: + device_ids = [ext_dist.my_local_rank] + dlrm.bot_l = DDP(dlrm.bot_l, device_ids=device_ids) + dlrm.top_l = DDP(dlrm.top_l, device_ids=device_ids) + else: + dlrm.bot_l = DDP(dlrm.bot_l) + dlrm.top_l = DDP(dlrm.top_l) + + # specify the loss function + if args.loss_function == "mse": + loss_fn = torch.nn.MSELoss(reduction="mean") + elif args.loss_function == "bce": + loss_fn = torch.nn.BCELoss(reduction="mean") + elif args.loss_function == "wbce": + loss_ws = torch.tensor(np.fromstring(args.loss_weights, dtype=float, sep="-")) + loss_fn = torch.nn.BCELoss(reduction="none") + else: + sys.exit("ERROR: --loss-function=" + args.loss_function + " is not supported") + + if not args.inference_only: + # specify the optimizer algorithm + + if ext_dist.my_size == 1: + optimizer = torch.optim.SGD(dlrm.parameters(), lr=args.learning_rate) + #lr_scheduler = LRPolicyScheduler(optimizer, args.lr_num_warmup_steps, args.lr_decay_start_step, + # args.lr_num_decay_steps) + else: + optimizer = torch.optim.SGD([ + {"params": [p for emb in dlrm.emb_l for p in emb.parameters()], "lr" : args.learning_rate}, + {"params": dlrm.bot_l.parameters(), "lr" : args.learning_rate * ext_dist.my_size}, + {"params": dlrm.top_l.parameters(), "lr" : args.learning_rate * ext_dist.my_size} + ], lr=args.learning_rate) + + ### main loop ### + def time_wrap(use_gpu): + if use_gpu: + torch.cuda.synchronize() + return time.time() + + def dlrm_wrap(X, lS_o, lS_i, use_gpu, device): + if use_gpu: # .cuda() + # lS_i can be either a list of tensors or a stacked tensor. + # Handle each case below: + tm.tmH2D.start() + lS_i = [S_i.to(device) for S_i in lS_i] if isinstance(lS_i, list) \ + else lS_i.to(device) + lS_o = [S_o.to(device) for S_o in lS_o] if isinstance(lS_o, list) \ + else lS_o.to(device) + X = X.to(device) + tm.tmH2D.stop() + + return dlrm( + X, + lS_o, + lS_i + ) + else: + return dlrm(X, lS_o, lS_i) + + def loss_fn_wrap(Z, T, use_gpu, device): + if args.loss_function == "mse" or args.loss_function == "bce": + if use_gpu: + return loss_fn(Z, T.to(device)) + else: + return loss_fn(Z, T) + elif args.loss_function == "wbce": + if use_gpu: + loss_ws_ = loss_ws[T.data.view(-1).long()].view_as(T).to(device) + loss_fn_ = loss_fn(Z, T.to(device)) + else: + loss_ws_ = loss_ws[T.data.view(-1).long()].view_as(T) + loss_fn_ = loss_fn(Z, T.to(device)) + loss_sc_ = loss_ws_ * loss_fn_ + # debug prints + # print(loss_ws_) + # print(loss_fn_) + return loss_sc_.mean() + + # training or inference + best_gA_test = 0 + best_auc_test = 0 + skip_upto_epoch = 0 + skip_upto_batch = 0 + total_time = 0 + total_loss = 0 + total_accu = 0 + total_iter = 0 + total_samp = 0 + k = 0 + + # Load model is specified + if not (args.load_model == ""): + print("Loading saved model {}".format(args.load_model)) + if use_gpu: + if dlrm.ndevices > 1: + # NOTE: when targeting inference on multiple GPUs, + # load the model as is on CPU or GPU, with the move + # to multiple GPUs to be done in parallel_forward + ld_model = torch.load(args.load_model) + else: + # NOTE: when targeting inference on single GPU, + # note that the call to .to(device) has already happened + ld_model = torch.load( + args.load_model, + map_location=torch.device('cuda') + # map_location=lambda storage, loc: storage.cuda(0) + ) + else: + # when targeting inference on CPU + ld_model = torch.load(args.load_model, map_location=torch.device('cpu')) + dlrm.load_state_dict(ld_model["state_dict"]) + ld_j = ld_model["iter"] + ld_k = ld_model["epoch"] + ld_nepochs = ld_model["nepochs"] + ld_nbatches = ld_model["nbatches"] + ld_nbatches_test = ld_model["nbatches_test"] + ld_gA = ld_model["train_acc"] + ld_gL = ld_model["train_loss"] + ld_total_loss = ld_model["total_loss"] + ld_total_accu = ld_model["total_accu"] + ld_gA_test = ld_model["test_acc"] + ld_gL_test = ld_model["test_loss"] + if not args.inference_only: + optimizer.load_state_dict(ld_model["opt_state_dict"]) + best_gA_test = ld_gA_test + total_loss = ld_total_loss + total_accu = ld_total_accu + skip_upto_epoch = ld_k # epochs + skip_upto_batch = ld_j # batches + else: + args.print_freq = ld_nbatches + args.test_freq = 0 + + print( + "Saved at: epoch = {:d}/{:d}, batch = {:d}/{:d}, ntbatch = {:d}".format( + ld_k, ld_nepochs, ld_j, ld_nbatches, ld_nbatches_test + ) + ) + print( + "Training state: loss = {:.6f}, accuracy = {:3.3f} %".format( + ld_gL, ld_gA * 100 + ) + ) + print( + "Testing state: loss = {:.6f}, accuracy = {:3.3f} %".format( + ld_gL_test, ld_gA_test * 100 + ) + ) + + ext_dist.barrier() + startTime = time.time() + startTime0 = startTime + skipped = 0 + + #print("Processing data") + #t1 = time.time() + syndatasetlen = min(65536 // args.mini_batch_size, nbatches) + #myobj = list(enumerate(train_ld)) + #t2 = time.time() + #print("Processing data takes {} seconds with len={} {} {} {}".format(t2-t1, len(myobj), nbatches, args.mini_batch_size, syndatasetlen)) + print("time/loss/accuracy (if enabled):") + with torch.autograd.profiler.profile(args.enable_profiling, use_gpu, record_shapes=True) as prof: + # with torch.autograd.profiler.emit_nvtx(): + + while k < args.nepochs: + if k < skip_upto_epoch: + continue + + if use_gpu: + tm.tmSync1.start() + torch.cuda.synchronize() + tm.tmSync1.stop() + accum_time_begin = time.time() + + if args.mlperf_logging: + previous_iteration_time = None + + # for j, (X, lS_o, lS_i, T) in enumerate(train_ld): + for j in range(nbatches): + tm.tmGetData.start() + # X, lS_o, lS_i, T = myobj[j%syndatasetlen][1] + X, lS_o, lS_i, T = train_data.__getitem__(j%syndatasetlen) + tm.tmGetData.stop() + + if j == 0 and args.save_onnx: + (X_onnx, lS_o_onnx, lS_i_onnx) = (X, lS_o, lS_i) + + if j < skip_upto_batch: + continue + + if (skipped == 2): + ext_dist.barrier() + startTime = time.time() + ext_dist.orig_print("ORIG TIME: ", startTime, accum_time_begin, startTime - accum_time_begin, " for process ", ext_dist.my_rank) + # torch.cuda.profiler.cudart().cudaProfilerStart() + torch.cuda.profiler.start() + tm.tmClear() + skipped = skipped + 1 + + if args.mlperf_logging: + current_time = time_wrap(use_gpu) + if previous_iteration_time: + iteration_time = current_time - previous_iteration_time + else: + iteration_time = 0 + previous_iteration_time = current_time + else: + if use_gpu: + tm.tmSync2.start() + torch.cuda.synchronize() + tm.tmSync2.stop() + t1 = time.time() + + # early exit if nbatches was set by the user and has been exceeded + if nbatches > 0 and j >= nbatches: + break + ''' + # debug prints + print("input and targets") + print(X.detach().cpu().numpy()) + print([np.diff(S_o.detach().cpu().tolist() + + list(lS_i[i].shape)).tolist() for i, S_o in enumerate(lS_o)]) + print([S_i.detach().cpu().numpy().tolist() for S_i in lS_i]) + print(T.detach().cpu().numpy()) + ''' + # Skip the batch if batch size not multiple of total ranks + if ext_dist.my_size > 1 and X.size(0) % ext_dist.my_size != 0: + print("Warning: Skiping the batch %d with size %d" % (j, X.size(0))) + continue + + + # forward pass + tm.tmFwd.start() + Z = dlrm_wrap(X, lS_o, lS_i, use_gpu, device) + tm.tmFwd.stop() + + # loss + tm.tmLoss.start() + E = loss_fn_wrap(Z, T, use_gpu, device) + ''' + # debug prints + print("output and loss") + print(Z.detach().cpu().numpy()) + print(E.detach().cpu().numpy()) + ''' + # compute loss and accuracy + L = E.detach().cpu().numpy() # numpy array + S = Z.detach().cpu().numpy() # numpy array + T = T.detach().cpu().numpy() # numpy array + mbs = T.shape[0] # = args.mini_batch_size except maybe for last + A = np.sum((np.round(S, 0) == T).astype(np.uint8)) + tm.tmLoss.stop() + + if not args.inference_only: + # scaled error gradient propagation + # (where we do not accumulate gradients across mini-batches) + tm.tmZero.start() + optimizer.zero_grad() + tm.tmZero.stop() + + # backward pass + tm.tmBwd.start() + E.backward() + tm.tmBwd.stop() + + # debug prints (check gradient norm) + # for l in mlp.layers: + # if hasattr(l, 'weight'): + # print(l.weight.grad.norm().item()) + + # optimizer + tm.tmOpt.start() + optimizer.step() + tm.tmOpt.stop() + + ### lr_scheduler.step() + + if args.mlperf_logging: + total_time += iteration_time + else: + if use_gpu: + tm.tmSync3.start() + torch.cuda.synchronize() + tm.tmSync3.stop() + t2 = time.time() + total_time += t2 - t1 + + total_accu += A + total_loss += L * mbs + total_iter += 1 + total_samp += mbs + + should_print = ((j + 1) % args.print_freq == 0) or (j + 1 == nbatches) + should_test = ( + (args.test_freq > 0) + and (args.data_generation == "dataset") + and (((j + 1) % args.test_freq == 0) or (j + 1 == nbatches)) + ) + + # print time, loss and accuracy + if should_print or should_test: + gT = 1000.0 * total_time / total_iter if args.print_time else -1 + total_time = 0 + + gA = total_accu / total_samp + total_accu = 0 + + gL = total_loss / total_samp + total_loss = 0 + + str_run_type = "inference" if args.inference_only else "training" + print( + "Finished {} it {}/{} of epoch {}, {:.2f} ms/it, ".format( + str_run_type, j + 1, nbatches, k, gT + ) + + "loss {:.6f}, accuracy {:3.3f} % it {} for task {} ".format(gL, + gA * 100, total_iter, ext_dist.my_rank) + ) + # Uncomment the line below to print out the total time with overhead + if ext_dist.my_rank < 2: + tt1 = time.time() + ext_dist.orig_print("Accumulated time so far: {} for process {} for step {} at {}" \ + .format(tt1 - accum_time_begin, ext_dist.my_rank, skipped, tt1)) + total_iter = 0 + total_samp = 0 + + # testing + if should_test and not args.inference_only: + # don't measure training iter time in a test iteration + if args.mlperf_logging: + previous_iteration_time = None + + test_accu = 0 + test_loss = 0 + test_samp = 0 + + accum_test_time_begin = time_wrap(use_gpu) + if args.mlperf_logging: + scores = [] + targets = [] + + for i, (X_test, lS_o_test, lS_i_test, T_test) in enumerate(test_ld): + # early exit if nbatches was set by the user and was exceeded + if nbatches > 0 and i >= nbatches: + break + + # Skip the batch if batch size not multiple of total ranks + if ext_dist.my_size > 1 and X_test.size(0) % ext_dist.my_size != 0: + print("Warning: Skiping the batch %d with size %d" % (i, X_test.size(0))) + continue + + t1_test = time_wrap(use_gpu) + + # forward pass + Z_test = dlrm_wrap( + X_test, lS_o_test, lS_i_test, use_gpu, device + ) + if args.mlperf_logging: + S_test = Z_test.detach().cpu().numpy() # numpy array + T_test = T_test.detach().cpu().numpy() # numpy array + scores.append(S_test) + targets.append(T_test) + else: + # loss + E_test = loss_fn_wrap(Z_test, T_test, use_gpu, device) + + # compute loss and accuracy + L_test = E_test.detach().cpu().numpy() # numpy array + S_test = Z_test.detach().cpu().numpy() # numpy array + T_test = T_test.detach().cpu().numpy() # numpy array + mbs_test = T_test.shape[0] # = mini_batch_size except last + A_test = np.sum((np.round(S_test, 0) == T_test).astype(np.uint8)) + test_accu += A_test + test_loss += L_test * mbs_test + test_samp += mbs_test + + t2_test = time_wrap(use_gpu) + + if args.mlperf_logging: + scores = np.concatenate(scores, axis=0) + targets = np.concatenate(targets, axis=0) + + metrics = { + 'loss' : sklearn.metrics.log_loss, + 'recall' : lambda y_true, y_score: + sklearn.metrics.recall_score( + y_true=y_true, + y_pred=np.round(y_score) + ), + 'precision' : lambda y_true, y_score: + sklearn.metrics.precision_score( + y_true=y_true, + y_pred=np.round(y_score) + ), + 'f1' : lambda y_true, y_score: + sklearn.metrics.f1_score( + y_true=y_true, + y_pred=np.round(y_score) + ), + 'ap' : sklearn.metrics.average_precision_score, + 'roc_auc' : sklearn.metrics.roc_auc_score, + 'accuracy' : lambda y_true, y_score: + sklearn.metrics.accuracy_score( + y_true=y_true, + y_pred=np.round(y_score) + ), + # 'pre_curve' : sklearn.metrics.precision_recall_curve, + # 'roc_curve' : sklearn.metrics.roc_curve, + } + + # print("Compute time for validation metric : ", end="") + # first_it = True + validation_results = {} + for metric_name, metric_function in metrics.items(): + # if first_it: + # first_it = False + # else: + # print(", ", end="") + # metric_compute_start = time_wrap(False) + validation_results[metric_name] = metric_function( + targets, + scores + ) + # metric_compute_end = time_wrap(False) + # met_time = metric_compute_end - metric_compute_start + # print("{} {:.4f}".format(metric_name, 1000 * (met_time)), + # end="") + # print(" ms") + gA_test = validation_results['accuracy'] + gL_test = validation_results['loss'] + else: + gA_test = test_accu / test_samp + gL_test = test_loss / test_samp + + is_best = gA_test > best_gA_test + if is_best: + best_gA_test = gA_test + if not (args.save_model == ""): + print("Saving model to {}".format(args.save_model)) + torch.save( + { + "epoch": k, + "nepochs": args.nepochs, + "nbatches": nbatches, + "nbatches_test": nbatches_test, + "iter": j + 1, + "state_dict": dlrm.state_dict(), + "train_acc": gA, + "train_loss": gL, + "test_acc": gA_test, + "test_loss": gL_test, + "total_loss": total_loss, + "total_accu": total_accu, + "opt_state_dict": optimizer.state_dict(), + }, + args.save_model, + ) + + if args.mlperf_logging: + is_best = validation_results['roc_auc'] > best_auc_test + if is_best: + best_auc_test = validation_results['roc_auc'] + + print( + "Testing at - {}/{} of epoch {},".format(j + 1, nbatches, k) + + " loss {:.6f}, recall {:.4f}, precision {:.4f},".format( + validation_results['loss'], + validation_results['recall'], + validation_results['precision'] + ) + + " f1 {:.4f}, ap {:.4f},".format( + validation_results['f1'], + validation_results['ap'], + ) + + " auc {:.4f}, best auc {:.4f},".format( + validation_results['roc_auc'], + best_auc_test + ) + + " accuracy {:3.3f} %, best accuracy {:3.3f} %".format( + validation_results['accuracy'] * 100, + best_gA_test * 100 + ) + ) + else: + print( + "Testing at - {}/{} of epoch {},".format(j + 1, nbatches, 0) + + " loss {:.6f}, accuracy {:3.3f} %, best {:3.3f} %".format( + gL_test, gA_test * 100, best_gA_test * 100 + ) + ) + # Uncomment the line below to print out the total time with overhead + # print("Total test time for this group: {}" \ + # .format(time_wrap(use_gpu) - accum_test_time_begin)) + + if (args.mlperf_logging + and (args.mlperf_acc_threshold > 0) + and (best_gA_test > args.mlperf_acc_threshold)): + print("MLPerf testing accuracy threshold " + + str(args.mlperf_acc_threshold) + + " reached, stop training") + break + + if (args.mlperf_logging + and (args.mlperf_auc_threshold > 0) + and (best_auc_test > args.mlperf_auc_threshold)): + print("MLPerf testing auc threshold " + + str(args.mlperf_auc_threshold) + + " reached, stop training") + break + + #if (ext_dist.my_rank == 0 and should_print): + # print("ITER : ", j, " from nvidia-smi") + # os.system("nvidia-smi") + + k += 1 # nepochs + + #if (ext_dist.my_rank == 0): + # # print(torch.cuda.memory_allocated(0)) + # print(torch.cuda.memory_summary(0)) + # # print("from nvidia-smi") + # os.system("nvidia-smi") + + tt2 = time.time() + endTime = tt2 - startTime + ext_dist.barrier() + tt3 = time.time() + finalTime = tt3 - startTime + # torch.cuda.profiler.cudart().cudaProfilerStop() + torch.cuda.profiler.stop() + if (skipped > 2): + skipped -= 2 + ext_dist.orig_print("Process {} Done with total time {:.6f} measure time {:.6f}s {:.6f}s, \ + iter {:.1f}ms {:.1f}ms steps {} {}".format(ext_dist.my_rank, tt3 - startTime0, + finalTime, endTime, finalTime*1000.0/skipped, endTime*1000.0/skipped, skipped, tt2), flush=True) + if (ext_dist.my_rank < 2): + tm.tmSummary(ext_dist.my_rank) + + file_prefix = "%s/dlrm_s_pytorch_r%d" % (args.out_dir, ext_dist.my_rank) + # profiling + if args.enable_profiling: + os.makedirs(args.out_dir, exist_ok=True) + with open("TT"+str(uuid.uuid4().hex), "w") as prof_f: + prof_f.write(prof.key_averages(group_by_input_shape=True).table( + sort_by="self_cpu_time_total", + )) + +# with open("%s.prof" % file_prefix, "w") as prof_f: +# prof_f.write(prof.key_averages().table(sort_by="cpu_time_total")) +# prof.export_chrome_trace("./%s.json" % file_prefix) +# print(prof.key_averages().table(sort_by="cpu_time_total")) + + # plot compute graph + if args.plot_compute_graph: + sys.exit( + "ERROR: Please install pytorchviz package in order to use the" + + " visualization. Then, uncomment its import above as well as" + + " three lines below and run the code again." + ) + # os.makedirs(args.out_dir, exist_ok=True) + # V = Z.mean() if args.inference_only else E + # dot = make_dot(V, params=dict(dlrm.named_parameters())) + # dot.render('%s_graph' % file_prefix) # write .pdf file + + # test prints + if not args.inference_only and args.debug_mode: + print("updated parameters (weights and bias):") + for param in dlrm.parameters(): + print(param.detach().cpu().numpy()) + + # export the model in onnx + if args.save_onnx: + + dlrm_pytorch_onnx_file = "dlrm_s_pytorch.onnx" + torch.onnx.export( + dlrm, (X_onnx, lS_o_onnx, lS_i_onnx), dlrm_pytorch_onnx_file, verbose=True, use_external_data_format=True + ) + + # recover the model back + dlrm_pytorch_onnx = onnx.load("%s.onnx" % file_prefix) + # check the onnx model + onnx.checker.check_model(dlrm_pytorch_onnx) diff --git a/tt.py b/tt.py index 152dce83..0662776c 100644 --- a/tt.py +++ b/tt.py @@ -1056,11 +1056,11 @@ def dlrm_wrap(X, lS_o, lS_i, use_gpu, device): # lS_i can be either a list of tensors or a stacked tensor. # Handle each case below: tm.tmH2D.start() - lS_i = [S_i.to(device) for S_i in lS_i] if isinstance(lS_i, list) \ - else lS_i.to(device) - lS_o = [S_o.to(device) for S_o in lS_o] if isinstance(lS_o, list) \ - else lS_o.to(device) - X = X.to(device) + #lS_i = [S_i.to(device) for S_i in lS_i] if isinstance(lS_i, list) \ + # else lS_i.to(device) + #lS_o = [S_o.to(device) for S_o in lS_o] if isinstance(lS_o, list) \ + # else lS_o.to(device) + #X = X.to(device) tm.tmH2D.stop() return dlrm( @@ -1074,7 +1074,8 @@ def dlrm_wrap(X, lS_o, lS_i, use_gpu, device): def loss_fn_wrap(Z, T, use_gpu, device): if args.loss_function == "mse" or args.loss_function == "bce": if use_gpu: - return loss_fn(Z, T.to(device)) + # return loss_fn(Z, T.to(device)) + return loss_fn(Z, T) else: return loss_fn(Z, T) elif args.loss_function == "wbce": @@ -1193,7 +1194,15 @@ def loss_fn_wrap(Z, T, use_gpu, device): for j in range(nbatches): tm.tmGetData.start() # X, lS_o, lS_i, T = myobj[j%syndatasetlen][1] - X, lS_o, lS_i, T = train_data.__getitem__(j%syndatasetlen) + if j==0 and use_gpu: + X, lS_o, lS_i, T = train_data.__getitem__(j%syndatasetlen) + lS_i = [S_i.to(device) for S_i in lS_i] if isinstance(lS_i, list) \ + else lS_i.to(device) + lS_o = [S_o.to(device) for S_o in lS_o] if isinstance(lS_o, list) \ + else lS_o.to(device) + X = X.to(device) + T = T.to(device) + tm.tmGetData.stop() if j == 0 and args.save_onnx: @@ -1260,9 +1269,9 @@ def loss_fn_wrap(Z, T, use_gpu, device): # compute loss and accuracy L = E.detach().cpu().numpy() # numpy array S = Z.detach().cpu().numpy() # numpy array - T = T.detach().cpu().numpy() # numpy array - mbs = T.shape[0] # = args.mini_batch_size except maybe for last - A = np.sum((np.round(S, 0) == T).astype(np.uint8)) + T0 = T.detach().cpu().numpy() # numpy array + mbs = T0.shape[0] # = args.mini_batch_size except maybe for last + A = np.sum((np.round(S, 0) == T0).astype(np.uint8)) tm.tmLoss.stop() if not args.inference_only: From 4692d0e9942f5e36c57dc209bdcd94044de42981 Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Thu, 17 Dec 2020 18:21:47 -0800 Subject: [PATCH 56/57] modify tt to reuse input --- tt.py | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/tt.py b/tt.py index 0662776c..a7e850d4 100644 --- a/tt.py +++ b/tt.py @@ -443,9 +443,10 @@ def distributed_forward(self, dense_x, lS_o, lS_i): if batch_size % ext_dist.my_size != 0: sys.exit("ERROR: batch_size %d can not split across %d ranks evenly" % (batch_size, ext_dist.my_size)) - dense_x = dense_x[ext_dist.get_my_slice(batch_size)] - lS_o = lS_o[self.local_emb_slice] - lS_i = lS_i[self.local_emb_slice] + ## already handled in input the data + ##dense_x = dense_x[ext_dist.get_my_slice(batch_size)] + ##lS_o = lS_o[self.local_emb_slice] + ##lS_i = lS_i[self.local_emb_slice] if (len(self.emb_l) != len(lS_o)) or (len(self.emb_l) != len(lS_i)): sys.exit("ERROR: corrupted model input detected in distributed_forward call") @@ -1192,16 +1193,35 @@ def loss_fn_wrap(Z, T, use_gpu, device): # for j, (X, lS_o, lS_i, T) in enumerate(train_ld): for j in range(nbatches): - tm.tmGetData.start() - # X, lS_o, lS_i, T = myobj[j%syndatasetlen][1] + + if (skipped == 2): + ext_dist.barrier() + startTime = time.time() + ext_dist.orig_print("ORIG TIME: ", startTime, accum_time_begin, startTime - accum_time_begin, " for process ", ext_dist.my_rank) + # torch.cuda.profiler.cudart().cudaProfilerStart() + if use_gpu: + torch.cuda.profiler.start() + tm.tmClear() + skipped = skipped + 1 + + tm.tmGetData.start() if j==0 and use_gpu: - X, lS_o, lS_i, T = train_data.__getitem__(j%syndatasetlen) + # X, lS_o, lS_i, T = train_data.__getitem__(j%syndatasetlen) + X, lS_o, lS_i, T = next(enumerate(train_ld) + + print("BB0 X size {} lS_i[0] size {}".format(X.size(), lS_i[0].size())) + mybatch_size = X.size()[0] + X = X[ext_dist.get_my_slice(mybatch_size)] + lS_o = lS_o[dlrm.local_emb_slice] + lS_i = lS_i[dlrm.local_emb_slice] + lS_i = [S_i.to(device) for S_i in lS_i] if isinstance(lS_i, list) \ else lS_i.to(device) lS_o = [S_o.to(device) for S_o in lS_o] if isinstance(lS_o, list) \ else lS_o.to(device) X = X.to(device) T = T.to(device) + print("BBB X size {} lS_i[0] size {}".format(X.size(), lS_i[0].size())) tm.tmGetData.stop() @@ -1211,15 +1231,6 @@ def loss_fn_wrap(Z, T, use_gpu, device): if j < skip_upto_batch: continue - if (skipped == 2): - ext_dist.barrier() - startTime = time.time() - ext_dist.orig_print("ORIG TIME: ", startTime, accum_time_begin, startTime - accum_time_begin, " for process ", ext_dist.my_rank) - # torch.cuda.profiler.cudart().cudaProfilerStart() - torch.cuda.profiler.start() - tm.tmClear() - skipped = skipped + 1 - if args.mlperf_logging: current_time = time_wrap(use_gpu) if previous_iteration_time: From 751a2ca4de7401aadb5dca3fc3599a4382d8e41f Mon Sep 17 00:00:00 2001 From: Hongzhang Shan Date: Mon, 21 Dec 2020 02:52:00 -0800 Subject: [PATCH 57/57] fix corner case in tt.py --- tt.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tt.py b/tt.py index a7e850d4..357ac3e4 100644 --- a/tt.py +++ b/tt.py @@ -438,10 +438,10 @@ def sequential_forward(self, dense_x, lS_o, lS_i): def distributed_forward(self, dense_x, lS_o, lS_i): batch_size = dense_x.size()[0] # WARNING: # of ranks must be <= batch size in distributed_forward call - if batch_size < ext_dist.my_size: - sys.exit("ERROR: batch_size (%d) must be larger than number of ranks (%d)" % (batch_size, ext_dist.my_size)) - if batch_size % ext_dist.my_size != 0: - sys.exit("ERROR: batch_size %d can not split across %d ranks evenly" % (batch_size, ext_dist.my_size)) + # if batch_size < ext_dist.my_size: + # sys.exit("ERROR: batch_size (%d) must be larger than number of ranks (%d)" % (batch_size, ext_dist.my_size)) + # if batch_size % ext_dist.my_size != 0: + # sys.exit("ERROR: batch_size %d can not split across %d ranks evenly" % (batch_size, ext_dist.my_size)) ## already handled in input the data ##dense_x = dense_x[ext_dist.get_my_slice(batch_size)] @@ -510,7 +510,7 @@ def distributed_forward(self, dense_x, lS_o, lS_i): # tensor is on GPU memory tm.tmAllGa.start() if z.is_cuda: torch.cuda.synchronize() - (_, batch_split_lengths) = ext_dist.get_split_lengths(batch_size) + (_, batch_split_lengths) = ext_dist.get_split_lengths(batch_size * ext_dist.my_size) z = ext_dist.all_gather(z, batch_split_lengths) tm.tmAllGa.stop() #print("Z: %s" % z) @@ -1211,9 +1211,10 @@ def loss_fn_wrap(Z, T, use_gpu, device): print("BB0 X size {} lS_i[0] size {}".format(X.size(), lS_i[0].size())) mybatch_size = X.size()[0] - X = X[ext_dist.get_my_slice(mybatch_size)] - lS_o = lS_o[dlrm.local_emb_slice] - lS_i = lS_i[dlrm.local_emb_slice] + if ext_dist.my_size > 1: + X = X[ext_dist.get_my_slice(mybatch_size)] + lS_o = lS_o[dlrm.local_emb_slice] + lS_i = lS_i[dlrm.local_emb_slice] lS_i = [S_i.to(device) for S_i in lS_i] if isinstance(lS_i, list) \ else lS_i.to(device)