From 9343f34b0be8d3f14f45338f520cb4cbe6694128 Mon Sep 17 00:00:00 2001 From: Raul Puri Date: Thu, 3 May 2018 18:27:18 -0700 Subject: [PATCH 1/8] train.py single gpu and 0.4 update --- train.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/train.py b/train.py index dd5de2e..f914cfc 100644 --- a/train.py +++ b/train.py @@ -74,14 +74,17 @@ def prepare_directories_and_logger(output_directory, log_directory, rank): logger = None return logger - def load_model(hparams): model = Tacotron2(hparams).cuda() model = batchnorm_to_float(model.half()) if hparams.fp16_run else model - model = DistributedDataParallel(model) \ - if hparams.distributed_run else DataParallel(model) - return model + tacotron_model = model + if hparams.distributed_run: + model = DistributedDataParallel(model) + elif torch.cuda.device_count() > 1: + model = DataParallel(model) + + return model, tacotron def warm_start_model(checkpoint_path, model): assert os.path.isfile(checkpoint_path) @@ -114,7 +117,7 @@ def save_checkpoint(model, optimizer, learning_rate, iteration, filepath): def validate(model, criterion, valset, iteration, batch_size, n_gpus, - collate_fn, logger, distributed_run, rank): + collate_fn, logger, distributed_run, rank, batch_parser): """Handles all the validation scoring and printing""" model.eval() with torch.no_grad(): @@ -125,7 +128,7 @@ def validate(model, criterion, valset, iteration, batch_size, n_gpus, val_loss = 0.0 for i, batch in enumerate(val_loader): - x, y = model.module.parse_batch(batch) + x, y = batch_parser(batch) y_pred = model(x) loss = criterion(y_pred, y) reduced_val_loss = reduce_tensor(loss.data, n_gpus)[0] \ @@ -193,11 +196,11 @@ def train(output_directory, log_directory, checkpoint_path, warm_start, n_gpus, param_group['lr'] = learning_rate model.zero_grad() - x, y = model.module.parse_batch(batch) + x, y = tacotron_model.parse_batch(batch) y_pred = model(x) loss = criterion(y_pred, y) - reduced_loss = reduce_tensor(loss.data, n_gpus)[0] \ - if hparams.distributed_run else loss.data[0] + reduced_loss = reduce_tensor(loss.data, n_gpus).item() \ + if hparams.distributed_run else loss.item() if hparams.fp16_run: optimizer.backward(loss) @@ -205,7 +208,7 @@ def train(output_directory, log_directory, checkpoint_path, warm_start, n_gpus, else: loss.backward() grad_norm = torch.nn.utils.clip_grad_norm( - model.module.parameters(), hparams.grad_clip_thresh) + tacotron_model.parameters(), hparams.grad_clip_thresh) optimizer.step() @@ -222,7 +225,7 @@ def train(output_directory, log_directory, checkpoint_path, warm_start, n_gpus, if not overflow and (iteration % hparams.iters_per_checkpoint == 0): reduced_val_loss = validate( model, criterion, valset, iteration, hparams.batch_size, - n_gpus, collate_fn, logger, hparams.distributed_run, rank) + n_gpus, collate_fn, logger, hparams.distributed_run, rank, tacotron_model.parse_batch) if rank == 0: print("Validation loss {}: {:9f} ".format( From 1b7b06c75e88409c6afffea1cd878d150e98ba0c Mon Sep 17 00:00:00 2001 From: Raul Puri Date: Thu, 3 May 2018 18:29:37 -0700 Subject: [PATCH 2/8] model.py 0.4 update --- model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model.py b/model.py index 1f9e7d1..404822e 100644 --- a/model.py +++ b/model.py @@ -471,7 +471,7 @@ class Tacotron2(nn.Module): output_lengths = batch text_padded = to_gpu(text_padded).long() input_lengths = to_gpu(input_lengths).long() - max_len = torch.max(input_lengths.data) + max_len = torch.max(input_lengths.data).item() mel_padded = to_gpu(mel_padded).float() gate_padded = to_gpu(gate_padded).float() output_lengths = to_gpu(output_lengths).long() From 84c83f96c9fe3e9de6a68d2a84032ef748ffdedb Mon Sep 17 00:00:00 2001 From: Rafael Valle Date: Thu, 3 May 2018 19:42:01 -0700 Subject: [PATCH 3/8] model.py: removing .item --- model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model.py b/model.py index 404822e..1f9e7d1 100644 --- a/model.py +++ b/model.py @@ -471,7 +471,7 @@ class Tacotron2(nn.Module): output_lengths = batch text_padded = to_gpu(text_padded).long() input_lengths = to_gpu(input_lengths).long() - max_len = torch.max(input_lengths.data).item() + max_len = torch.max(input_lengths.data) mel_padded = to_gpu(mel_padded).float() gate_padded = to_gpu(gate_padded).float() output_lengths = to_gpu(output_lengths).long() From 646ab0d8c868c094d46a9feeb0549dccad0f9499 Mon Sep 17 00:00:00 2001 From: Rafael Valle Date: Thu, 3 May 2018 19:42:37 -0700 Subject: [PATCH 4/8] model.py removing top of three code, cleanup --- train.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/train.py b/train.py index f914cfc..7103321 100644 --- a/train.py +++ b/train.py @@ -74,22 +74,23 @@ def prepare_directories_and_logger(output_directory, log_directory, rank): logger = None return logger + def load_model(hparams): model = Tacotron2(hparams).cuda() model = batchnorm_to_float(model.half()) if hparams.fp16_run else model - tacotron_model = model if hparams.distributed_run: model = DistributedDataParallel(model) elif torch.cuda.device_count() > 1: model = DataParallel(model) - return model, tacotron + return model + def warm_start_model(checkpoint_path, model): assert os.path.isfile(checkpoint_path) print("Warm starting model from checkpoint '{}'".format(checkpoint_path)) - checkpoint_dict = torch.load(checkpoint_path) + checkpoint_dict = torch.load(checkpoint_path, map_location='cpu') model.load_state_dict(checkpoint_dict['state_dict']) return model @@ -117,7 +118,7 @@ def save_checkpoint(model, optimizer, learning_rate, iteration, filepath): def validate(model, criterion, valset, iteration, batch_size, n_gpus, - collate_fn, logger, distributed_run, rank, batch_parser): + collate_fn, logger, distributed_run, rank): """Handles all the validation scoring and printing""" model.eval() with torch.no_grad(): @@ -128,7 +129,7 @@ def validate(model, criterion, valset, iteration, batch_size, n_gpus, val_loss = 0.0 for i, batch in enumerate(val_loader): - x, y = batch_parser(batch) + x, y = model.parse_batch(batch) y_pred = model(x) loss = criterion(y_pred, y) reduced_val_loss = reduce_tensor(loss.data, n_gpus)[0] \ @@ -196,11 +197,11 @@ def train(output_directory, log_directory, checkpoint_path, warm_start, n_gpus, param_group['lr'] = learning_rate model.zero_grad() - x, y = tacotron_model.parse_batch(batch) + x, y = model.parse_batch(batch) y_pred = model(x) loss = criterion(y_pred, y) - reduced_loss = reduce_tensor(loss.data, n_gpus).item() \ - if hparams.distributed_run else loss.item() + reduced_loss = reduce_tensor(loss.data, n_gpus)[0] \ + if hparams.distributed_run else loss.data[0] if hparams.fp16_run: optimizer.backward(loss) @@ -208,7 +209,7 @@ def train(output_directory, log_directory, checkpoint_path, warm_start, n_gpus, else: loss.backward() grad_norm = torch.nn.utils.clip_grad_norm( - tacotron_model.parameters(), hparams.grad_clip_thresh) + model.parameters(), hparams.grad_clip_thresh) optimizer.step() @@ -225,7 +226,7 @@ def train(output_directory, log_directory, checkpoint_path, warm_start, n_gpus, if not overflow and (iteration % hparams.iters_per_checkpoint == 0): reduced_val_loss = validate( model, criterion, valset, iteration, hparams.batch_size, - n_gpus, collate_fn, logger, hparams.distributed_run, rank, tacotron_model.parse_batch) + n_gpus, collate_fn, logger, hparams.distributed_run, rank) if rank == 0: print("Validation loss {}: {:9f} ".format( From 0d4218fb2054eb1d2db795a1926eb71b81e70f36 Mon Sep 17 00:00:00 2001 From: Raul Puri Date: Thu, 3 May 2018 22:12:18 -0700 Subject: [PATCH 5/8] single variable for single gpu model --- train.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/train.py b/train.py index 7103321..c64ea1c 100644 --- a/train.py +++ b/train.py @@ -128,8 +128,13 @@ def validate(model, criterion, valset, iteration, batch_size, n_gpus, pin_memory=False, collate_fn=collate_fn) val_loss = 0.0 + if distributed_run or torch.cuda.device_count() > 1: + batch_parser = model.module.parse_batch + else: + batch_parser = model.parse_batch + for i, batch in enumerate(val_loader): - x, y = model.parse_batch(batch) + x, y = batch_parser(batch) y_pred = model(x) loss = criterion(y_pred, y) reduced_val_loss = reduce_tensor(loss.data, n_gpus)[0] \ @@ -157,6 +162,7 @@ def train(output_directory, log_directory, checkpoint_path, warm_start, n_gpus, if hparams.distributed_run: init_distributed(hparams, n_gpus, rank, group_name) + torch.manual_seed(hparams.seed) torch.cuda.manual_seed(hparams.seed) @@ -188,6 +194,10 @@ def train(output_directory, log_directory, checkpoint_path, warm_start, n_gpus, epoch_offset = max(0, int(iteration / len(train_loader))) model.train() + if distributed_run or torch.cuda.device_count() > 1: + batch_parser = model.module.parse_batch + else: + batch_parser = model.parse_batch # ================ MAIN TRAINNIG LOOP! =================== for epoch in range(epoch_offset, hparams.epochs): print("Epoch: {}".format(epoch)) @@ -197,7 +207,7 @@ def train(output_directory, log_directory, checkpoint_path, warm_start, n_gpus, param_group['lr'] = learning_rate model.zero_grad() - x, y = model.parse_batch(batch) + x, y = batch_parser(batch) y_pred = model(x) loss = criterion(y_pred, y) reduced_loss = reduce_tensor(loss.data, n_gpus)[0] \ From 4d5826e8949f44aab79016b0f1596859ab0c9e62 Mon Sep 17 00:00:00 2001 From: Raul Puri Date: Fri, 4 May 2018 06:26:51 -0700 Subject: [PATCH 6/8] train.py typo --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index c64ea1c..4a83921 100644 --- a/train.py +++ b/train.py @@ -194,7 +194,7 @@ def train(output_directory, log_directory, checkpoint_path, warm_start, n_gpus, epoch_offset = max(0, int(iteration / len(train_loader))) model.train() - if distributed_run or torch.cuda.device_count() > 1: + if hparams.distributed_run or torch.cuda.device_count() > 1: batch_parser = model.module.parse_batch else: batch_parser = model.parse_batch From 2aba10cc0a87e654a465c4c5f745cd9f573fd1be Mon Sep 17 00:00:00 2001 From: Rafael Valle Date: Fri, 4 May 2018 09:09:24 -0700 Subject: [PATCH 7/8] README.md: updating multi-gpu syntax --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 107e713..3a904d0 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Distributed and FP16 support relies on work by Christian Sarofeen and NVIDIA's 2. (OPTIONAL) `tensorboard --logdir=outdir/logdir` ## Multi-GPU (distributed) and FP16 Training -1. `python -m multiproc train.py --output_directory=/outdir --log_directory=/logdir --hparams=distributed_run=True --fp16_run=True` +1. `python -m multiproc train.py --output_directory=outdir --log_directory=logdir --hparams=distributed_run=True,fp16_run=True` ## Inference 1. `jupyter notebook --ip=127.0.0.1 --port=31337` From be5fdd6d20e6bd5e4353993aa9de93306f1acc1e Mon Sep 17 00:00:00 2001 From: Rafael Valle Date: Fri, 4 May 2018 09:10:53 -0700 Subject: [PATCH 8/8] train.py: layout changes --- train.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/train.py b/train.py index 4a83921..ee01b07 100644 --- a/train.py +++ b/train.py @@ -132,7 +132,7 @@ def validate(model, criterion, valset, iteration, batch_size, n_gpus, batch_parser = model.module.parse_batch else: batch_parser = model.parse_batch - + for i, batch in enumerate(val_loader): x, y = batch_parser(batch) y_pred = model(x) @@ -162,7 +162,6 @@ def train(output_directory, log_directory, checkpoint_path, warm_start, n_gpus, if hparams.distributed_run: init_distributed(hparams, n_gpus, rank, group_name) - torch.manual_seed(hparams.seed) torch.cuda.manual_seed(hparams.seed)