From 6cce4bee9dd02edaa925d23fd6bea77ef1b9a67b Mon Sep 17 00:00:00 2001 From: chunyang Date: Wed, 27 May 2020 22:13:03 +0800 Subject: [PATCH 1/3] fix: typo --- deepspeed/pt/deepspeed_checkpointing.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/deepspeed/pt/deepspeed_checkpointing.py b/deepspeed/pt/deepspeed_checkpointing.py index 68d3b6bff4f7..a38db1f498db 100755 --- a/deepspeed/pt/deepspeed_checkpointing.py +++ b/deepspeed/pt/deepspeed_checkpointing.py @@ -14,6 +14,7 @@ # Parts of the code here are adapted from PyTorch # repo: https://github.com/pytorch/pytorch import contextlib +import copy import torch.distributed as dist import torch from torch import _C @@ -108,7 +109,7 @@ def detach_variable(inputs, device=None): def _set_cuda_rng_state(new_state, device=-1): """Sets the random number generator state of the current GPU. - Argumentss: + Arguments: new_state (torch.ByteTensor): The desired state This function is adapted from PyTorch repo (torch.cuda.set_rng_state) with a single change: the input state is not cloned. Cloning caused @@ -160,10 +161,7 @@ def reset(self): def get_states(self): """Get rng states. Copy the dictionary so we have direct pointers to the states, not just a pointer to the dictionary.""" - states = {} - for name in self.states_: - states[name] = self.states_[name] - return states + return copy.copy(self.states_) def set_states(self, states): """Set the rng states. For efficiency purposes, we do not check @@ -720,5 +718,4 @@ def is_configured(): Return: True of configured, else False """ - global deepspeed_checkpointing_enabled return deepspeed_checkpointing_enabled From d9567ce7b0a149a27fa08d4ffda5940ca6279b88 Mon Sep 17 00:00:00 2001 From: chunyang Date: Wed, 27 May 2020 22:19:02 +0800 Subject: [PATCH 2/3] more pythonic code --- deepspeed/pt/deepspeed_config_utils.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/deepspeed/pt/deepspeed_config_utils.py b/deepspeed/pt/deepspeed_config_utils.py index d99c5ab197db..a416f500cf9f 100755 --- a/deepspeed/pt/deepspeed_config_utils.py +++ b/deepspeed/pt/deepspeed_config_utils.py @@ -8,18 +8,12 @@ def get_scalar_param(param_dict, param_name, param_default_value): - if param_name in param_dict.keys(): - return param_dict[param_name] - else: - return param_default_value + return param_dict.get(param_name, param_default_value) def dict_raise_error_on_duplicate_keys(ordered_pairs): """Reject duplicate keys.""" - d = {} - for k, v in ordered_pairs: - if k in d: - raise ValueError("Duplicate key in DeepSpeed config: %r" % (k, )) - else: - d[k] = v + d = dict((k, v) for k, v in ordered_pairs) + if len(d) != len(ordered_pairs): + raise ValueError("Duplicate key in DeepSpeed config: %r" % (k, )) return d From b4167cf93ccdfe52ceab3f7e7507ef69352e4ca6 Mon Sep 17 00:00:00 2001 From: chengfu Date: Thu, 28 May 2020 16:54:20 +0800 Subject: [PATCH 3/3] fix message --- deepspeed/pt/deepspeed_config_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/deepspeed/pt/deepspeed_config_utils.py b/deepspeed/pt/deepspeed_config_utils.py index a416f500cf9f..38fdb647f61d 100755 --- a/deepspeed/pt/deepspeed_config_utils.py +++ b/deepspeed/pt/deepspeed_config_utils.py @@ -6,6 +6,8 @@ Collection of DeepSpeed configuration utilities """ +from collections import Counter + def get_scalar_param(param_dict, param_name, param_default_value): return param_dict.get(param_name, param_default_value) @@ -15,5 +17,7 @@ def dict_raise_error_on_duplicate_keys(ordered_pairs): """Reject duplicate keys.""" d = dict((k, v) for k, v in ordered_pairs) if len(d) != len(ordered_pairs): - raise ValueError("Duplicate key in DeepSpeed config: %r" % (k, )) + counter = Counter([pair[0] for pair in ordered_pairs]) + keys = [key for key, value in counter.items() if value > 1] + raise ValueError("Duplicate keys in DeepSpeed config: {}".format(keys)) return d