From b6484640061372fc4b940ca71843bb68cb76bf6c Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Fri, 10 Feb 2023 01:21:49 +0000 Subject: [PATCH 1/7] Add more models to unsupported list. Add more assertions for better program exit --- deepspeed/module_inject/auto_tp.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deepspeed/module_inject/auto_tp.py b/deepspeed/module_inject/auto_tp.py index 32aa1fbb934b..47a680467789 100644 --- a/deepspeed/module_inject/auto_tp.py +++ b/deepspeed/module_inject/auto_tp.py @@ -25,13 +25,14 @@ def get_module_list(model): return mlist def supported(model): - unsupported = ['bloom', 'codegen', 'flaubert', 'xlm'] + unsupported = ['bloom', 'codegen', 'deberta', 'flaubert', 'fsmt', 'led', 'longformer', 'xlm'] model = str(model) key = re.search(r": (.*?)Model", model) if key is None: key = re.search(r": (.*?)Stack", model) if key is None: key = re.match(r"(.*?)Model", model) + assert key is not None, "Not able to determine model policy automatically. Please provide policy." if key.group(1).lower() in unsupported: return False return True @@ -73,4 +74,5 @@ def tp_parser(model): if gem_list != []: policy_list.append(tuple([type(module), gem_list])) gem_list = [] + assert len(policy_list), "Not able to determine model policy automatically. Please provide policy." return policy_list From 01a8635bbc354387217c7d3c50a2171a4278a90b Mon Sep 17 00:00:00 2001 From: Molly Smith <112220543+molly-smith@users.noreply.github.com> Date: Thu, 16 Feb 2023 13:18:32 -0800 Subject: [PATCH 2/7] add xlnet to unsupported models --- deepspeed/module_inject/auto_tp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepspeed/module_inject/auto_tp.py b/deepspeed/module_inject/auto_tp.py index 47a680467789..11f5ea5847a9 100644 --- a/deepspeed/module_inject/auto_tp.py +++ b/deepspeed/module_inject/auto_tp.py @@ -25,7 +25,7 @@ def get_module_list(model): return mlist def supported(model): - unsupported = ['bloom', 'codegen', 'deberta', 'flaubert', 'fsmt', 'led', 'longformer', 'xlm'] + unsupported = ['bloom', 'codegen', 'deberta', 'flaubert', 'fsmt', 'led', 'longformer', 'xlm', 'xlnet'] model = str(model) key = re.search(r": (.*?)Model", model) if key is None: From d5b6eefd6fbbe796a838cdd8695bb54befde2596 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Tue, 28 Feb 2023 18:52:12 +0000 Subject: [PATCH 3/7] Move gpt2 to unsupported list --- deepspeed/module_inject/auto_tp.py | 2 +- docs/_tutorials/automatic-tensor-parallelism.md | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/deepspeed/module_inject/auto_tp.py b/deepspeed/module_inject/auto_tp.py index 32243217f13d..17ac9282b919 100644 --- a/deepspeed/module_inject/auto_tp.py +++ b/deepspeed/module_inject/auto_tp.py @@ -27,7 +27,7 @@ def get_module_list(model): return mlist def supported(model): - unsupported = ['bloom', 'codegen', 'deberta', 'flaubert', 'fsmt', 'led', 'longformer', 'xlm', 'xlnet'] + unsupported = ['bloom', 'codegen', 'deberta', 'flaubert', 'fsmt', 'gpt2', 'led', 'longformer', 'xlm', 'xlnet'] model = str(model) key = re.search(r": (.*?)Model", model) if key is None: diff --git a/docs/_tutorials/automatic-tensor-parallelism.md b/docs/_tutorials/automatic-tensor-parallelism.md index 89bddb7b9686..6991d5caf925 100644 --- a/docs/_tutorials/automatic-tensor-parallelism.md +++ b/docs/_tutorials/automatic-tensor-parallelism.md @@ -88,6 +88,7 @@ deepspeed --num_gpus DeepSpeedExamples/inference/huggingface/text-gen The following results were collected using V100 SXM2 32GB GPUs. ### Max New Tokens = 50 + | Test | Memory Allocated per GPU | Max Batch Size | Max Throughput per GPU | | ---------- | -------------------------- | ---------------- | ------------------------ | | No TP | 23.94 GB | 64 | 18.84 TFlops | @@ -95,6 +96,7 @@ The following results were collected using V100 SXM2 32GB GPUs. | 4 GPU TP | 6.36 GB | 664 | 27.63 TFlops | ### Max New Tokens = 1024 + | Test | Memory Allocated per GPU | Max Batch Size | Max Throughput per GPU | | ---------- | -------------------------- | ---------------- | ------------------------ | | No TP | 23.94 GB | 2 | 1.65 TFlops | @@ -113,7 +115,6 @@ The following model families have been successfully tested with automatic tensor - electra - ernie - esm -- gpt2 - gpt-j - gpt-neo - gpt-neox @@ -146,6 +147,7 @@ The following models are not currently supported with automatic tensor paralleli - deberta - flaubert - fsmt +- gpt2 - led - longformer - xlm From bb5a48292daedfefe27b19ebd9be5decc6df9e1f Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Tue, 28 Feb 2023 20:06:59 +0000 Subject: [PATCH 4/7] Indentation of last assert --- deepspeed/module_inject/auto_tp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepspeed/module_inject/auto_tp.py b/deepspeed/module_inject/auto_tp.py index 17ac9282b919..a79e09abee67 100644 --- a/deepspeed/module_inject/auto_tp.py +++ b/deepspeed/module_inject/auto_tp.py @@ -92,5 +92,5 @@ def tp_parser(model): gem_list = list(set(gem_list)) policy_list = AutoTP.update_policy_list(policy_list, module, gem_list) gem_list = [] - assert len(policy_list), "Not able to determine model policy automatically. Please provide policy." + assert len(policy_list), "Not able to determine model policy automatically. Please provide policy." return policy_list From 6e48884eebcb64e43f1f8e4cf956a250a807c158 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Tue, 28 Feb 2023 20:20:22 +0000 Subject: [PATCH 5/7] precommit --- deepspeed/module_inject/auto_tp.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/deepspeed/module_inject/auto_tp.py b/deepspeed/module_inject/auto_tp.py index a79e09abee67..d55e204aeb46 100644 --- a/deepspeed/module_inject/auto_tp.py +++ b/deepspeed/module_inject/auto_tp.py @@ -27,7 +27,18 @@ def get_module_list(model): return mlist def supported(model): - unsupported = ['bloom', 'codegen', 'deberta', 'flaubert', 'fsmt', 'gpt2', 'led', 'longformer', 'xlm', 'xlnet'] + unsupported = [ + 'bloom', + 'codegen', + 'deberta', + 'flaubert', + 'fsmt', + 'gpt2', + 'led', + 'longformer', + 'xlm', + 'xlnet' + ] model = str(model) key = re.search(r": (.*?)Model", model) if key is None: From a2586ac6ce48d64f1b3cf2cd13e44dc7576a7c17 Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Wed, 1 Mar 2023 06:22:11 +0000 Subject: [PATCH 6/7] use better prompts --- deepspeed/module_inject/auto_tp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/deepspeed/module_inject/auto_tp.py b/deepspeed/module_inject/auto_tp.py index d55e204aeb46..1f7b747befac 100644 --- a/deepspeed/module_inject/auto_tp.py +++ b/deepspeed/module_inject/auto_tp.py @@ -103,5 +103,6 @@ def tp_parser(model): gem_list = list(set(gem_list)) policy_list = AutoTP.update_policy_list(policy_list, module, gem_list) gem_list = [] + print(policy_list) assert len(policy_list), "Not able to determine model policy automatically. Please provide policy." return policy_list From 1009a22b6b6ee3f47387b1ff235744ec0510618c Mon Sep 17 00:00:00 2001 From: Molly Smith Date: Wed, 1 Mar 2023 06:23:36 +0000 Subject: [PATCH 7/7] Remove accidental commit change --- deepspeed/module_inject/auto_tp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/deepspeed/module_inject/auto_tp.py b/deepspeed/module_inject/auto_tp.py index 1f7b747befac..d55e204aeb46 100644 --- a/deepspeed/module_inject/auto_tp.py +++ b/deepspeed/module_inject/auto_tp.py @@ -103,6 +103,5 @@ def tp_parser(model): gem_list = list(set(gem_list)) policy_list = AutoTP.update_policy_list(policy_list, module, gem_list) gem_list = [] - print(policy_list) assert len(policy_list), "Not able to determine model policy automatically. Please provide policy." return policy_list