From 1b1c57bd7b727254937e0105df6ccf578d827bf2 Mon Sep 17 00:00:00 2001
From: Manjunath Janardhan <35326118+manjunathshiva@users.noreply.github.com>
Date: Mon, 7 Oct 2024 11:58:43 +0530
Subject: [PATCH] [DOC UPDATE ]Fix ImportError: cannot import name
 'ModelFilter' from 'huggingface_hub' in slm_pretraining_sft.ipynb

latest huggingface_hub with nemo_toolkit==1.23.0 does not have  ModelFilter.

python /opt/NeMo/scripts/nlp_language_modeling/preprocess_data_for_megatron.py \
        --input=cosmopedia-100k.jsonl \
        --json-keys=text \
        --tokenizer-library=megatron \
        --tokenizer-type=GPT2BPETokenizer \
        --dataset-impl=mmap \
        --merge-file=merges.txt \
        --vocab-file=vocab.json \
        --output-prefix=cosmopedia-100k \
        --append-eod \
        --workers=4

Traceback (most recent call last):
  File "/opt/NeMo/scripts/nlp_language_modeling/preprocess_data_for_megatron.py", line 97, in <module>
    from nemo.collections.nlp.data.language_modeling.megatron import indexed_dataset
  File "/opt/NeMo/nemo/collections/nlp/__init__.py", line 15, in <module>
    from nemo.collections.nlp import data, losses, models, modules
  File "/opt/NeMo/nemo/collections/nlp/data/__init__.py", line 16, in <module>
    from nemo.collections.nlp.data.entity_linking.entity_linking_dataset import EntityLinkingDataset
  File "/opt/NeMo/nemo/collections/nlp/data/entity_linking/__init__.py", line 15, in <module>
    from nemo.collections.nlp.data.entity_linking.entity_linking_dataset import EntityLinkingDataset
  File "/opt/NeMo/nemo/collections/nlp/data/entity_linking/entity_linking_dataset.py", line 22, in <module>
    from nemo.core.classes import Dataset
  File "/opt/NeMo/nemo/core/__init__.py", line 16, in <module>
    from nemo.core.classes import *
  File "/opt/NeMo/nemo/core/classes/__init__.py", line 20, in <module>
    from nemo.core.classes.common import (
  File "/opt/NeMo/nemo/core/classes/common.py", line 31, in <module>
    from huggingface_hub import HfApi, HfFolder, ModelFilter, hf_hub_download
ImportError: cannot import name 'ModelFilter' from 'huggingface_hub' (/usr/local/lib/python3.10/dist-packages/huggingface_hub/__init__.py)
---
 finetuning/NeMo/slm/slm_pretraining_sft.ipynb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/finetuning/NeMo/slm/slm_pretraining_sft.ipynb b/finetuning/NeMo/slm/slm_pretraining_sft.ipynb
index 214330359..469b6a340 100644
--- a/finetuning/NeMo/slm/slm_pretraining_sft.ipynb
+++ b/finetuning/NeMo/slm/slm_pretraining_sft.ipynb
@@ -121,7 +121,7 @@
    "outputs": [],
    "source": [
     "!pip install --quiet ipywidgets\n",
-    "!pip install --upgrade --quiet huggingface_hub"
+    "!pip install --quiet huggingface_hub==0.22.0"
    ]
   },
   {