fix: added trust_remote_code to tokenizer
Browse files- modeling_bert.py +2 -2
modeling_bert.py
CHANGED
|
@@ -377,7 +377,7 @@ class BertModel(BertPreTrainedModel):
|
|
| 377 |
if self.emb_pooler is not None:
|
| 378 |
from transformers import AutoTokenizer
|
| 379 |
|
| 380 |
-
self.tokenizer = AutoTokenizer.from_pretrained(config._name_or_path)
|
| 381 |
else:
|
| 382 |
self.tokenizer = None
|
| 383 |
|
|
@@ -507,7 +507,7 @@ class BertModel(BertPreTrainedModel):
|
|
| 507 |
self.emb_pooler = 'mean'
|
| 508 |
from transformers import AutoTokenizer
|
| 509 |
|
| 510 |
-
self.tokenizer = AutoTokenizer.from_pretrained(self._name_or_path)
|
| 511 |
if self.emb_pooler != 'mean':
|
| 512 |
raise NotImplementedError
|
| 513 |
|
|
|
|
| 377 |
if self.emb_pooler is not None:
|
| 378 |
from transformers import AutoTokenizer
|
| 379 |
|
| 380 |
+
self.tokenizer = AutoTokenizer.from_pretrained(config._name_or_path, trust_remote_code=True)
|
| 381 |
else:
|
| 382 |
self.tokenizer = None
|
| 383 |
|
|
|
|
| 507 |
self.emb_pooler = 'mean'
|
| 508 |
from transformers import AutoTokenizer
|
| 509 |
|
| 510 |
+
self.tokenizer = AutoTokenizer.from_pretrained(self._name_or_path, trust_remote_code=True)
|
| 511 |
if self.emb_pooler != 'mean':
|
| 512 |
raise NotImplementedError
|
| 513 |
|