Morgan Funtowicz
commited on
Commit
·
6ce5654
1
Parent(s):
276cf66
feat(embeddings): flush denormal numbers
Browse files- handler.py +4 -1
handler.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
| 1 |
-
import os
|
| 2 |
import platform
|
| 3 |
from typing import Union, Sequence, Sized
|
| 4 |
|
|
@@ -48,6 +47,10 @@ class SentenceTransformerHandler(Handler):
|
|
| 48 |
self._allocate_model()
|
| 49 |
|
| 50 |
def _allocate_model(self):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 51 |
dtype = torch.bfloat16 if has_bf16_support() else torch.float32
|
| 52 |
model = SentenceTransformer(self._config.model_id, device="cpu", model_kwargs={"torch_dtype": dtype})
|
| 53 |
|
|
|
|
|
|
|
| 1 |
import platform
|
| 2 |
from typing import Union, Sequence, Sized
|
| 3 |
|
|
|
|
| 47 |
self._allocate_model()
|
| 48 |
|
| 49 |
def _allocate_model(self):
|
| 50 |
+
# Denormal number is used to store extremely small numbers that are close to 0.
|
| 51 |
+
# Computations with denormal numbers are remarkably slower than normalized number.
|
| 52 |
+
torch.set_flush_denormal(True)
|
| 53 |
+
|
| 54 |
dtype = torch.bfloat16 if has_bf16_support() else torch.float32
|
| 55 |
model = SentenceTransformer(self._config.model_id, device="cpu", model_kwargs={"torch_dtype": dtype})
|
| 56 |
|