fix local check if dir does not exist

85bab25ac0b531969d2c4d0022568d1a8a1d1a08 · 2024-08-24 10:13:10 +0100 · Alex Cheema

Files touched

M exo/inference/tokenizers.py

Diff

commit 85bab25ac0b531969d2c4d0022568d1a8a1d1a08
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Sat Aug 24 10:13:10 2024 +0100

    fix local check if dir does not exist
---
 exo/inference/tokenizers.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/exo/inference/tokenizers.py b/exo/inference/tokenizers.py
index 82b95520..e0bc332d 100644
--- a/exo/inference/tokenizers.py
+++ b/exo/inference/tokenizers.py
@@ -6,10 +6,14 @@ from exo.helpers import DEBUG
 
 async def resolve_tokenizer(model_id: str):
   local_path = await get_local_snapshot_dir(model_id)
-  if await aios.path.exists(local_path):
-    if DEBUG >= 2: print(f"Resolving tokenizer for {model_id=} from {local_path=}")
-    return await _resolve_tokenizer(local_path)
-  if DEBUG >= 2: print(f"Local check for {local_path=} failed. Resolving tokenizer for {model_id=} normally...")
+  if DEBUG >= 2: print(f"Checking if local path exists to load tokenizer from local {local_path=}")
+  try:
+    if await aios.path.exists(local_path):
+      if DEBUG >= 2: print(f"Resolving tokenizer for {model_id=} from {local_path=}")
+      return await _resolve_tokenizer(local_path)
+  except:
+    if DEBUG >= 5: print(f"Local check for {local_path=} failed. Resolving tokenizer for {model_id=} normally...")
+    if DEBUG >= 5: traceback.print_exc()
   return await _resolve_tokenizer(model_id)
 
 async def _resolve_tokenizer(model_id_or_local_path: str):

← 59c4393d first try loading tokenizer from local path instead of alway · back to Exo · download tinychat dependencies all to local dir so we dont n 3791e669 →