explicitly enable TOKENIZERS_PARALLELISM=true

a8a9e3ffa12e57b1213c5e3316e5b44d25a0be0b · 2025-01-23 01:26:27 +0000 · Alex Cheema

Files touched

M exo/main.py

Diff

commit a8a9e3ffa12e57b1213c5e3316e5b44d25a0be0b
Author: Alex Cheema <alexcheema123@gmail.com>
Date:   Thu Jan 23 01:26:27 2025 +0000

    explicitly enable TOKENIZERS_PARALLELISM=true
---
 exo/main.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/exo/main.py b/exo/main.py
index cf37ac06..d58224ca 100644
--- a/exo/main.py
+++ b/exo/main.py
@@ -42,6 +42,7 @@ import psutil
 # TODO: figure out why this is happening
 os.environ["GRPC_VERBOSITY"] = "error"
 os.environ["TRANSFORMERS_VERBOSITY"] = "error"
+os.environ["TOKENIZERS_PARALLELISM"] = "true"
 
 # Configure uvloop for maximum performance
 def configure_uvloop():

← 5c9bcb86 set GRPC_VERBOSITY=error; TRANSFORMERS_VERBOSITY=error · back to Exo · add linux tinygrad test 790c08af →