Fix tiktoken_util.py to set TIKTOKEN_CACHE_DIR dynamically

2024-04-29 15:12:45 +08:00
2 changed files with 100259 additions and 0 deletions
--- a/unit_tests/tools/tiktoken_cache/9b5ad71b2ce5302211f9c61530b329a4922fc6a4
+++ b/unit_tests/tools/tiktoken_cache/9b5ad71b2ce5302211f9c61530b329a4922fc6a4
--- a/unit_tests/tools/tiktoken_util.py
+++ b/unit_tests/tools/tiktoken_util.py
@ -1,5 +1,8 @@
+import os
 import tiktoken

+script_dir = os.path.dirname(os.path.realpath(__file__))
+os.environ['TIKTOKEN_CACHE_DIR'] = os.path.join(script_dir, 'tiktoken_cache')

 def get_encoding(encoding_name: str):
    """