Skip to content

fix(model): recognize symlink aliases as the loaded model during inline loading #436

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Anai-Guo wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(model): recognize symlink aliases as the loaded model during inline loading #436

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions common/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,12 @@ async def load_model_gen(model_path: pathlib.Path, **kwargs):
if container and container.model:
loaded_model_name = container.model_dir.name

if loaded_model_name == model_path.name and container.loaded:
# Compare resolved paths, not just directory names, so a symlink
# alias pointing at the already-loaded model is recognized as the
# same model instead of triggering a spurious unload/reload (#379).
already_loaded = container.model_dir.resolve() == model_path.resolve()

if already_loaded and container.loaded:
xlogger.info(f'Model "{loaded_model_name}" is already loaded')

# Emit a terminal progress event so API clients always
Expand Down Expand Up @@ -320,4 +325,4 @@ def check_context_length(
container.validate_context_length(prompt, params, mm_embeddings)
except ContextLengthExceededError as exc:
error_message = handle_request_error(str(exc), exc_info=False).error.message
raise ContextLengthHTTPException(error_message) from exc
raise ContextLengthHTTPException(error_message) from exc
12 changes: 8 additions & 4 deletions endpoints/OAI/utils/common_.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,13 @@ def aggregate_usage_stats(usage_stats_list: list[UsageStats]) -> UsageStats:
async def load_inline_model(model_name: str, request: Request):
"""Load a model from the data.model parameter"""

# Return if the model container already exists and the model is fully loaded
if model.container and model.container.model_dir.name == model_name and model.container.loaded:
return
# Return if the model container already exists and the model is fully loaded.
# Compare resolved paths so a symlink alias for the loaded model is treated
# as already loaded instead of triggering a spurious reload (#379).
if model.container and model.container.loaded:
requested_dir = pathlib.Path(config.model.model_dir) / model_name
if model.container.model_dir.resolve() == requested_dir.resolve():
return

# Return if inline loading is disabled
# Also warn if an admin key is used
Expand Down Expand Up @@ -113,4 +117,4 @@ async def load_inline_model(model_name: str, request: Request):
await model.load_model(
model_path,
draft_model=config.draft_model.model_dump(include={"draft_model_dir"}),
)
)