Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 26 additions & 25 deletions src/server.cc
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// Copyright 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
Expand Down Expand Up @@ -452,14 +452,21 @@ InferenceServer::IsReady(bool* ready)
// backend has become unhealthy at runtime (e.g., Python backend
// stub process died).
if (vs.second.first == ModelReadyState::READY) {
bool model_ready = false;
Status status = ModelIsReady(mv.first.name_, vs.first, &model_ready);
if (!status.IsOk() || !model_ready) {
LOG_VERBOSE(1) << "Model '" << mv.first.name_ << "' version "
<< vs.first
std::shared_ptr<Model> model;
if (!GetModel(mv.first, vs.first, &model).IsOk()) {
LOG_VERBOSE(1) << "Model '" << mv.first << "' version " << vs.first
<< " failed model lookup during server readiness "
"evaluation";
*ready = false;
goto strict_done;
}
Status status = model->IsReady();
if (!status.IsOk()) {
LOG_VERBOSE(1) << "Model '" << mv.first << "' version " << vs.first
<< " is in READY lifecycle state but failed "
"runtime readiness check during server "
"readiness evaluation";
"readiness evaluation: "
<< status.Message();
*ready = false;
goto strict_done;
}
Expand All @@ -473,8 +480,7 @@ InferenceServer::IsReady(bool* ready)
}

Status
InferenceServer::ModelIsReady(
const std::string& model_name, const int64_t model_version, bool* ready)
InferenceServer::ModelIsReady(const Model& model, bool* ready)
{
*ready = false;

Expand All @@ -484,22 +490,17 @@ InferenceServer::ModelIsReady(

ScopedAtomicIncrement inflight(inflight_request_counter_);

std::shared_ptr<Model> model;
if (GetModel(model_name, model_version, &model).IsOk()) {
ModelReadyState state;
if (model_repository_manager_
->ModelState(model_name, model->Version(), &state)
.IsOk()) {
*ready = (state == ModelReadyState::READY);
if (*ready) {
Status status = model->IsReady();
if (!status.IsOk()) {
*ready = false;
LOG_VERBOSE(1) << "Model '" << model_name << "' version "
<< model->Version()
<< " is not ready: " << status.Message();
}
}
ModelReadyState state;
if (model_repository_manager_
->ModelState(model.Name(), model.Version(), &state)
.IsOk() &&
(state == ModelReadyState::READY)) {
const Status status = model.IsReady();
*ready = status.IsOk();
if (!*ready) {
LOG_VERBOSE(1) << "Model '" << model.Name() << "' version "
<< model.Version()
<< " is not ready: " << status.Message();
}
}

Expand Down
10 changes: 7 additions & 3 deletions src/server.h
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2018-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// Copyright 2018-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
Expand Down Expand Up @@ -99,8 +99,7 @@ class InferenceServer {
Status IsReady(bool* ready);

// Model health
Status ModelIsReady(
const std::string& model_name, const int64_t model_version, bool* ready);
Status ModelIsReady(const Model& model, bool* ready);

// Return the ready versions of specific model
Status ModelReadyVersions(
Expand Down Expand Up @@ -289,6 +288,11 @@ class InferenceServer {
void SetRepoAgentDir(const std::string& d) { repoagent_dir_ = d; }

// Return the requested model object.
//
// WARNING: This overload resolves by model name only. When model namespacing
// is enabled and the same model name exists in multiple namespaces, lookup
// fails with an ambiguity error. Use the ModelIdentifier overload below to
// specify namespace and name explicitly.
Status GetModel(
const std::string& model_name, const int64_t model_version,
std::shared_ptr<Model>* model)
Expand Down
7 changes: 4 additions & 3 deletions src/tritonserver.cc
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// Copyright 2019-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions
Expand Down Expand Up @@ -2730,8 +2730,9 @@ TRITONSERVER_ServerModelIsReady(
{
tc::InferenceServer* lserver = reinterpret_cast<tc::InferenceServer*>(server);

RETURN_IF_STATUS_ERROR(
lserver->ModelIsReady(model_name, model_version, ready));
std::shared_ptr<tc::Model> model;
RETURN_IF_STATUS_ERROR(lserver->GetModel(model_name, model_version, &model));
RETURN_IF_STATUS_ERROR(lserver->ModelIsReady(*model, ready));
return nullptr; // Success
}

Expand Down
Loading