fix(generators): raise clear error on HF InferenceAPI 404 response

garak/generators/huggingface.py

@@ -43,6 +43,10 @@ class HFInternalServerError(GarakException):
     pass
 
 
+class ModelNotFoundError(GarakException):
+    pass
+
+
 class Pipeline(Generator, HFCompatible):
     """Get text generations from a locally-run Hugging Face pipeline"""
 
@@ -256,6 +260,13 @@ def _call_model(
             self.wait_for_model = True
             raise HFLoadingException
 
+        if req_response.status_code == 404:
+            raise ModelNotFoundError(
+                f"🤗 Inference API returned 404 for model '{self.name}'. The model may not "
+                "be available via the legacy Inference API and may require selecting an "
+                "Inference Provider instead; see https://huggingface.co/docs/inference-providers"
+            )
+
         # if we get this far, reset the model load wait. let's hope 503 is only for model loading :|
         if self.wait_for_model:
             self.wait_for_model = False

tests/generators/test_huggingface.py

@@ -99,6 +99,21 @@ def test_inference(mocker, hf_mock_response, hf_generator_config):
         assert isinstance(item, Message)
 
 
+def test_inference_model_not_found(mocker, hf_generator_config):
+    target_name = "Qwen/Qwen2.5-7B-Instruct"
+    mock_resp = requests.Response()
+    mock_resp.status_code = 404
+    mock_resp._content = b"Not Found"
+    mocker.patch.object(requests, "request", return_value=mock_resp)
+
+    g = garak.generators.huggingface.InferenceAPI(
+        target_name, config_root=hf_generator_config
+    )
+    conv = Conversation([Turn("user", Message(""))])
+    with pytest.raises(garak.generators.huggingface.ModelNotFoundError):
+        g.generate(conv)
+
+
 def test_endpoint(mocker, hf_mock_response, hf_generator_config):
     target_name = "https://localhost:8000/gpt2"
     mock_request = mocker.patch.object(requests, "post", return_value=hf_mock_response)