diff --git a/examples/chat/README.md b/examples/chat/README.md index 56359f2..5c23e1e 100644 --- a/examples/chat/README.md +++ b/examples/chat/README.md @@ -44,9 +44,12 @@ print(response.choices[0].message.content.root) ### Streaming Chat Completion ```python +import json + +from pydantic import ValidationError + from inference_gateway import InferenceGatewayClient, Message from inference_gateway.models import CreateChatCompletionStreamResponse -import json client = InferenceGatewayClient("http://localhost:8080/v1") @@ -65,8 +68,9 @@ for chunk in stream: # Parse the raw JSON data data = json.loads(chunk.data) - # Unmarshal to the structured model for type safety. The first - # chunk carries only `role` (no content) and validates fine. + # Unmarshal to the structured model for type safety. The model is + # strict: a delta without `content`, or a choice with a null + # `finish_reason`, raises `ValidationError` - skip those chunks. structured_chunk = CreateChatCompletionStreamResponse.model_validate(data) if structured_chunk.choices and len(structured_chunk.choices) > 0: @@ -74,8 +78,8 @@ for chunk in stream: if choice.delta.content: print(choice.delta.content, end="", flush=True) - except json.JSONDecodeError: - pass + except (json.JSONDecodeError, ValidationError): + continue ``` ## Error Handling diff --git a/examples/chat/main.py b/examples/chat/main.py index b907fa3..64c64b4 100644 --- a/examples/chat/main.py +++ b/examples/chat/main.py @@ -1,6 +1,8 @@ import json import os +from pydantic import ValidationError + from inference_gateway import InferenceGatewayClient, Message from inference_gateway.client import InferenceGatewayAPIError, InferenceGatewayError from inference_gateway.models import CreateChatCompletionStreamResponse, SSEvent @@ -77,6 +79,9 @@ def main() -> None: # Handle non-JSON SSE data print(f"[Non-JSON chunk: {chunk.data}]", end="", flush=True) + except ValidationError: + continue + print("\n") except (InferenceGatewayAPIError, InferenceGatewayError) as e: