Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions examples/chat/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,9 +44,12 @@ print(response.choices[0].message.content.root)
### Streaming Chat Completion

```python
import json

from pydantic import ValidationError

from inference_gateway import InferenceGatewayClient, Message
from inference_gateway.models import CreateChatCompletionStreamResponse
import json

client = InferenceGatewayClient("http://localhost:8080/v1")

Expand All @@ -65,17 +68,18 @@ for chunk in stream:
# Parse the raw JSON data
data = json.loads(chunk.data)

# Unmarshal to the structured model for type safety. The first
# chunk carries only `role` (no content) and validates fine.
# Unmarshal to the structured model for type safety. The model is
# strict: a delta without `content`, or a choice with a null
# `finish_reason`, raises `ValidationError` - skip those chunks.
structured_chunk = CreateChatCompletionStreamResponse.model_validate(data)

if structured_chunk.choices and len(structured_chunk.choices) > 0:
choice = structured_chunk.choices[0]
if choice.delta.content:
print(choice.delta.content, end="", flush=True)

except json.JSONDecodeError:
pass
except (json.JSONDecodeError, ValidationError):
continue
```

## Error Handling
Expand Down
5 changes: 5 additions & 0 deletions examples/chat/main.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import json
import os

from pydantic import ValidationError

from inference_gateway import InferenceGatewayClient, Message
from inference_gateway.client import InferenceGatewayAPIError, InferenceGatewayError
from inference_gateway.models import CreateChatCompletionStreamResponse, SSEvent
Expand Down Expand Up @@ -77,6 +79,9 @@ def main() -> None:
# Handle non-JSON SSE data
print(f"[Non-JSON chunk: {chunk.data}]", end="", flush=True)

except ValidationError:
continue

print("\n")

except (InferenceGatewayAPIError, InferenceGatewayError) as e:
Expand Down
Loading