nkozobrod
86c579be84
- _generate_one_stream: streams responses, optional live repeat-loop detection (abort) that closes the stream early and flags the partial result as repeat=True so the existing retry loop re-runs with higher temperature - _should_retry honors result.repeat; stream_mode threaded through both first attempt and retry calls in chat_completions_batch - GenerationResult.repeat field; SURYA_STREAM_MODE setting (default off) - diagnostic retry logs (reason=repeat|error|detected, temp) - docker-compose: SURYA_STREAM_MODE=off, 4090 GPU profile, logs volume - logger.yaml: TimedRotatingFileHandler for error handler (fixes startup crash without maxBytes)
49 lines
1.5 KiB
Python
49 lines
1.5 KiB
Python
from dataclasses import dataclass, field
|
|
from typing import Any, List, Optional
|
|
|
|
from PIL import Image
|
|
|
|
|
|
PROMPT_TYPE_LAYOUT = "layout"
|
|
PROMPT_TYPE_BLOCK = "block"
|
|
PROMPT_TYPE_TABLE_REC = "table_rec"
|
|
PROMPT_TYPE_HIGH_ACCURACY_BBOX = "high_accuracy_bbox"
|
|
|
|
|
|
@dataclass
|
|
class BatchInputItem:
|
|
image: Image.Image
|
|
prompt_type: str
|
|
prompt: Optional[str] = None # If set, overrides the default prompt for prompt_type
|
|
max_tokens: Optional[int] = None
|
|
request_logprobs: bool = False
|
|
# vllm-native guided decoding — JSON schema, regex, or grammar string.
|
|
# When set, the server constrains the decode tokens to match the schema.
|
|
guided_json: Optional[dict] = None
|
|
guided_regex: Optional[str] = None
|
|
metadata: dict = field(default_factory=dict) # Free-form, passes through to output
|
|
|
|
|
|
@dataclass
|
|
class GenerationResult:
|
|
raw: str
|
|
token_count: int
|
|
error: bool = False
|
|
# True when streaming mode aborted generation early due to a detected
|
|
# token repetition loop (see SURYA_STREAM_MODE=abort)
|
|
repeat: bool = False
|
|
# Mean of exp(logprob) across response tokens, if logprobs requested
|
|
mean_token_prob: Optional[float] = None
|
|
# Per-token logprobs (raw OpenAI-style content list), if requested - phase 2 use
|
|
logprobs: Optional[List[Any]] = None
|
|
|
|
|
|
@dataclass
|
|
class BatchOutputItem:
|
|
raw: str
|
|
token_count: int
|
|
error: bool
|
|
mean_token_prob: Optional[float] = None
|
|
logprobs: Optional[List[Any]] = None
|
|
metadata: dict = field(default_factory=dict)
|