From 76f7f22c4430725b3b9bfda5d65ed9f9c8ac70cc Mon Sep 17 00:00:00 2001 From: "d.savchenkov" Date: Tue, 19 May 2026 15:50:52 +0300 Subject: [PATCH 1/2] [quantization] Support Evaluation of Qwen3-VL With llava-bench-in-the-wild Dataset This change adds support for llava-bench-in-the-wild benchmark to quantize_qwen3_vl_with_gptq.py. TICO-DCO-1.0-Signed-off-by: d.savchenkov --- .../quantization/evaluation/vlm_eval_utils.py | 75 ++++++++++++++----- .../examples/quantize_qwen3_vl_with_gptq.py | 63 ++++++++++++++++ 2 files changed, 118 insertions(+), 20 deletions(-) diff --git a/tico/quantization/evaluation/vlm_eval_utils.py b/tico/quantization/evaluation/vlm_eval_utils.py index be23104f..cb138357 100644 --- a/tico/quantization/evaluation/vlm_eval_utils.py +++ b/tico/quantization/evaluation/vlm_eval_utils.py @@ -179,8 +179,22 @@ def get_item_coco(ex: dict[str, Any]) -> dict[str, Any]: """ return { "image": ex["image"], - "question": ex.get("question", ""), - "golds": _extract_golds(ex.get("answer")), + "question": ex["question"], + "id": ex["id"], + "image_id": ex["question_id"], + "file_name": ex["file_name"], + "golds": ex["answer"], + } + + +def get_item_llama_bench_in_the_wild(ex: dict[str, Any]) -> dict[str, Any]: + return { + "image": ex["image"], + "question": ex["question"], + "id": ex["question_id"], + "image_id": ex["image_id"], + "file_name": ex["image_id"], + "golds": [ex["gpt_answer"]], } @@ -248,6 +262,13 @@ def get_item_alpaca(ex: dict[str, Any]) -> dict[str, Any]: ], "is_text_only": False, }, + "llama_bench": { + "default_split": "train", + "adapter": get_item_llama_bench_in_the_wild, + "candidates": [ + "lmms-lab/llava-bench-in-the-wild", + ], + }, "wikitext2": { "default_split": "test", "adapter": get_item_wikitext2, @@ -560,6 +581,7 @@ def compute_bleu_scores( def get_coco_scores_on_dataset( model, processor, + dataset_name: str, ds: Iterable[dict[str, Any]], device: str | torch.device, max_new_tokens: int = 30, @@ -620,25 +642,38 @@ def get_coco_scores_on_dataset( images: list[CocoImage] = [] annotations: list[CocoAnnotation] = [] + if "coco" in dataset_name.lower(): + get_item = get_item_coco + elif "llama_bench" in dataset_name.lower(): + get_item = get_item_llama_bench_in_the_wild + else: + raise ValueError(f"Invalid dataset_name={dataset_name}") + for i, ex in enumerate(ds, 1): - image: Any = ex["image"] - question: str = ex["question"] - id: int = ex["id"] - image_id: str = ex["question_id"] - file_name: str = ex["file_name"] - gold_answers: list[str] = ex["answer"] - - # Generate caption - pred = generate_answer( - model=model, - processor=processor, - image=image, - question=question, - device=device, - max_new_tokens=max_new_tokens, - temperature=temperature, - max_seq_len=max_seq_len, - ) + sample: dict[str, Any] = get_item(ex) + + image: Any = sample["image"] + question: str = sample["question"] + id: int = sample["id"] + image_id: str = sample["image_id"] + file_name: str = sample["file_name"] + gold_answers: list[str] = sample["golds"] + + try: + pred = generate_answer( + model=model, + processor=processor, + image=image, + question=question, + device=device, + max_new_tokens=max_new_tokens, + temperature=temperature, + max_seq_len=max_seq_len, + ) + except (ValueError, RuntimeError) as error: + print(f"[WARNING] The prompt was too long. Skipping.") + print(f"Error: {error}") + continue # Store result result: CocoResult = {"image_id": image_id, "caption": pred} diff --git a/tico/quantization/wrapq/examples/quantize_qwen3_vl_with_gptq.py b/tico/quantization/wrapq/examples/quantize_qwen3_vl_with_gptq.py index 660fe938..a9a12444 100644 --- a/tico/quantization/wrapq/examples/quantize_qwen3_vl_with_gptq.py +++ b/tico/quantization/wrapq/examples/quantize_qwen3_vl_with_gptq.py @@ -583,6 +583,40 @@ def evaluate_model( def evaluate_model_coco( + model, + processor, + device: str, + dataset_name: str, + nsamples: int = 50, + max_seq_len: Optional[int] = None, +): + """ + Evaluate a model on the mini COCO captioning benchmark. + + Args: + model: Model to evaluate. + processor: Hugging Face processor. + device: Target device string. + nsamples: Number of evaluation samples. -1 means full dataset. + max_seq_len: Optional maximum text sequence length. + + Returns: + COCO metric dictionary. + """ + + ds, _ = get_dataset(dataset_name, n=nsamples) + result = get_coco_scores_on_dataset( + model=model, + processor=processor, + dataset_name=dataset_name, + ds=ds, + device=device, + max_seq_len=max_seq_len, + ) + return result + + +def evaluate_model_llama_bench( model, processor, device: str, @@ -611,6 +645,7 @@ def evaluate_model_coco( result = get_coco_scores_on_dataset( model=model, processor=processor, + dataset_name="llama_bench", ds=ds, device=device, max_seq_len=max_seq_len, @@ -1175,6 +1210,20 @@ def evaluate_original_model(model, processor, args): model=model, processor=processor, device=args.device, + dataset_name="coco", + nsamples=args.nsamples_for_evaluation, + max_seq_len=args.max_seq_len, + ) + for metric, value in results.items(): + print(f"{metric:<10} {value:.3f}") + + if "llama_bench" in args.eval_tasks: + print("\n=== Llama Bench Evaluation (Original Model) ===") + results = evaluate_model_coco( + model=model, + processor=processor, + device=args.device, + dataset_name="llama_bench", nsamples=args.nsamples_for_evaluation, max_seq_len=args.max_seq_len, ) @@ -1269,6 +1318,20 @@ def evaluate_quantized_model(model, processor, args, original_results=None) -> N model=model, processor=processor, device=args.device, + dataset_name="coco", + nsamples=args.nsamples_for_evaluation, + max_seq_len=args.max_seq_len, + ) + for metric, value in results.items(): + print(f"{metric:<10} {value:.3f}") + + if "llama_bench" in args.eval_tasks: + print("\n=== Llama Bench Evaluation (Original Model) ===") + results = evaluate_model_coco( + model=model, + processor=processor, + device=args.device, + dataset_name="llama_bench", nsamples=args.nsamples_for_evaluation, max_seq_len=args.max_seq_len, ) From 7fa39c246199cdead2aa36bc548a4c89c5604ed9 Mon Sep 17 00:00:00 2001 From: Evgenii Maltsev Date: Thu, 21 May 2026 12:42:14 +0300 Subject: [PATCH 2/2] Fixed title of benchmark name and relaxed assert This commit fixes title of benchmark name and relaxed assert when all samples were skipped and no evaluation results were collected. Additionally, exception string was reformulated by excluding word error when a prompt is too long. TICO-DCO-1.0-Signed-off-by: Evgenii Maltsev --- .../quantization/evaluation/vlm_eval_utils.py | 41 ++++++++---- tico/quantization/recipes/evaluation/vlm.py | 1 + .../examples/quantize_qwen3_vl_with_gptq.py | 62 ++++--------------- 3 files changed, 44 insertions(+), 60 deletions(-) diff --git a/tico/quantization/evaluation/vlm_eval_utils.py b/tico/quantization/evaluation/vlm_eval_utils.py index cb138357..e11fc952 100644 --- a/tico/quantization/evaluation/vlm_eval_utils.py +++ b/tico/quantization/evaluation/vlm_eval_utils.py @@ -187,13 +187,13 @@ def get_item_coco(ex: dict[str, Any]) -> dict[str, Any]: } -def get_item_llama_bench_in_the_wild(ex: dict[str, Any]) -> dict[str, Any]: +def get_item_llava_bench_in_the_wild(ex: dict[str, Any]) -> dict[str, Any]: return { "image": ex["image"], "question": ex["question"], "id": ex["question_id"], - "image_id": ex["image_id"], - "file_name": ex["image_id"], + "image_id": ex["question_id"], # unique evaluation key + "file_name": ex["image_id"], # original image filename "golds": [ex["gpt_answer"]], } @@ -262,9 +262,9 @@ def get_item_alpaca(ex: dict[str, Any]) -> dict[str, Any]: ], "is_text_only": False, }, - "llama_bench": { + "llava_bench": { "default_split": "train", - "adapter": get_item_llama_bench_in_the_wild, + "adapter": get_item_llava_bench_in_the_wild, "candidates": [ "lmms-lab/llava-bench-in-the-wild", ], @@ -423,7 +423,9 @@ def generate_answer( image=image, question=question, return_tensors="pt", - max_seq_len=max_seq_len, + # length of (inputs + max_new_tokens) should not exceed model's max_seq_len + # because quantized model has precomputed static causal mask and RoPE for max_seq_len + max_seq_len=max_seq_len - max_new_tokens, # type: ignore[operator] ) inputs = move_inputs_to_device(inputs, device) @@ -644,8 +646,8 @@ def get_coco_scores_on_dataset( if "coco" in dataset_name.lower(): get_item = get_item_coco - elif "llama_bench" in dataset_name.lower(): - get_item = get_item_llama_bench_in_the_wild + elif "llava_bench" in dataset_name.lower(): + get_item = get_item_llava_bench_in_the_wild else: raise ValueError(f"Invalid dataset_name={dataset_name}") @@ -671,8 +673,21 @@ def get_coco_scores_on_dataset( max_seq_len=max_seq_len, ) except (ValueError, RuntimeError) as error: - print(f"[WARNING] The prompt was too long. Skipping.") - print(f"Error: {error}") + message = str(error).lower() + if not any( + marker in message + for marker in ( + "too long", + "max_position_embeddings", + "maximum context length", + "sequence length", + "truncation", + ) + ): + raise + + print("[WARNING] The prompt was too long. Skipping.") + print(f"{type(error).__name__}: {error}") continue # Store result @@ -703,7 +718,11 @@ def get_coco_scores_on_dataset( print("golds[:10]:", [repr(x) for x in gold_answers[:10]]) print("-" * 60) - assert results + if not results: + raise RuntimeError( + "No evaluation results were collected. " + "All samples may have been skipped due to prompt length errors." + ) assert images assert annotations diff --git a/tico/quantization/recipes/evaluation/vlm.py b/tico/quantization/recipes/evaluation/vlm.py index da75d4ce..4d5da93c 100644 --- a/tico/quantization/recipes/evaluation/vlm.py +++ b/tico/quantization/recipes/evaluation/vlm.py @@ -74,6 +74,7 @@ def evaluate_coco( return get_coco_scores_on_dataset( model=model, processor=processor, + dataset_name="coco", ds=dataset, device=device, max_seq_len=max_seq_len, diff --git a/tico/quantization/wrapq/examples/quantize_qwen3_vl_with_gptq.py b/tico/quantization/wrapq/examples/quantize_qwen3_vl_with_gptq.py index a9a12444..2b8b4195 100644 --- a/tico/quantization/wrapq/examples/quantize_qwen3_vl_with_gptq.py +++ b/tico/quantization/wrapq/examples/quantize_qwen3_vl_with_gptq.py @@ -582,7 +582,7 @@ def evaluate_model( return results -def evaluate_model_coco( +def evaluate_model_with_coco_score( model, processor, device: str, @@ -591,12 +591,13 @@ def evaluate_model_coco( max_seq_len: Optional[int] = None, ): """ - Evaluate a model on the mini COCO captioning benchmark. + Evaluate a model on the provided dataset with COCO score Args: model: Model to evaluate. processor: Hugging Face processor. device: Target device string. + dataset_name: Dataset name for evaluation nsamples: Number of evaluation samples. -1 means full dataset. max_seq_len: Optional maximum text sequence length. @@ -616,43 +617,6 @@ def evaluate_model_coco( return result -def evaluate_model_llama_bench( - model, - processor, - device: str, - nsamples: int = 50, - max_seq_len: Optional[int] = None, -): - """ - Evaluate a model on the mini COCO captioning benchmark. - - Args: - model: Model to evaluate. - processor: Hugging Face processor. - device: Target device string. - nsamples: Number of evaluation samples. -1 means full dataset. - max_seq_len: Optional maximum text sequence length. - - Returns: - COCO metric dictionary. - """ - with ( - io.StringIO() as buffer, - contextlib.redirect_stdout(buffer), - contextlib.redirect_stderr(buffer), - ): - ds, _ = get_dataset("coco", n=nsamples) - result = get_coco_scores_on_dataset( - model=model, - processor=processor, - dataset_name="llama_bench", - ds=ds, - device=device, - max_seq_len=max_seq_len, - ) - return result - - def move_batch_to_device( batch: dict[str, Any], device: str | torch.device, @@ -1206,7 +1170,7 @@ def evaluate_original_model(model, processor, args): if "coco" in args.eval_tasks: print("\n=== COCO Evaluation (Original Model) ===") - results = evaluate_model_coco( + results = evaluate_model_with_coco_score( model=model, processor=processor, device=args.device, @@ -1217,13 +1181,13 @@ def evaluate_original_model(model, processor, args): for metric, value in results.items(): print(f"{metric:<10} {value:.3f}") - if "llama_bench" in args.eval_tasks: - print("\n=== Llama Bench Evaluation (Original Model) ===") - results = evaluate_model_coco( + if "llava_bench" in args.eval_tasks: + print("\n=== Llava Bench Evaluation (Original Model) ===") + results = evaluate_model_with_coco_score( model=model, processor=processor, device=args.device, - dataset_name="llama_bench", + dataset_name="llava_bench", nsamples=args.nsamples_for_evaluation, max_seq_len=args.max_seq_len, ) @@ -1314,7 +1278,7 @@ def evaluate_quantized_model(model, processor, args, original_results=None) -> N if "coco" in args.eval_tasks: print("\n=== COCO Evaluation (Quantized Model) ===") - results = evaluate_model_coco( + results = evaluate_model_with_coco_score( model=model, processor=processor, device=args.device, @@ -1325,13 +1289,13 @@ def evaluate_quantized_model(model, processor, args, original_results=None) -> N for metric, value in results.items(): print(f"{metric:<10} {value:.3f}") - if "llama_bench" in args.eval_tasks: - print("\n=== Llama Bench Evaluation (Original Model) ===") - results = evaluate_model_coco( + if "llava_bench" in args.eval_tasks: + print("\n=== Llava Bench Evaluation (Quantized Model) ===") + results = evaluate_model_with_coco_score( model=model, processor=processor, device=args.device, - dataset_name="llama_bench", + dataset_name="llava_bench", nsamples=args.nsamples_for_evaluation, max_seq_len=args.max_seq_len, )