{ "meta": { "status": "portfolio_demo_v2_ready", "title": "BizHallu Interactive Demo V2", "case_count": 9, "span_count": 15, "primary_question_ids": [ "q_0064", "q_0069" ], "label_lock_basis": "assistant_full_review", "label_lock_status": "presentation_labels_locked", "simple_threshold": 0.334333, "entropy_threshold": 0.004817, "energy_threshold": 0.000225, "best_test_auprc": 0.835073, "best_test_f1": 0.779412, "annotation_status": "15 selected spans with additional assistant presentation review", "independent_human_annotation": false, "automatic_claim_extraction": false, "metric_selection_status": "exploratory_test_maxima", "presentation_revision": "english_evidence_review_2026_09_05", "business_metric_note": "Positive and negative transaction values are sign-based accounting quantities. Negative value is not a verified measure of physical returns; fees and adjustments may be included. Merchandise scope uses a stock-code heuristic, and the historical product grain is stock code plus description.", "guardrail": "Selected presentation spans only; no independent human annotation, automatic claim extraction, or production detector claim." }, "filters": { "fact_types": [ "comparison_direction", "currency_amount", "percentage", "product_name", "ranking" ], "labels": [ "correct_key_fact", "hallucinated_key_fact" ], "outcomes": [ "caught", "missed", "false alarm", "cleared" ], "detectors": [ { "id": "simple", "label": "Top-2 margin", "score_field": "one_minus_min_top2_margin", "description": "Flags spans when the model's top two token choices were close at the least-certain token." }, { "id": "entropy", "label": "Entropy", "score_field": "mean_token_entropy", "description": "Flags spans with higher average uncertainty across their generated tokens." }, { "id": "energy", "label": "Top-2 residual mass", "score_field": "mean_spilled_probability_mass_after_top2", "description": "Probability mass outside the top two token choices is a concentration control, not an independent energy method or semantic verifier." } ] }, "cases": [ { "question_id": "q_0064", "is_primary": true, "split": "test", "question_type": "top3_products_month", "difficulty": "medium", "question": "What were the top 3 products by merchandise net revenue in April 2011? Give the rank, stock code, product name, and net revenue.", "gold_short_answer": "Top 3 products in April 2011: 1. 22423 (REGENCY CAKESTAND 3 TIER), GBP 14,280.90; 2. 47566 (PARTY BUNTING), GBP 10,323.87; 3. 22084 (PAPER CHAIN KIT EMPIRE), GBP 6,619.51.", "generated_text": "The top 3 products by merchandise net revenue in April 2011 were: \n1. **REGENCY CAKESTAND 3 TIER** (rank 1) with a net revenue of GBP 14,280.90. \n2. **PARTY BUNTING** (rank 2) with a net revenue of GBP 10,323.87. \n3. **WOODEN UNION JACK BUNTING** (rank 3) with a net revenue of GBP 4,173.18.", "evidence_columns": [ "stock_code", "description", "net_revenue", "gross_positive_revenue", "cancellation_revenue" ], "prompt_evidence_rows": [ { "stock_code": "22423", "description": "REGENCY CAKESTAND 3 TIER", "net_revenue": 14280.9, "gross_positive_revenue": 14812.95, "cancellation_revenue": -532.05 }, { "stock_code": "22499", "description": "WOODEN UNION JACK BUNTING", "net_revenue": 4173.18, "gross_positive_revenue": 4191.03, "cancellation_revenue": -17.85 }, { "stock_code": "47566", "description": "PARTY BUNTING", "net_revenue": 10323.87, "gross_positive_revenue": 10778.47, "cancellation_revenue": -454.6 }, { "stock_code": "85099B", "description": "JUMBO BAG RED RETROSPOT", "net_revenue": 4870.92, "gross_positive_revenue": 4878.08, "cancellation_revenue": -7.16 }, { "stock_code": "85123A", "description": "WHITE HANGING HEART T-LIGHT HOLDER", "net_revenue": 5078.59, "gross_positive_revenue": 10017.79, "cancellation_revenue": -4939.2 }, { "stock_code": "22084", "description": "PAPER CHAIN KIT EMPIRE", "net_revenue": 6619.51, "gross_positive_revenue": 6634.26, "cancellation_revenue": -14.75 }, { "stock_code": "84879", "description": "ASSORTED COLOUR BIRD ORNAMENT", "net_revenue": 4174.24, "gross_positive_revenue": 4174.24, "cancellation_revenue": 0.0 }, { "stock_code": "22087", "description": "PAPER BUNTING WHITE LACE", "net_revenue": 3512.24, "gross_positive_revenue": 3579.69, "cancellation_revenue": -67.45 } ], "takeaway": "The third product and its amount match a source row, but that product is seventh in the shown evidence, not third.", "spans": [ { "annotation_id": "ann_full100_draft_q_0064_004", "confirmation_item_id": "confirm_006", "question_id": "q_0064", "span_text": "GBP 14,280.90", "span_start_char": 131, "span_end_char": 144, "fact_type": "currency_amount", "label": "correct_key_fact", "publish_use": "primary_demo", "presentation_use": "paired_contrast_example", "demo_priority": "high", "review_note": "The rank-1 amount is supported by gold evidence. It works well in the April 2011 paired example, where nearby spans include both supported and hallucinated claims.", "detector_role_text": "simple false_positive via one_minus_min_top2_margin", "simple_score": 0.799101, "entropy_score": 0.155575, "energy_score": 0.016624, "simple_outcome": "false alarm", "entropy_outcome": "false alarm", "energy_outcome": "false alarm", "gold_short_answer": "Top 3 products in April 2011: 1. 22423 (REGENCY CAKESTAND 3 TIER), GBP 14,280.90; 2. 47566 (PARTY BUNTING), GBP 10,323.87; 3. 22084 (PAPER CHAIN KIT EMPIRE), GBP 6,619.51." }, { "annotation_id": "ann_full100_draft_q_0064_006", "confirmation_item_id": "confirm_007", "question_id": "q_0064", "span_text": "PARTY BUNTING", "span_start_char": 153, "span_end_char": 166, "fact_type": "product_name", "label": "correct_key_fact", "publish_use": "primary_demo", "presentation_use": "paired_contrast_example", "demo_priority": "high", "review_note": "The rank-2 product name is supported by gold evidence. This is a clean correct span inside the April 2011 paired contrast example.", "detector_role_text": "simple false_positive via one_minus_min_top2_margin", "simple_score": 0.769828, "entropy_score": 0.189533, "energy_score": 0.012085, "simple_outcome": "false alarm", "entropy_outcome": "false alarm", "energy_outcome": "false alarm", "gold_short_answer": "Top 3 products in April 2011: 1. 22423 (REGENCY CAKESTAND 3 TIER), GBP 14,280.90; 2. 47566 (PARTY BUNTING), GBP 10,323.87; 3. 22084 (PAPER CHAIN KIT EMPIRE), GBP 6,619.51." }, { "annotation_id": "ann_full100_draft_q_0064_008", "confirmation_item_id": "confirm_008", "question_id": "q_0064", "span_text": "3.", "span_start_char": 217, "span_end_char": 219, "fact_type": "ranking", "label": "hallucinated_key_fact", "publish_use": "primary_demo", "presentation_use": "paired_contrast_example", "demo_priority": "high", "review_note": "The rank-3 marker is attached to WOODEN UNION JACK BUNTING, but gold rank 3 is PAPER CHAIN KIT EMPIRE. This is a clear hallucinated ranking span.", "detector_role_text": "energy false_negative via mean_spilled_probability_mass_after_top2; simple false_negative via one_minus_min_top2_margin", "simple_score": 5.99999999995e-06, "entropy_score": 3.6e-05, "energy_score": 1e-06, "simple_outcome": "missed", "entropy_outcome": "missed", "energy_outcome": "missed", "gold_short_answer": "Top 3 products in April 2011: 1. 22423 (REGENCY CAKESTAND 3 TIER), GBP 14,280.90; 2. 47566 (PARTY BUNTING), GBP 10,323.87; 3. 22084 (PAPER CHAIN KIT EMPIRE), GBP 6,619.51." }, { "annotation_id": "ann_full100_draft_q_0064_010", "confirmation_item_id": "confirm_009", "question_id": "q_0064", "span_text": "GBP 4,173.18", "span_start_char": 281, "span_end_char": 293, "fact_type": "currency_amount", "label": "hallucinated_key_fact", "publish_use": "primary_demo", "presentation_use": "paired_contrast_example", "demo_priority": "high", "review_note": "The rank-3 amount is for the wrong generated product and does not match the gold rank-3 revenue. This is a strong amount-level hallucination in the April 2011 example.", "detector_role_text": "energy false_negative via mean_spilled_probability_mass_after_top2; simple false_negative via one_minus_min_top2_margin", "simple_score": 1.9e-05, "entropy_score": 5.9e-05, "energy_score": 1e-06, "simple_outcome": "missed", "entropy_outcome": "missed", "energy_outcome": "missed", "gold_short_answer": "Top 3 products in April 2011: 1. 22423 (REGENCY CAKESTAND 3 TIER), GBP 14,280.90; 2. 47566 (PARTY BUNTING), GBP 10,323.87; 3. 22084 (PAPER CHAIN KIT EMPIRE), GBP 6,619.51." } ], "span_count": 4, "presentation_walkthrough": { "review_basis": "assistant-curated historical walkthrough with deterministic source checks; not independent human labels or verifier predictions", "automatic_claim_extraction": false, "scope": "Ranking within the eight rows shown to Qwen; historical corpus scope is separately documented.", "claims": [ { "quote": "1. **REGENCY CAKESTAND 3 TIER** (rank 1) with a net revenue of GBP 14,280.90.", "start": 68, "end": 145, "period": "April 2011", "product_name": "REGENCY CAKESTAND 3 TIER", "stock_code_from_evidence": "22423", "stated_rank": 1, "rank_in_shown_evidence": 1, "amount_lexical": "14,280.90", "row_value_supported": true, "rank_supported_in_shown_evidence": true, "source_row": 1, "expected_product_at_stated_rank": "REGENCY CAKESTAND 3 TIER", "expected_amount_at_stated_rank": "14,280.90" }, { "quote": "2. **PARTY BUNTING** (rank 2) with a net revenue of GBP 10,323.87.", "start": 148, "end": 214, "period": "April 2011", "product_name": "PARTY BUNTING", "stock_code_from_evidence": "47566", "stated_rank": 2, "rank_in_shown_evidence": 2, "amount_lexical": "10,323.87", "row_value_supported": true, "rank_supported_in_shown_evidence": true, "source_row": 3, "expected_product_at_stated_rank": "PARTY BUNTING", "expected_amount_at_stated_rank": "10,323.87" }, { "quote": "3. **WOODEN UNION JACK BUNTING** (rank 3) with a net revenue of GBP 4,173.18.", "start": 217, "end": 294, "period": "April 2011", "product_name": "WOODEN UNION JACK BUNTING", "stock_code_from_evidence": "22499", "stated_rank": 3, "rank_in_shown_evidence": 7, "amount_lexical": "4,173.18", "row_value_supported": true, "rank_supported_in_shown_evidence": false, "source_row": 2, "expected_product_at_stated_rank": "PAPER CHAIN KIT EMPIRE", "expected_amount_at_stated_rank": "6,619.51" } ], "information_timing_note": "A list marker is generated before the following product and amount. Its low token uncertainty does not establish that the model was confident about the completed business relationship. Raw teacher-forced token scores and completed-answer evidence checks have different information budgets.", "coverage_note": "Both answers omit the requested stock codes. The six curated relationships are not a whole-answer correctness score." } }, { "question_id": "q_0069", "is_primary": true, "split": "test", "question_type": "top3_products_month", "difficulty": "medium", "question": "What were the top 3 products by merchandise net revenue in September 2011? Give the rank, stock code, product name, and net revenue.", "gold_short_answer": "Top 3 products in September 2011: 1. 23243 (SET OF TEA COFFEE SUGAR TINS PANTRY), GBP 9,971.51; 2. 22423 (REGENCY CAKESTAND 3 TIER), GBP 9,315.03; 3. 85099B (JUMBO BAG RED RETROSPOT), GBP 8,630.45.", "generated_text": "The top 3 products by merchandise net revenue in September 2011 were: \n1. **JUMBO BAG RED RETROSPOT** (rank 1) with a net revenue of GBP 8630.45. \n2. **PAPER CHAIN KIT 50'S CHRISTMAS** (rank 2) with a net revenue of GBP 5997.25. \n3. **REGENCY CAKESTAND 3 TIER** (rank 3) with a net revenue of GBP 9315.03.", "evidence_columns": [ "stock_code", "description", "net_revenue", "gross_positive_revenue", "cancellation_revenue" ], "prompt_evidence_rows": [ { "stock_code": "85123A", "description": "WHITE HANGING HEART T-LIGHT HOLDER", "net_revenue": 6880.76, "gross_positive_revenue": 6945.66, "cancellation_revenue": -64.9 }, { "stock_code": "22720", "description": "SET OF 3 CAKE TINS PANTRY DESIGN", "net_revenue": 6250.48, "gross_positive_revenue": 6285.13, "cancellation_revenue": -34.65 }, { "stock_code": "22423", "description": "REGENCY CAKESTAND 3 TIER", "net_revenue": 9315.03, "gross_positive_revenue": 9619.53, "cancellation_revenue": -304.5 }, { "stock_code": "23355", "description": "HOT WATER BOTTLE KEEP CALM", "net_revenue": 6229.79, "gross_positive_revenue": 6254.54, "cancellation_revenue": -24.75 }, { "stock_code": "23243", "description": "SET OF TEA COFFEE SUGAR TINS PANTRY", "net_revenue": 9971.51, "gross_positive_revenue": 9981.41, "cancellation_revenue": -9.9 }, { "stock_code": "22086", "description": "PAPER CHAIN KIT 50'S CHRISTMAS", "net_revenue": 5997.25, "gross_positive_revenue": 5997.25, "cancellation_revenue": 0.0 }, { "stock_code": "85099B", "description": "JUMBO BAG RED RETROSPOT", "net_revenue": 8630.45, "gross_positive_revenue": 8880.17, "cancellation_revenue": -249.72 }, { "stock_code": "47566", "description": "PARTY BUNTING", "net_revenue": 6113.7, "gross_positive_revenue": 6383.15, "cancellation_revenue": -269.45 } ], "takeaway": "All three product-amount pairs match source rows; their stated ranks do not. Correct copying is not correct ranking.", "spans": [ { "annotation_id": "ann_full100_draft_q_0069_005", "confirmation_item_id": "confirm_011", "question_id": "q_0069", "span_text": "2.", "span_start_char": 149, "span_end_char": 151, "fact_type": "ranking", "label": "hallucinated_key_fact", "publish_use": "primary_demo", "presentation_use": "strong_false_negative_example", "demo_priority": "high", "review_note": "The rank-2 marker is tied to PAPER CHAIN KIT 50'S CHRISTMAS, while gold rank 2 is REGENCY CAKESTAND 3 TIER. This is a clear detector-missed hallucinated ranking span.", "detector_role_text": "energy false_negative via mean_spilled_probability_mass_after_top2; simple false_negative via one_minus_min_top2_margin", "simple_score": 1.6e-05, "entropy_score": 7.9e-05, "energy_score": 2e-06, "simple_outcome": "missed", "entropy_outcome": "missed", "energy_outcome": "missed", "gold_short_answer": "Top 3 products in September 2011: 1. 23243 (SET OF TEA COFFEE SUGAR TINS PANTRY), GBP 9,971.51; 2. 22423 (REGENCY CAKESTAND 3 TIER), GBP 9,315.03; 3. 85099B (JUMBO BAG RED RETROSPOT), GBP 8,630.45." }, { "annotation_id": "ann_full100_draft_q_0069_008", "confirmation_item_id": "confirm_012", "question_id": "q_0069", "span_text": "3.", "span_start_char": 233, "span_end_char": 235, "fact_type": "ranking", "label": "hallucinated_key_fact", "publish_use": "primary_demo", "presentation_use": "strong_false_negative_example", "demo_priority": "high", "review_note": "The rank-3 marker is tied to REGENCY CAKESTAND 3 TIER, while gold rank 3 is JUMBO BAG RED RETROSPOT. This is a clear detector-missed hallucinated ranking span.", "detector_role_text": "energy false_negative via mean_spilled_probability_mass_after_top2; simple false_negative via one_minus_min_top2_margin", "simple_score": 6.99999999998e-06, "entropy_score": 4.8e-05, "energy_score": 2e-06, "simple_outcome": "missed", "entropy_outcome": "missed", "energy_outcome": "missed", "gold_short_answer": "Top 3 products in September 2011: 1. 23243 (SET OF TEA COFFEE SUGAR TINS PANTRY), GBP 9,971.51; 2. 22423 (REGENCY CAKESTAND 3 TIER), GBP 9,315.03; 3. 85099B (JUMBO BAG RED RETROSPOT), GBP 8,630.45." }, { "annotation_id": "ann_full100_draft_q_0069_010", "confirmation_item_id": "confirm_013", "question_id": "q_0069", "span_text": "GBP 9315.03", "span_start_char": 296, "span_end_char": 307, "fact_type": "currency_amount", "label": "hallucinated_key_fact", "publish_use": "primary_demo", "presentation_use": "strong_false_negative_example", "demo_priority": "high", "review_note": "The amount is a real rank-2 amount but is used for rank 3. This is a useful example of a value that is locally plausible but wrong in its assigned business context.", "detector_role_text": "energy false_negative via mean_spilled_probability_mass_after_top2; simple false_negative via one_minus_min_top2_margin", "simple_score": 3.2e-05, "entropy_score": 5.9e-05, "energy_score": 2e-06, "simple_outcome": "missed", "entropy_outcome": "missed", "energy_outcome": "missed", "gold_short_answer": "Top 3 products in September 2011: 1. 23243 (SET OF TEA COFFEE SUGAR TINS PANTRY), GBP 9,971.51; 2. 22423 (REGENCY CAKESTAND 3 TIER), GBP 9,315.03; 3. 85099B (JUMBO BAG RED RETROSPOT), GBP 8,630.45." } ], "span_count": 3, "presentation_walkthrough": { "review_basis": "assistant-curated historical walkthrough with deterministic source checks; not independent human labels or verifier predictions", "automatic_claim_extraction": false, "scope": "Ranking within the eight rows shown to Qwen; historical corpus scope is separately documented.", "claims": [ { "quote": "1. **JUMBO BAG RED RETROSPOT** (rank 1) with a net revenue of GBP 8630.45.", "start": 72, "end": 146, "period": "September 2011", "product_name": "JUMBO BAG RED RETROSPOT", "stock_code_from_evidence": "85099B", "stated_rank": 1, "rank_in_shown_evidence": 3, "amount_lexical": "8630.45", "row_value_supported": true, "rank_supported_in_shown_evidence": false, "source_row": 7, "expected_product_at_stated_rank": "SET OF TEA COFFEE SUGAR TINS PANTRY", "expected_amount_at_stated_rank": "9,971.51" }, { "quote": "2. **PAPER CHAIN KIT 50'S CHRISTMAS** (rank 2) with a net revenue of GBP 5997.25.", "start": 149, "end": 230, "period": "September 2011", "product_name": "PAPER CHAIN KIT 50'S CHRISTMAS", "stock_code_from_evidence": "22086", "stated_rank": 2, "rank_in_shown_evidence": 8, "amount_lexical": "5997.25", "row_value_supported": true, "rank_supported_in_shown_evidence": false, "source_row": 6, "expected_product_at_stated_rank": "REGENCY CAKESTAND 3 TIER", "expected_amount_at_stated_rank": "9,315.03" }, { "quote": "3. **REGENCY CAKESTAND 3 TIER** (rank 3) with a net revenue of GBP 9315.03.", "start": 233, "end": 308, "period": "September 2011", "product_name": "REGENCY CAKESTAND 3 TIER", "stock_code_from_evidence": "22423", "stated_rank": 3, "rank_in_shown_evidence": 2, "amount_lexical": "9315.03", "row_value_supported": true, "rank_supported_in_shown_evidence": false, "source_row": 3, "expected_product_at_stated_rank": "JUMBO BAG RED RETROSPOT", "expected_amount_at_stated_rank": "8,630.45" } ], "information_timing_note": "A list marker is generated before the following product and amount. Its low token uncertainty does not establish that the model was confident about the completed business relationship. Raw teacher-forced token scores and completed-answer evidence checks have different information budgets.", "coverage_note": "Both answers omit the requested stock codes. The six curated relationships are not a whole-answer correctness score." } }, { "question_id": "q_0039", "is_primary": false, "split": "test", "question_type": "country_comparison_month", "difficulty": "medium", "question": "In October 2011, did Germany or France generate more net revenue, and by how much?", "gold_short_answer": "Germany generated more net revenue than France by GBP 5,586.63 in October 2011.", "generated_text": "France generated more net revenue (25,017.64 GBP) than Germany (30,604.27 GBP), with France earning a net revenue of -8,453.41 GBP. The difference in net revenue is 5,586.83 GBP.", "evidence_columns": [ "country", "net_revenue", "gross_positive_revenue", "cancellation_revenue", "invoice_count" ], "prompt_evidence_rows": [ { "country": "France", "net_revenue": 25017.64, "gross_positive_revenue": 33471.05, "cancellation_revenue": -8453.41, "invoice_count": 43 }, { "country": "Germany", "net_revenue": 30604.27, "gross_positive_revenue": 31628.42, "cancellation_revenue": -1024.15, "invoice_count": 73 }, { "country": "Netherlands", "net_revenue": 40708.65, "gross_positive_revenue": 40708.65, "cancellation_revenue": 0.0, "invoice_count": 15 }, { "country": "United Kingdom", "net_revenue": 876246.45, "gross_positive_revenue": 935404.14, "cancellation_revenue": -59157.69, "invoice_count": 2126 } ], "takeaway": "A historically correct-labeled span was flagged. This example does not establish the cause of the false alarm.", "spans": [ { "annotation_id": "ann_full100_draft_q_0039_006", "confirmation_item_id": "confirm_001", "question_id": "q_0039", "span_text": "5,586.83 GBP", "span_start_char": 165, "span_end_char": 177, "fact_type": "currency_amount", "label": "correct_key_fact", "publish_use": "caveat_demo", "presentation_use": "span_level_caveat", "demo_priority": "medium", "review_note": "The selected delta amount is within tolerance, but the generated sentence also says France generated more despite listing Germany as higher. Use only as a span-level false-positive example, not as a whole-answer correctness example.", "detector_role_text": "energy false_positive via mean_spilled_probability_mass_after_top2", "simple_score": 0.550849, "entropy_score": 0.291525, "energy_score": 0.021932, "simple_outcome": "false alarm", "entropy_outcome": "false alarm", "energy_outcome": "false alarm", "gold_short_answer": "Germany generated more net revenue than France by GBP 5,586.63 in October 2011." } ], "span_count": 1, "presentation_walkthrough": null }, { "question_id": "q_0054", "is_primary": false, "split": "test", "question_type": "monthly_revenue_change", "difficulty": "hard", "question": "How did net revenue change from May 2011 to June 2011? Provide the absolute change and percentage change.", "gold_short_answer": "Net revenue decreased by GBP 32,116.87 (-4.45%) from May 2011 to June 2011.", "generated_text": "From May 2011 to June 2011, net revenue decreased by **GBP 3,121.37**, representing a **-4.14%** change. The absolute change is **-GBP 3,121.37**, and the percentage change is **-4.14%**.", "evidence_columns": [ "net_revenue", "gross_positive_revenue", "cancellation_revenue", "invoice_count", "year_month" ], "prompt_evidence_rows": [ { "year_month": "2011-05", "net_revenue": 722094.1, "gross_positive_revenue": 769296.61, "cancellation_revenue": -47202.51, "invoice_count": 1995 }, { "year_month": "2011-06", "net_revenue": 689977.23, "gross_positive_revenue": 760547.01, "cancellation_revenue": -70569.78, "invoice_count": 1862 } ], "takeaway": "A historically correct-labeled span was flagged. This example does not establish the cause of the false alarm.", "spans": [ { "annotation_id": "ann_full100_draft_q_0054_003", "confirmation_item_id": "confirm_002", "question_id": "q_0054", "span_text": "decreased", "span_start_char": 40, "span_end_char": 49, "fact_type": "comparison_direction", "label": "correct_key_fact", "publish_use": "caveat_demo", "presentation_use": "span_level_caveat", "demo_priority": "medium", "review_note": "The direction word is correct, while the amount in the same answer is off by a factor of about ten. This is safe only when framed as span-level labeling.", "detector_role_text": "energy false_positive via mean_spilled_probability_mass_after_top2", "simple_score": 0.06377, "entropy_score": 0.269832, "energy_score": 0.028835, "simple_outcome": "cleared", "entropy_outcome": "false alarm", "energy_outcome": "false alarm", "gold_short_answer": "Net revenue decreased by GBP 32,116.87 (-4.45%) from May 2011 to June 2011." }, { "annotation_id": "ann_full100_draft_q_0054_005", "confirmation_item_id": "confirm_003", "question_id": "q_0054", "span_text": "-4.14%", "span_start_char": 88, "span_end_char": 94, "fact_type": "percentage", "label": "correct_key_fact", "publish_use": "caveat_demo", "presentation_use": "span_level_caveat", "demo_priority": "medium", "review_note": "The percentage is within the configured tolerance, but it appears in an answer with a wrong absolute amount. If shown, mention the tolerance and avoid implying the full answer is correct.", "detector_role_text": "energy false_positive via mean_spilled_probability_mass_after_top2; simple false_positive via one_minus_min_top2_margin", "simple_score": 0.969707, "entropy_score": 0.997715, "energy_score": 0.242012, "simple_outcome": "false alarm", "entropy_outcome": "false alarm", "energy_outcome": "false alarm", "gold_short_answer": "Net revenue decreased by GBP 32,116.87 (-4.45%) from May 2011 to June 2011." } ], "span_count": 2, "presentation_walkthrough": null }, { "question_id": "q_0059", "is_primary": false, "split": "test", "question_type": "monthly_revenue_change", "difficulty": "hard", "question": "How did net revenue change from October 2011 to November 2011? Provide the absolute change and percentage change.", "gold_short_answer": "Net revenue increased by GBP 386,777.57 (36.17%) from October 2011 to November 2011.", "generated_text": "The net revenue for November 2011 was 145,614.50 GBP, which represents a **29.75%** increase from October 2011's 106,936.83 GBP. The absolute change is **38,677.67 GBP**, and the percentage change is calculated by dividing the absolute change by the previous month's net revenue.", "evidence_columns": [ "net_revenue", "gross_positive_revenue", "cancellation_revenue", "invoice_count", "year_month" ], "prompt_evidence_rows": [ { "year_month": "2011-10", "net_revenue": 1069368.23, "gross_positive_revenue": 1151263.73, "cancellation_revenue": -81895.5, "invoice_count": 2402 }, { "year_month": "2011-11", "net_revenue": 1456145.8, "gross_positive_revenue": 1503866.78, "cancellation_revenue": -47720.98, "invoice_count": 3210 } ], "takeaway": "Historical selected span for inspecting provisional labels and detector readouts.", "spans": [ { "annotation_id": "ann_full100_draft_q_0059_004", "confirmation_item_id": "confirm_004", "question_id": "q_0059", "span_text": "increase", "span_start_char": 84, "span_end_char": 92, "fact_type": "comparison_direction", "label": "correct_key_fact", "publish_use": "caveat_demo", "presentation_use": "span_level_caveat", "demo_priority": "medium", "review_note": "The increase direction is correct, but the generated revenue and change figures are scaled down. Use as a span-level false-positive example only.", "detector_role_text": "energy false_positive via mean_spilled_probability_mass_after_top2", "simple_score": 0.209808, "entropy_score": 0.469527, "energy_score": 0.023611, "simple_outcome": "cleared", "entropy_outcome": "false alarm", "energy_outcome": "false alarm", "gold_short_answer": "Net revenue increased by GBP 386,777.57 (36.17%) from October 2011 to November 2011." } ], "span_count": 1, "presentation_walkthrough": null }, { "question_id": "q_0063", "is_primary": false, "split": "dev", "question_type": "top3_products_month", "difficulty": "medium", "question": "What were the top 3 products by merchandise net revenue in March 2011? Give the rank, stock code, product name, and net revenue.", "gold_short_answer": "Top 3 products in March 2011: 1. 22423 (REGENCY CAKESTAND 3 TIER), GBP 16,817.85; 2. 85099B (JUMBO BAG RED RETROSPOT), GBP 10,116.43; 3. 47566 (PARTY BUNTING), GBP 9,098.27.", "generated_text": "The top 3 products by merchandise net revenue in March 2011 were: \n1. **JUMBO BAG STRAWBERRY** (85099F) with a net revenue of GBP 5853.80. \n2. **JUMBO BAG PINK POLKADOT** (22386) with a net revenue of GBP 5079.29. \n3. **VINTAGE UNION JACK BUNTING** (21621) with a net revenue of GBP 5214.33. \n\nThe rankings are based on the provided net_revenue_gbp values.", "evidence_columns": [ "stock_code", "description", "net_revenue", "gross_positive_revenue", "cancellation_revenue" ], "prompt_evidence_rows": [ { "stock_code": "85123A", "description": "WHITE HANGING HEART T-LIGHT HOLDER", "net_revenue": 5550.95, "gross_positive_revenue": 5597.9, "cancellation_revenue": -46.95 }, { "stock_code": "22423", "description": "REGENCY CAKESTAND 3 TIER", "net_revenue": 16817.85, "gross_positive_revenue": 18359.25, "cancellation_revenue": -1541.4 }, { "stock_code": "47566", "description": "PARTY BUNTING", "net_revenue": 9098.27, "gross_positive_revenue": 9123.02, "cancellation_revenue": -24.75 }, { "stock_code": "85099F", "description": "JUMBO BAG STRAWBERRY", "net_revenue": 5853.8, "gross_positive_revenue": 5853.8, "cancellation_revenue": 0.0 }, { "stock_code": "22386", "description": "JUMBO BAG PINK POLKADOT", "net_revenue": 5079.29, "gross_positive_revenue": 5079.29, "cancellation_revenue": 0.0 }, { "stock_code": "22084", "description": "PAPER CHAIN KIT EMPIRE", "net_revenue": 4743.0, "gross_positive_revenue": 4743.0, "cancellation_revenue": 0.0 }, { "stock_code": "21621", "description": "VINTAGE UNION JACK BUNTING", "net_revenue": 5214.33, "gross_positive_revenue": 5299.33, "cancellation_revenue": -85.0 }, { "stock_code": "85099B", "description": "JUMBO BAG RED RETROSPOT", "net_revenue": 10116.43, "gross_positive_revenue": 10118.38, "cancellation_revenue": -1.95 } ], "takeaway": "A historically incorrect-labeled span was not flagged. Its score is not a whole-relationship confidence estimate.", "spans": [ { "annotation_id": "ann_full100_draft_q_0063_010", "confirmation_item_id": "confirm_005", "question_id": "q_0063", "span_text": "3.", "span_start_char": 218, "span_end_char": 220, "fact_type": "ranking", "label": "hallucinated_key_fact", "publish_use": "qa_regression_only", "presentation_use": "offset_check_only", "demo_priority": "low", "review_note": "The span is the rank-3 list marker and is tied to the wrong generated product. This is mainly useful as an offset-regression check, not as a detector error example.", "detector_role_text": "No detector-error row; offset check only.", "simple_score": 4e-06, "entropy_score": 3.1e-05, "energy_score": 1e-06, "simple_outcome": "missed", "entropy_outcome": "missed", "energy_outcome": "missed", "gold_short_answer": "Top 3 products in March 2011: 1. 22423 (REGENCY CAKESTAND 3 TIER), GBP 16,817.85; 2. 85099B (JUMBO BAG RED RETROSPOT), GBP 10,116.43; 3. 47566 (PARTY BUNTING), GBP 9,098.27." } ], "span_count": 1, "presentation_walkthrough": null }, { "question_id": "q_0068", "is_primary": false, "split": "dev", "question_type": "top3_products_month", "difficulty": "medium", "question": "What were the top 3 products by merchandise net revenue in August 2011? Give the rank, stock code, product name, and net revenue.", "gold_short_answer": "Top 3 products in August 2011: 1. 85099B (JUMBO BAG RED RETROSPOT), GBP 10,441.77; 2. 84879 (ASSORTED COLOUR BIRD ORNAMENT), GBP 9,867.04; 3. 47566 (PARTY BUNTING), GBP 9,655.25.", "generated_text": "In August 2011, the top 3 merchandise net revenue products were: \n1. **JUMBO BAG PINK POLKADOT** (rank 1), with a net revenue of GBP 4,728.73. \n2. **JUMBO BAG RED RETROSPOT** (rank 2), with a net revenue of GBP 10,441.77. \n3. **SPOTTY BUNTING** (rank 3), with a net revenue of GBP 7,864.43.", "evidence_columns": [ "stock_code", "description", "net_revenue", "gross_positive_revenue", "cancellation_revenue" ], "prompt_evidence_rows": [ { "stock_code": "85123A", "description": "WHITE HANGING HEART T-LIGHT HOLDER", "net_revenue": 5589.72, "gross_positive_revenue": 5620.57, "cancellation_revenue": -30.85 }, { "stock_code": "22386", "description": "JUMBO BAG PINK POLKADOT", "net_revenue": 4728.73, "gross_positive_revenue": 4728.73, "cancellation_revenue": 0.0 }, { "stock_code": "47566", "description": "PARTY BUNTING", "net_revenue": 9655.25, "gross_positive_revenue": 9694.85, "cancellation_revenue": -39.6 }, { "stock_code": "85099B", "description": "JUMBO BAG RED RETROSPOT", "net_revenue": 10441.77, "gross_positive_revenue": 10622.56, "cancellation_revenue": -180.79 }, { "stock_code": "23199", "description": "JUMBO BAG APPLES", "net_revenue": 6204.89, "gross_positive_revenue": 6233.72, "cancellation_revenue": -28.83 }, { "stock_code": "84879", "description": "ASSORTED COLOUR BIRD ORNAMENT", "net_revenue": 9867.04, "gross_positive_revenue": 9868.73, "cancellation_revenue": -1.69 }, { "stock_code": "23298", "description": "SPOTTY BUNTING", "net_revenue": 7864.43, "gross_positive_revenue": 7869.38, "cancellation_revenue": -4.95 }, { "stock_code": "22423", "description": "REGENCY CAKESTAND 3 TIER", "net_revenue": 8393.55, "gross_positive_revenue": 10521.3, "cancellation_revenue": -2127.75 } ], "takeaway": "A historically incorrect-labeled span was not flagged. Its score is not a whole-relationship confidence estimate.", "spans": [ { "annotation_id": "ann_full100_draft_q_0068_008", "confirmation_item_id": "confirm_010", "question_id": "q_0068", "span_text": "3.", "span_start_char": 226, "span_end_char": 228, "fact_type": "ranking", "label": "hallucinated_key_fact", "publish_use": "qa_regression_only", "presentation_use": "offset_check_only", "demo_priority": "low", "review_note": "The span is the rank-3 list marker, not a decimal inside the nearby amount. Keep this primarily as an offset-regression check.", "detector_role_text": "No detector-error row; offset check only.", "simple_score": 5.00000000003e-06, "entropy_score": 3.7e-05, "energy_score": 1e-06, "simple_outcome": "missed", "entropy_outcome": "missed", "energy_outcome": "missed", "gold_short_answer": "Top 3 products in August 2011: 1. 85099B (JUMBO BAG RED RETROSPOT), GBP 10,441.77; 2. 84879 (ASSORTED COLOUR BIRD ORNAMENT), GBP 9,867.04; 3. 47566 (PARTY BUNTING), GBP 9,655.25." } ], "span_count": 1, "presentation_walkthrough": null }, { "question_id": "q_0093", "is_primary": false, "split": "test", "question_type": "return_impact_month", "difficulty": "hard", "question": "In April 2011, how much did cancellations and returns reduce gross positive revenue, and what was the final net revenue?", "gold_short_answer": "Cancellations and returns reduced gross positive revenue by GBP 44,600.65 (8.31% of gross), leaving net revenue of GBP 492,367.84.", "generated_text": "In April 2011, cancellations and returns reduced gross positive revenue by \u00a344,600.65, resulting in a final net revenue of \u00a344,600.65.", "evidence_columns": [ "net_revenue", "gross_positive_revenue", "cancellation_revenue", "invoice_count", "year_month" ], "prompt_evidence_rows": [ { "year_month": "2011-04", "net_revenue": 492367.84, "gross_positive_revenue": 536968.49, "cancellation_revenue": -44600.65, "invoice_count": 1486 } ], "takeaway": "A historically correct-labeled span was flagged. This example does not establish the cause of the false alarm.", "spans": [ { "annotation_id": "ann_full100_draft_q_0093_002", "confirmation_item_id": "confirm_014", "question_id": "q_0093", "span_text": "\u00a344,600.65", "span_start_char": 75, "span_end_char": 85, "fact_type": "currency_amount", "label": "correct_key_fact", "publish_use": "caveat_demo", "presentation_use": "span_level_caveat", "demo_priority": "medium", "review_note": "The reduction amount is correct, but the generated final net revenue repeats that same value and is wrong. Use only to show a correct span that a detector flags, with a clear span-level caveat.", "detector_role_text": "energy false_positive via mean_spilled_probability_mass_after_top2; simple false_positive via one_minus_min_top2_margin", "simple_score": 0.910483, "entropy_score": 0.350166, "energy_score": 0.075874, "simple_outcome": "false alarm", "entropy_outcome": "false alarm", "energy_outcome": "false alarm", "gold_short_answer": "Cancellations and returns reduced gross positive revenue by GBP 44,600.65 (8.31% of gross), leaving net revenue of GBP 492,367.84." } ], "span_count": 1, "presentation_walkthrough": null }, { "question_id": "q_0098", "is_primary": false, "split": "test", "question_type": "return_impact_month", "difficulty": "hard", "question": "In September 2011, how much did cancellations and returns reduce gross positive revenue, and what was the final net revenue?", "gold_short_answer": "Cancellations and returns reduced gross positive revenue by GBP 38,838.51 (3.68% of gross), leaving net revenue of GBP 1,017,596.68.", "generated_text": "In September 2011, cancellations and returns reduced gross positive revenue by GBP 38,838.51, resulting in a net revenue of GBP 101,759.68.", "evidence_columns": [ "net_revenue", "gross_positive_revenue", "cancellation_revenue", "invoice_count", "year_month" ], "prompt_evidence_rows": [ { "year_month": "2011-09", "net_revenue": 1017596.68, "gross_positive_revenue": 1056435.19, "cancellation_revenue": -38838.51, "invoice_count": 2170 } ], "takeaway": "A historically correct-labeled span was flagged. This example does not establish the cause of the false alarm.", "spans": [ { "annotation_id": "ann_full100_draft_q_0098_002", "confirmation_item_id": "confirm_015", "question_id": "q_0098", "span_text": "GBP 38,838.51", "span_start_char": 79, "span_end_char": 92, "fact_type": "currency_amount", "label": "correct_key_fact", "publish_use": "caveat_demo", "presentation_use": "span_level_caveat", "demo_priority": "medium", "review_note": "The reduction amount is correct, but the final net revenue is off by a factor of ten. Use only as a span-level false-positive example.", "detector_role_text": "simple false_positive via one_minus_min_top2_margin", "simple_score": 0.700917, "entropy_score": 0.129517, "energy_score": 0.018201, "simple_outcome": "false alarm", "entropy_outcome": "false alarm", "energy_outcome": "false alarm", "gold_short_answer": "Cancellations and returns reduced gross positive revenue by GBP 38,838.51 (3.68% of gross), leaving net revenue of GBP 1,017,596.68." } ], "span_count": 1, "presentation_walkthrough": null } ], "statistical_review": { "source": "reports/bizhallu_statistics_v2_report.json", "source_sha256": "096bae57edec06e74c6f4e029419b844adb421d252a1f8c60335ae1859dc0727", "metric_version": "binary_metrics_v2_tied_ap", "scope": { "dev": { "questions": 17, "spans": 102, "positive_spans": 61, "missing_question_ids": [ "q_0048" ] }, "test": { "questions": 18, "spans": 103, "positive_spans": 61, "missing_question_ids": [] } }, "test_rows": [ { "arm": "saved_trace_precision", "signal": "one_minus_min_top2_margin", "split": "test", "threshold": 0.33433282375335693, "tp": 41, "fp": 7, "tn": 35, "fn": 20, "precision": 0.8541666666666666, "recall": 0.6721311475409836, "specificity": 0.8333333333333334, "f1": 0.7522935779816514, "accuracy": 0.7378640776699029, "balanced_accuracy": 0.7527322404371585, "mcc": 0.4979414071698933, "average_precision": 0.8350726705510306, "auroc": 0.7568306010928961, "positive_prevalence": 0.5922330097087378, "span_count": 103 }, { "arm": "saved_trace_precision", "signal": "mean_token_entropy", "split": "test", "threshold": 0.004817123346063974, "tp": 53, "fp": 22, "tn": 20, "fn": 8, "precision": 0.7066666666666667, "recall": 0.8688524590163934, "specificity": 0.47619047619047616, "f1": 0.7794117647058824, "accuracy": 0.7087378640776699, "balanced_accuracy": 0.6725214676034348, "mcc": 0.3811123785789828, "average_precision": 0.8094737529413718, "auroc": 0.7361436377829821, "positive_prevalence": 0.5922330097087378, "span_count": 103 }, { "arm": "saved_trace_precision", "signal": "all_positive", "split": "test", "threshold": 0.5, "tp": 61, "fp": 42, "tn": 0, "fn": 0, "precision": 0.5922330097087378, "recall": 1.0, "specificity": 0.0, "f1": 0.7439024390243902, "accuracy": 0.5922330097087378, "balanced_accuracy": 0.5, "mcc": 0.0, "average_precision": 0.5922330097087378, "auroc": 0.5, "positive_prevalence": 0.5922330097087378, "span_count": 103 }, { "arm": "saved_trace_precision", "signal": "all_negative", "split": "test", "threshold": 0.5, "tp": 0, "fp": 0, "tn": 42, "fn": 61, "precision": 0.0, "recall": 0.0, "specificity": 1.0, "f1": 0.0, "accuracy": 0.4077669902912621, "balanced_accuracy": 0.5, "mcc": 0.0, "average_precision": 0.5922330097087378, "auroc": 0.5, "positive_prevalence": 0.5922330097087378, "span_count": 103 }, { "arm": "saved_trace_precision", "signal": "dev_fact_type_prior", "split": "test", "threshold": 0.575, "tp": 61, "fp": 24, "tn": 18, "fn": 0, "precision": 0.7176470588235294, "recall": 1.0, "specificity": 0.42857142857142855, "f1": 0.8356164383561644, "accuracy": 0.7669902912621359, "balanced_accuracy": 0.7142857142857143, "mcc": 0.5545836503270576, "average_precision": 0.7598384536560515, "auroc": 0.7681498829039812, "positive_prevalence": 0.5922330097087378, "span_count": 103 } ], "entropy_minus_all_positive": { "signal": "mean_token_entropy", "reference": "all_positive", "metric": "f1", "point_difference": 0.03550932568149212, "lower_95": -0.035603741496598666, "upper_95": 0.10473150125972673, "valid_replicates": 5000, "undefined_replicates": 0 }, "independent_human_annotation": false, "confirmatory": false } }