{
  "workflow": "Support email triage (Gmail + OpenAI + Pinecone)",
  "ranAt": "2026-10-25T18:07:56.403Z",
  "kb": [
    {
      "id": "kb-01",
      "source": "Incident playbook §1",
      "text": "If a client website is down, we check hosting status within 15 minutes during business hours. Clients can see live status at status.brightline.example. Critical outages are worked on until resolved and the client gets an update every 60 minutes."
    },
    {
      "id": "kb-02",
      "source": "Billing FAQ §2",
      "text": "Invoice copies can be downloaded in the client portal under Billing > Invoices. On request we email a PDF copy of any invoice from the last 24 months. Payment terms are 14 days net."
    },
    {
      "id": "kb-03",
      "source": "Change requests §1",
      "text": "Small change requests (under 2 hours) are scheduled within 3 business days. Larger changes get a written estimate first. Submit changes through the portal form or by email with the page URL and a screenshot."
    },
    {
      "id": "kb-04",
      "source": "Access & passwords §1",
      "text": "For CMS password resets, use the Forgot password link on the login page. If the reset email does not arrive within 10 minutes, check spam, then ask support to resend the invitation. We never send passwords by email."
    },
    {
      "id": "kb-05",
      "source": "Hosting & SSL §3",
      "text": "SSL certificates renew automatically 30 days before expiry. If the browser shows a certificate warning, clear the cache first; if it persists, support reissues the certificate the same business day."
    },
    {
      "id": "kb-06",
      "source": "Support hours",
      "text": "Support hours are Monday to Friday, 8:00 to 18:00 Central time. Emails outside these hours are answered the next business day; critical outages are monitored around the clock."
    }
  ],
  "emails": [
    {
      "id": "m01",
      "threadId": "t01",
      "from": "dana@harborbakery.example",
      "subject": "Our website is down!!",
      "text": "Hi, since this morning our site harborbakery.example shows an error page. Customers cannot order. Is the website down on your side?",
      "expect": "auto_reply",
      "scenario": "Support email, strong KB match"
    },
    {
      "id": "m02",
      "threadId": "t02",
      "from": "accounts@peakdental.example",
      "subject": "Copy of invoice 2026-0412",
      "text": "Hello, our accountant needs a copy of invoice 2026-0412 as PDF. Where can I download invoices?",
      "expect": "auto_reply",
      "scenario": "Billing question, strong KB match"
    },
    {
      "id": "m03",
      "threadId": "t03",
      "from": "news@growthweekly.example",
      "subject": "5 growth hacks for agencies this week",
      "text": "Unsubscribe anytime. This week: five growth hacks, a webinar invitation and our partner offers.",
      "expect": "not_support",
      "scenario": "Newsletter, filtered without reply"
    },
    {
      "id": "m04",
      "threadId": "t04",
      "from": "mark@ridgeoutdoor.example",
      "subject": "Change the banner on the homepage",
      "text": "Can you change the homepage banner text and image? How long does a small change request take?",
      "expect": "auto_reply",
      "scenario": "Change request, KB match"
    },
    {
      "id": "m05",
      "threadId": "t01",
      "from": "dana@harborbakery.example",
      "subject": "Re: Our website is down!!",
      "text": "Thanks for the quick answer. The site is still down, do we get an update on the outage status?",
      "expect": "auto_reply",
      "scenario": "Follow-up in existing thread (thread continuity)"
    },
    {
      "id": "m06",
      "threadId": "t06",
      "from": "li@oakfurniture.example",
      "subject": "Custom ERP sync for our product feed",
      "text": "We want our ERP product feed to sync with the shop every hour through a custom middleware. Can you build that and what would it cost?",
      "expect": "escalated",
      "scenario": "Support question the KB cannot answer"
    },
    {
      "id": "m07",
      "threadId": "t07",
      "from": "sam@peakdental.example",
      "subject": "Automatic reply: Out of office",
      "text": "I am out of office until Monday with limited access to email. For urgent matters contact the front desk.",
      "expect": "not_support",
      "scenario": "Auto-reply, filtered without reply"
    },
    {
      "id": "m08",
      "threadId": "t08",
      "from": "nina@citylaw.example",
      "subject": "Password reset email not arriving",
      "text": "I clicked forgot password on the CMS login but the reset email never arrived. Can you help me with the password reset?",
      "expect": "escalated",
      "scenario": "Credential failure on the reply step (OpenAI 401), retried, then escalated",
      "failNode": "AI draft reply"
    }
  ],
  "results": [
    {
      "id": "m01",
      "scenario": "Support email, strong KB match",
      "from": "dana@harborbakery.example",
      "subject": "Our website is down!!",
      "threadId": "t01",
      "category": "outage",
      "route": "auto_reply",
      "expect": "auto_reply",
      "pass": true,
      "topScore": 0.95,
      "sources": [
        "Incident playbook §1"
      ],
      "note": "Replied in original thread",
      "reply": "Hi Dana,\n\nthanks for reaching out. If a client website is down, we check hosting status within 15 minutes during business hours. Clients can see live status at status.brightline.example. Critical outages are worked on until resolved and the client gets an update every 60 minutes.\n\nIf anything is still unclear, just reply to this email.\n\nThe Support Team",
      "trace": [
        "Gmail Trigger (new unread email)",
        "Normalize email",
        "AI classify (LLM step (client's own API key), mocked)",
        "Parse classification",
        "Is support? (true)",
        "Embed question (OpenAI embeddings (client key), mocked)",
        "Pinecone query (local keyword index in place of Pinecone)",
        "Knowledge gate",
        "Enough knowledge? (true)",
        "AI draft reply (LLM step (client's own API key), mocked)",
        "Reply in thread (Gmail reply to m01, thread t01)",
        "Log reply",
        "Write log row (Google Sheets \"Support log\")"
      ]
    },
    {
      "id": "m02",
      "scenario": "Billing question, strong KB match",
      "from": "accounts@peakdental.example",
      "subject": "Copy of invoice 2026-0412",
      "threadId": "t02",
      "category": "outage",
      "route": "auto_reply",
      "expect": "auto_reply",
      "pass": true,
      "topScore": 0.95,
      "sources": [
        "Incident playbook §1"
      ],
      "note": "Replied in original thread",
      "reply": "Hi Accounts,\n\nthanks for reaching out. If a client website is down, we check hosting status within 15 minutes during business hours. Clients can see live status at status.brightline.example. Critical outages are worked on until resolved and the client gets an update every 60 minutes.\n\nIf anything is still unclear, just reply to this email.\n\nThe Support Team",
      "trace": [
        "Gmail Trigger (new unread email)",
        "Normalize email",
        "AI classify (LLM step (client's own API key), mocked)",
        "Parse classification",
        "Is support? (true)",
        "Embed question (OpenAI embeddings (client key), mocked)",
        "Pinecone query (local keyword index in place of Pinecone)",
        "Knowledge gate",
        "Enough knowledge? (true)",
        "AI draft reply (LLM step (client's own API key), mocked)",
        "Reply in thread (Gmail reply to m02, thread t02)",
        "Log reply",
        "Write log row (Google Sheets \"Support log\")"
      ]
    },
    {
      "id": "m03",
      "scenario": "Newsletter, filtered without reply",
      "from": "news@growthweekly.example",
      "subject": "5 growth hacks for agencies this week",
      "threadId": "t03",
      "category": "newsletter",
      "route": "not_support",
      "expect": "not_support",
      "pass": true,
      "topScore": "",
      "sources": [],
      "note": "Labelled and left in inbox, no reply generated",
      "reply": "",
      "trace": [
        "Gmail Trigger (new unread email)",
        "Normalize email",
        "AI classify (LLM step (client's own API key), mocked)",
        "Parse classification",
        "Is support? (false)",
        "Label not-support (Gmail label \"Not support\")",
        "Log skip",
        "Write log row (Google Sheets \"Support log\")"
      ]
    },
    {
      "id": "m04",
      "scenario": "Change request, KB match",
      "from": "mark@ridgeoutdoor.example",
      "subject": "Change the banner on the homepage",
      "threadId": "t04",
      "category": "change_request",
      "route": "auto_reply",
      "expect": "auto_reply",
      "pass": true,
      "topScore": 0.905,
      "sources": [
        "Change requests §1"
      ],
      "note": "Replied in original thread",
      "reply": "Hi Mark,\n\nthanks for reaching out. Small change requests (under 2 hours) are scheduled within 3 business days. Larger changes get a written estimate first. Submit changes through the portal form or by email with the page URL and a screenshot.\n\nIf anything is still unclear, just reply to this email.\n\nThe Support Team",
      "trace": [
        "Gmail Trigger (new unread email)",
        "Normalize email",
        "AI classify (LLM step (client's own API key), mocked)",
        "Parse classification",
        "Is support? (true)",
        "Embed question (OpenAI embeddings (client key), mocked)",
        "Pinecone query (local keyword index in place of Pinecone)",
        "Knowledge gate",
        "Enough knowledge? (true)",
        "AI draft reply (LLM step (client's own API key), mocked)",
        "Reply in thread (Gmail reply to m04, thread t04)",
        "Log reply",
        "Write log row (Google Sheets \"Support log\")"
      ]
    },
    {
      "id": "m05",
      "scenario": "Follow-up in existing thread (thread continuity)",
      "from": "dana@harborbakery.example",
      "subject": "Re: Our website is down!!",
      "threadId": "t01",
      "category": "outage",
      "route": "auto_reply",
      "expect": "auto_reply",
      "pass": true,
      "topScore": 0.95,
      "sources": [
        "Incident playbook §1"
      ],
      "note": "Replied in original thread",
      "reply": "Hi Dana,\n\nthanks for reaching out. If a client website is down, we check hosting status within 15 minutes during business hours. Clients can see live status at status.brightline.example. Critical outages are worked on until resolved and the client gets an update every 60 minutes.\n\nIf anything is still unclear, just reply to this email.\n\nThe Support Team",
      "trace": [
        "Gmail Trigger (new unread email)",
        "Normalize email",
        "AI classify (LLM step (client's own API key), mocked)",
        "Parse classification",
        "Is support? (true)",
        "Embed question (OpenAI embeddings (client key), mocked)",
        "Pinecone query (local keyword index in place of Pinecone)",
        "Knowledge gate",
        "Enough knowledge? (true)",
        "AI draft reply (LLM step (client's own API key), mocked)",
        "Reply in thread (Gmail reply to m05, thread t01)",
        "Log reply",
        "Write log row (Google Sheets \"Support log\")"
      ]
    },
    {
      "id": "m06",
      "scenario": "Support question the KB cannot answer",
      "from": "li@oakfurniture.example",
      "subject": "Custom ERP sync for our product feed",
      "threadId": "t06",
      "category": "other_support",
      "route": "escalated",
      "expect": "escalated",
      "pass": true,
      "topScore": 0.5,
      "sources": [],
      "note": "Knowledge base not sufficient, forwarded to support lead",
      "reply": "",
      "trace": [
        "Gmail Trigger (new unread email)",
        "Normalize email",
        "AI classify (LLM step (client's own API key), mocked)",
        "Parse classification",
        "Is support? (true)",
        "Embed question (OpenAI embeddings (client key), mocked)",
        "Pinecone query (local keyword index in place of Pinecone)",
        "Knowledge gate",
        "Enough knowledge? (false)",
        "Prepare escalation",
        "Escalate to human (Gmail to support lead)",
        "Log escalation",
        "Write log row (Google Sheets \"Support log\")"
      ]
    },
    {
      "id": "m07",
      "scenario": "Auto-reply, filtered without reply",
      "from": "sam@peakdental.example",
      "subject": "Automatic reply: Out of office",
      "threadId": "t07",
      "category": "auto_reply",
      "route": "not_support",
      "expect": "not_support",
      "pass": true,
      "topScore": "",
      "sources": [],
      "note": "Labelled and left in inbox, no reply generated",
      "reply": "",
      "trace": [
        "Gmail Trigger (new unread email)",
        "Normalize email",
        "AI classify (LLM step (client's own API key), mocked)",
        "Parse classification",
        "Is support? (false)",
        "Label not-support (Gmail label \"Not support\")",
        "Log skip",
        "Write log row (Google Sheets \"Support log\")"
      ]
    },
    {
      "id": "m08",
      "scenario": "Credential failure on the reply step (OpenAI 401), retried, then escalated",
      "from": "nina@citylaw.example",
      "subject": "Password reset email not arriving",
      "threadId": "t08",
      "category": "access",
      "route": "escalated",
      "expect": "escalated",
      "pass": true,
      "topScore": 0.95,
      "sources": [
        "Access & passwords §1"
      ],
      "note": "Integration error: 401 Incorrect API key provided (simulated credential failure)",
      "reply": "",
      "trace": [
        "Gmail Trigger (new unread email)",
        "Normalize email",
        "AI classify (LLM step (client's own API key), mocked)",
        "Parse classification",
        "Is support? (true)",
        "Embed question (OpenAI embeddings (client key), mocked)",
        "Pinecone query (local keyword index in place of Pinecone)",
        "Knowledge gate",
        "Enough knowledge? (true)",
        "AI draft reply (401 invalid_api_key · try 1/3)",
        "AI draft reply (401 · try 2/3 (after 2 s))",
        "AI draft reply (401 · try 3/3 → error output)",
        "Prepare escalation",
        "Escalate to human (Gmail to support lead)",
        "Log escalation",
        "Write log row (Google Sheets \"Support log\")"
      ]
    }
  ]
}