{
  "title": "OpenLawn residential vision-model smoke test",
  "date": "2026-09-06",
  "publisher": "OpenLawn",
  "scope": "Internal workflow comparison, not independently audited or a universal model leaderboard. Three residential properties; only one has a user-corrected reference, not surveyed ground truth.",
  "protocol": {
    "initial": "Same saved pinned aerial, parcel, prompt, tools and parser per residence. Medium reasoning, 2400 output tokens per turn, up to 7 turns. No SegFormer. Corrected reference withheld.",
    "compatible": "Gemini: 8192 tokens per turn. Claude Fable: auto tool choice instead of forced tool choice.",
    "low": "Astra: low reasoning.",
    "repeat": "One extra corrected-residence run each for Astra medium and Opus.",
    "limits": "90 seconds per turn; 180 seconds per run; at most 3 concurrent runs. Fixed image frame: reframe requests remain incomplete.",
    "timing": "Direct model-pipeline wall time, not end-to-end app time. Excludes geocoding, parcel lookup, queue, browser rendering and saving. First polygon means tool callback arrival.",
    "limitations": "Prompt was developed for the existing Astra workflow. Equal effort labels are not equal compute across providers. Gateway routing, load and cache warmth were not controlled. Too few references or repeats for general accuracy or reliability claims.",
    "privacy": "Addresses, coordinates, imagery, record identifiers and raw responses are withheld. This export supports metric auditing, not full independent reproduction."
  },
  "sourceHashes": {
    "app/lib/lawn-mapping-step.ts": "9cacdfa7d9b55d2eae3156a6c0837ba0a71f1daca734f382ce3f6a2483af9b6c",
    "app/lib/lawn-mapping-prompt.ts": "16e7db03b42b6764ec8e9286fc01e411f8d55e5f91f8c75a5f815d40c93c03b8",
    "app/lib/lawn-mapping-tools.ts": "fbc1e71147904f7f2ff46090909033150eba4fd9c4ea046eedad8082f83cb150",
    "app/lib/gateway-tool-stream.ts": "e3e5529ab0d53c8a6ba2bf31cfc28447b5d0c9f8b57fbb6057d31adc87283c61",
    "app/lib/lawn-job-result.ts": "19c2d0d91249b5f41768c864adfc393d824b15c3bcac6ad9a5f366b8cce45ae9"
  },
  "attempts": 41,
  "completed": 25,
  "models": [
    "anthropic/claude-sonnet-5",
    "openai/gpt-5.6-terra",
    "anthropic/claude-fable-5.1",
    "openai/gpt-6-astra",
    "google/gemini-3.8-flash",
    "anthropic/claude-opus-5",
    "google/gemini-3.1-pro-preview",
    "bytedance/seed-1.8",
    "spacexai/grok-4.6"
  ],
  "profiles": [
    {
      "model": "anthropic/claude-sonnet-5",
      "phase": "screen",
      "attempted": 3,
      "completed": 3,
      "meanSeconds": 16.112333333333332,
      "meanFirstPolygonSeconds": 3.4229999999999996,
      "meanOverlapPercent": 31.027891913083554,
      "referenceTrials": 1
    },
    {
      "model": "openai/gpt-5.6-terra",
      "phase": "screen",
      "attempted": 3,
      "completed": 3,
      "meanSeconds": 30.497333333333334,
      "meanFirstPolygonSeconds": 11.266666666666666,
      "meanOverlapPercent": 61.73219886537454,
      "referenceTrials": 1
    },
    {
      "model": "anthropic/claude-fable-5.1",
      "phase": "screen",
      "attempted": 3,
      "completed": 0,
      "meanSeconds": null,
      "meanFirstPolygonSeconds": null,
      "meanOverlapPercent": null,
      "referenceTrials": 0
    },
    {
      "model": "openai/gpt-6-astra",
      "phase": "screen",
      "attempted": 3,
      "completed": 3,
      "meanSeconds": 46.37466666666666,
      "meanFirstPolygonSeconds": 12.898666666666665,
      "meanOverlapPercent": 90.68263400910789,
      "referenceTrials": 1
    },
    {
      "model": "google/gemini-3.8-flash",
      "phase": "screen",
      "attempted": 3,
      "completed": 0,
      "meanSeconds": null,
      "meanFirstPolygonSeconds": null,
      "meanOverlapPercent": null,
      "referenceTrials": 0
    },
    {
      "model": "anthropic/claude-opus-5",
      "phase": "screen",
      "attempted": 3,
      "completed": 3,
      "meanSeconds": 28.679333333333332,
      "meanFirstPolygonSeconds": 5.171,
      "meanOverlapPercent": 76.24232962355713,
      "referenceTrials": 1
    },
    {
      "model": "google/gemini-3.1-pro-preview",
      "phase": "screen",
      "attempted": 3,
      "completed": 0,
      "meanSeconds": null,
      "meanFirstPolygonSeconds": null,
      "meanOverlapPercent": null,
      "referenceTrials": 0
    },
    {
      "model": "bytedance/seed-1.8",
      "phase": "screen",
      "attempted": 3,
      "completed": 3,
      "meanSeconds": 34.418,
      "meanFirstPolygonSeconds": 9.615,
      "meanOverlapPercent": 73.40545116824259,
      "referenceTrials": 1
    },
    {
      "model": "spacexai/grok-4.6",
      "phase": "screen",
      "attempted": 3,
      "completed": 0,
      "meanSeconds": null,
      "meanFirstPolygonSeconds": null,
      "meanOverlapPercent": null,
      "referenceTrials": 0
    },
    {
      "model": "anthropic/claude-fable-5.1",
      "phase": "compatible",
      "attempted": 3,
      "completed": 1,
      "meanSeconds": 33.123,
      "meanFirstPolygonSeconds": 21.403,
      "meanOverlapPercent": null,
      "referenceTrials": 0
    },
    {
      "model": "google/gemini-3.8-flash",
      "phase": "compatible",
      "attempted": 3,
      "completed": 1,
      "meanSeconds": 88.25,
      "meanFirstPolygonSeconds": 45.044,
      "meanOverlapPercent": null,
      "referenceTrials": 0
    },
    {
      "model": "google/gemini-3.1-pro-preview",
      "phase": "compatible",
      "attempted": 3,
      "completed": 3,
      "meanSeconds": 123.94633333333333,
      "meanFirstPolygonSeconds": 54.964666666666666,
      "meanOverlapPercent": 82.73713644751419,
      "referenceTrials": 1
    },
    {
      "model": "openai/gpt-6-astra",
      "phase": "low",
      "attempted": 3,
      "completed": 3,
      "meanSeconds": 25.403333333333332,
      "meanFirstPolygonSeconds": 5.963666666666666,
      "meanOverlapPercent": 65.49155142811937,
      "referenceTrials": 1
    },
    {
      "model": "openai/gpt-6-astra",
      "phase": "repeat",
      "attempted": 1,
      "completed": 1,
      "meanSeconds": 45.497,
      "meanFirstPolygonSeconds": 13.152,
      "meanOverlapPercent": 75.15934120643114,
      "referenceTrials": 1
    },
    {
      "model": "anthropic/claude-opus-5",
      "phase": "repeat",
      "attempted": 1,
      "completed": 1,
      "meanSeconds": 32.285,
      "meanFirstPolygonSeconds": 4.811,
      "meanOverlapPercent": 75.10077065375162,
      "referenceTrials": 1
    }
  ],
  "trials": [
    {
      "phase": "screen",
      "residence": "Residence B",
      "model": "anthropic/claude-sonnet-5",
      "completed": true,
      "seconds": 17.188,
      "firstPolygonSeconds": 3.666,
      "correctedReferenceOverlapPercent": 31.027891913083554,
      "absoluteAreaDifferencePercent": 61.48264984227129
    },
    {
      "phase": "screen",
      "residence": "Residence B",
      "model": "openai/gpt-5.6-terra",
      "completed": true,
      "seconds": 29.213,
      "firstPolygonSeconds": 12.436,
      "correctedReferenceOverlapPercent": 61.73219886537454,
      "absoluteAreaDifferencePercent": 27.57097791798107
    },
    {
      "phase": "screen",
      "residence": "Residence B",
      "model": "anthropic/claude-fable-5.1",
      "completed": false,
      "seconds": 2.12,
      "firstPolygonSeconds": null,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence B",
      "model": "openai/gpt-6-astra",
      "completed": true,
      "seconds": 38.165,
      "firstPolygonSeconds": 10.472,
      "correctedReferenceOverlapPercent": 90.68263400910789,
      "absoluteAreaDifferencePercent": 1.4616193480546793
    },
    {
      "phase": "screen",
      "residence": "Residence B",
      "model": "google/gemini-3.8-flash",
      "completed": false,
      "seconds": 16.006,
      "firstPolygonSeconds": null,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence B",
      "model": "anthropic/claude-opus-5",
      "completed": true,
      "seconds": 30.047,
      "firstPolygonSeconds": 5.213,
      "correctedReferenceOverlapPercent": 76.24232962355713,
      "absoluteAreaDifferencePercent": 0.22082018927444794
    },
    {
      "phase": "screen",
      "residence": "Residence B",
      "model": "google/gemini-3.1-pro-preview",
      "completed": false,
      "seconds": 36.513,
      "firstPolygonSeconds": 18.084,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence B",
      "model": "bytedance/seed-1.8",
      "completed": true,
      "seconds": 33.876,
      "firstPolygonSeconds": 9.104,
      "correctedReferenceOverlapPercent": 73.40545116824259,
      "absoluteAreaDifferencePercent": 1.2513144058885384
    },
    {
      "phase": "screen",
      "residence": "Residence A",
      "model": "anthropic/claude-fable-5.1",
      "completed": false,
      "seconds": 3.295,
      "firstPolygonSeconds": null,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence A",
      "model": "google/gemini-3.8-flash",
      "completed": false,
      "seconds": 16.653,
      "firstPolygonSeconds": null,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence A",
      "model": "anthropic/claude-opus-5",
      "completed": true,
      "seconds": 26.865,
      "firstPolygonSeconds": 5.02,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence A",
      "model": "google/gemini-3.1-pro-preview",
      "completed": false,
      "seconds": 21.661,
      "firstPolygonSeconds": null,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence B",
      "model": "spacexai/grok-4.6",
      "completed": false,
      "seconds": 90.004,
      "firstPolygonSeconds": null,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence A",
      "model": "bytedance/seed-1.8",
      "completed": true,
      "seconds": 36.775,
      "firstPolygonSeconds": 9.701,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence A",
      "model": "openai/gpt-6-astra",
      "completed": true,
      "seconds": 49.226,
      "firstPolygonSeconds": 14.666,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence A",
      "model": "openai/gpt-5.6-terra",
      "completed": true,
      "seconds": 30.764,
      "firstPolygonSeconds": 9.038,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence A",
      "model": "spacexai/grok-4.6",
      "completed": false,
      "seconds": 90.003,
      "firstPolygonSeconds": null,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence A",
      "model": "anthropic/claude-sonnet-5",
      "completed": true,
      "seconds": 16.031,
      "firstPolygonSeconds": 3.518,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence C",
      "model": "google/gemini-3.1-pro-preview",
      "completed": false,
      "seconds": 22.2,
      "firstPolygonSeconds": null,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence C",
      "model": "bytedance/seed-1.8",
      "completed": true,
      "seconds": 32.603,
      "firstPolygonSeconds": 10.04,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence C",
      "model": "openai/gpt-6-astra",
      "completed": true,
      "seconds": 51.733,
      "firstPolygonSeconds": 13.558,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence C",
      "model": "openai/gpt-5.6-terra",
      "completed": true,
      "seconds": 31.515,
      "firstPolygonSeconds": 12.326,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence C",
      "model": "anthropic/claude-sonnet-5",
      "completed": true,
      "seconds": 15.118,
      "firstPolygonSeconds": 3.085,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence C",
      "model": "spacexai/grok-4.6",
      "completed": false,
      "seconds": 90.005,
      "firstPolygonSeconds": null,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence C",
      "model": "anthropic/claude-fable-5.1",
      "completed": false,
      "seconds": 2.585,
      "firstPolygonSeconds": null,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence C",
      "model": "google/gemini-3.8-flash",
      "completed": false,
      "seconds": 14.649,
      "firstPolygonSeconds": null,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "screen",
      "residence": "Residence C",
      "model": "anthropic/claude-opus-5",
      "completed": true,
      "seconds": 29.126,
      "firstPolygonSeconds": 5.28,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "compatible",
      "residence": "Residence B",
      "model": "anthropic/claude-fable-5.1",
      "completed": false,
      "seconds": 28.371,
      "firstPolygonSeconds": 15.603,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "compatible",
      "residence": "Residence B",
      "model": "google/gemini-3.8-flash",
      "completed": false,
      "seconds": 46.374,
      "firstPolygonSeconds": null,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "compatible",
      "residence": "Residence A",
      "model": "anthropic/claude-fable-5.1",
      "completed": true,
      "seconds": 33.123,
      "firstPolygonSeconds": 21.403,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "compatible",
      "residence": "Residence B",
      "model": "google/gemini-3.1-pro-preview",
      "completed": true,
      "seconds": 104.386,
      "firstPolygonSeconds": 53.674,
      "correctedReferenceOverlapPercent": 82.73713644751419,
      "absoluteAreaDifferencePercent": 8.79074658254469
    },
    {
      "phase": "compatible",
      "residence": "Residence A",
      "model": "google/gemini-3.8-flash",
      "completed": true,
      "seconds": 88.25,
      "firstPolygonSeconds": 45.044,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "compatible",
      "residence": "Residence C",
      "model": "anthropic/claude-fable-5.1",
      "completed": false,
      "seconds": 34.184,
      "firstPolygonSeconds": 28.706,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "compatible",
      "residence": "Residence A",
      "model": "google/gemini-3.1-pro-preview",
      "completed": true,
      "seconds": 96.543,
      "firstPolygonSeconds": 56.37,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "compatible",
      "residence": "Residence C",
      "model": "google/gemini-3.1-pro-preview",
      "completed": true,
      "seconds": 170.91,
      "firstPolygonSeconds": 54.85,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "compatible",
      "residence": "Residence C",
      "model": "google/gemini-3.8-flash",
      "completed": false,
      "seconds": 180.003,
      "firstPolygonSeconds": 47.186,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "low",
      "residence": "Residence B",
      "model": "openai/gpt-6-astra",
      "completed": true,
      "seconds": 27.97,
      "firstPolygonSeconds": 6.648,
      "correctedReferenceOverlapPercent": 65.49155142811937,
      "absoluteAreaDifferencePercent": 29.737118822292324
    },
    {
      "phase": "low",
      "residence": "Residence A",
      "model": "openai/gpt-6-astra",
      "completed": true,
      "seconds": 20.836,
      "firstPolygonSeconds": 6.438,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "low",
      "residence": "Residence C",
      "model": "openai/gpt-6-astra",
      "completed": true,
      "seconds": 27.404,
      "firstPolygonSeconds": 4.805,
      "correctedReferenceOverlapPercent": null,
      "absoluteAreaDifferencePercent": null
    },
    {
      "phase": "repeat",
      "residence": "Residence B",
      "model": "openai/gpt-6-astra",
      "completed": true,
      "seconds": 45.497,
      "firstPolygonSeconds": 13.152,
      "correctedReferenceOverlapPercent": 75.15934120643114,
      "absoluteAreaDifferencePercent": 18.506834910620398
    },
    {
      "phase": "repeat",
      "residence": "Residence B",
      "model": "anthropic/claude-opus-5",
      "completed": true,
      "seconds": 32.285,
      "firstPolygonSeconds": 4.811,
      "correctedReferenceOverlapPercent": 75.10077065375162,
      "absoluteAreaDifferencePercent": 7.539432176656152
    }
  ]
}