a step · two agents in parallel, one contract, a scorer

const researchStep = createStep({
  id: 'research',
  execute: async ({ inputData }) => {
    // two sources at once: the docs agent from the last slide,
    // and a coding agent reading the repository, minutes
    const [docs, analysis] = await Promise.all([
      docsAgent.generate(inputData.body, {
        structuredOutput: { schema: docsAnswerSchema },
      }),
      taskRock8CloudAgent({
        service: 'demo', type: 'research', prompt: inputData.body,
      }),
    ])
    // this step gathers. The next one decides.
    return { ...inputData, docs: docs.object, analysis: analysis.text }
  },
  // online: a scorer on a quarter of real runs
  scorers: {
    grounded: { scorer: isGrounded,
      sampling: { type: 'ratio', rate: 0.25 } },
  },
})

a scorer · is the research grounded?

export const isGrounded = createScorer({
  id: 'is-grounded',
  description:
    'Do both sources point at something real: ' +
    'a docs page, and a file in the repository?',
}).generateScore(({ run }) => {
  const { docs, analysis } = run.output
  const checks = [
    docs.cited.length > 0,
    // a file path in the analysis text
    /[\w\/-]+\.\w+/.test(analysis),
  ]
  const passed = checks.filter(Boolean).length
  return passed / checks.length
})