构建支持语音的 OSDK 应用

npm install @openai/agents

import { createClient, type Client } from "@osdk/client";
import { createPublicOauthClient, type PublicOauthClient } from "@osdk/oauth";

function getMetaTagContent(tagName: string): string {
  const elements = document.querySelectorAll(`meta[name="${tagName}"]`);
  const element = elements.item(elements.length - 1);
  const value = element ? element.getAttribute("content") : null;
  if (value == null || value === "") {
    throw new Error(`Meta tag ${tagName} not found or empty`);
  }
  return value;
}

export const foundryUrl = getMetaTagContent("osdk-foundryUrl");
const clientId = getMetaTagContent("osdk-clientId");
const redirectUrl = getMetaTagContent("osdk-redirectUrl");
const ontologyRid = getMetaTagContent("osdk-ontologyRid");

const scopes = [
  "language-model-service:use-model",
  // Other scopes your application requires, for example:
  // "api:read-data",
  // "api:use-ontologies-read",
];

export const auth: PublicOauthClient = createPublicOauthClient(
  clientId,
  foundryUrl,
  redirectUrl,
  { scopes }
);

export const client: Client = createClient(foundryUrl, ontologyRid, auth);
export default client;

import { auth } from "../client";

let token = await auth.getTokenOrUndefined();
if (!token) {
  await auth.signIn();
  token = await auth.getTokenOrUndefined();
}
if (!token) {
  throw new Error("Unable to obtain access token. Please try again.");
}

wss://<your-foundry-domain>/language-model-service/ws/v1/open-ai/realtime?model=gpt-realtime-2

import {
  RealtimeAgent,
  RealtimeSession,
  OpenAIRealtimeWebSocket,
} from "@openai/agents/realtime";
import { foundryUrl } from "../client";

const PROXY_URL =
  `wss://${new URL(foundryUrl).host}/language-model-service/ws/v1/open-ai/realtime` +
  `?model=gpt-realtime-2`;
const CONNECT_TIMEOUT_MS = 10_000;

function createTransport(token: string): OpenAIRealtimeWebSocket {
  return new OpenAIRealtimeWebSocket({
    url: PROXY_URL,
    useInsecureApiKey: true, // auth is via the subprotocol below, not OpenAI's apiKey
    // eslint-disable-next-line @typescript-eslint/no-explicit-any
    createWebSocket: async ({ url }: { url: string }): Promise<any> => {
      const ws = new WebSocket(url, [`Bearer-${token}`]);

      ws.addEventListener("error", () => {
        console.error("WebSocket connection failed (check network tab for details)");
      });

      ws.addEventListener("close", (ev) => {
        // 1000 = normal close, do not surface as an error.
        if (ev.code !== 1000) {
          console.error(`WebSocket closed: ${ev.code}${ev.reason ? ` (${ev.reason})` : ""}`);
        }
      });

      return ws;
    },
  });
}

function createSession(
  agent: RealtimeAgent,
  transport: OpenAIRealtimeWebSocket,
): RealtimeSession {
  return new RealtimeSession(agent, {
    transport,
    model: "gpt-realtime-2",
    tracingDisabled: true,
    config: {
      outputModalities: ["audio"],
      audio: {
        input: {
          format: "pcm16",
          transcription: { model: "whisper-1" },
          turnDetection: {
            type: "server_vad",
            threshold: 0.8,
            silenceDurationMs: 500,
            prefixPaddingMs: 300,
          },
        },
        output: { format: "pcm16" },
      },
    },
  });
}

function awaitSessionReady(session: RealtimeSession): Promise<RealtimeSession> {
  return new Promise<RealtimeSession>((resolve, reject) => {
    const timeout = setTimeout(() => {
      reject(new Error("Connection timeout"));
    }, CONNECT_TIMEOUT_MS);

    let created = false;
    let updatedCount = 0;

    session.on("transport_event", (ev) => {
      if (ev.type === "session.created") {
        created = true;
      }
      if (ev.type === "session.updated" && created) {
        updatedCount++;
        if (updatedCount === 1) {
          setTimeout(() => {
            clearTimeout(timeout);
            resolve(session);
          }, 200);
        }
      }
    });

    session.connect({ apiKey: "unused" });
  });
}

export async function startVoiceSession(
  agent: RealtimeAgent,
  token: string,
): Promise<RealtimeSession> {
  const transport = createTransport(token);
  const session = createSession(agent, transport);
  return awaitSessionReady(session);
}

import { RealtimeAgent, tool } from "@openai/agents/realtime";
import { z } from "zod";
import { client } from "../client";

const lookupProduct = tool({
  name: "lookup_product",
  description: "Look up a product by name. Use this when the user asks about a specific product.",
  parameters: z.object({
    name: z.string().describe("The product name to look up"),
  }),
  execute: async ({ name }) => {
    // TODO: replace this stub with an OSDK query against your Ontology.
    // For example, if your Ontology has a Product object type:
    //
    //   const page = await client(Product)
    //     .where({ name: { $startsWith: name } })
    //     .fetchPage({ $pageSize: 5 });
    //   return JSON.stringify(page.data.map((p) => ({
    //     id: p.id, name: p.name, price: p.price, stock: p.stock,
    //   })));
    //
    return JSON.stringify({
      id: "stub-1",
      name,
      price: 0,
      stock: 0,
      note: "Replace the tool body with an OSDK query against your Ontology.",
    });
  },
});

const agent = new RealtimeAgent({
  name: "Assistant",
  instructions:
    "You help users find products. When the user asks about a product, call the lookup_product tool.",
  tools: [lookupProduct],
});

const session = await startVoiceSession(agent, token);

// Surface mid-session errors from the SDK. WebSocket-level transport errors
// (such as a failure to establish the connection) are logged to the console
// from the handlers in `createTransport`; if the connection never establishes,
// `startVoiceSession` rejects after `CONNECT_TIMEOUT_MS`.
session.on("error", (event) => {
  console.error("Realtime session error:", event.error);
});

// Play back audio from the model
session.on("audio", (event) => {
  playbackQueue.enqueue(event.data); // your playback implementation
});

// Clear playback when the model is interrupted (for example, the user starts speaking)
session.on("audio_interrupted", () => {
  playbackQueue.clear();
});

// Send mic audio to the model as 16-bit PCM chunks at 24 kHz arrive
startMicCapture((pcm16) => session.sendAudio(pcm16)); // your mic capture implementation

// Optional: prompt the agent to greet the user immediately rather than
// waiting for the user to speak first.
session.sendMessage("(Session started. Greet the user briefly.)");