509 lines
17 KiB
TypeScript
509 lines
17 KiB
TypeScript
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { chmod, mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
|
|
import { spawn } from "node:child_process";
|
|
import { tmpdir } from "node:os";
|
|
import { dirname, join } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
// Tests spawn a wrapper subprocess and rely on the wrapper to exit quickly.
|
|
// When the full test suite runs with concurrency the host can be CPU-constrained
|
|
// and subprocess startup/teardown can be delayed. Keep this timeout relaxed
|
|
// to avoid flakiness while preserving the test's intent.
|
|
function waitForExit(child: ReturnType<typeof spawn>, timeoutMs = 8000): Promise<number> {
|
|
return new Promise((resolve, reject) => {
|
|
const timeout = setTimeout(() => {
|
|
child.kill("SIGKILL");
|
|
reject(new Error(`wrapper did not exit within ${timeoutMs}ms`));
|
|
}, timeoutMs);
|
|
|
|
child.on("error", (error) => {
|
|
clearTimeout(timeout);
|
|
reject(error);
|
|
});
|
|
|
|
child.on("close", (code) => {
|
|
clearTimeout(timeout);
|
|
resolve(code ?? 0);
|
|
});
|
|
});
|
|
}
|
|
|
|
async function runWrapperWithFakePi(requestedModel: string, resolvedModel?: string) {
|
|
const dir = await mkdtemp(join(tmpdir(), "pi-subagents-wrapper-"));
|
|
const metaPath = join(dir, "meta.json");
|
|
const resultPath = join(dir, "result.json");
|
|
const capturePath = join(dir, "capture.json");
|
|
const piPath = join(dir, "pi");
|
|
|
|
const resolved = typeof resolvedModel === "string" ? resolvedModel : requestedModel;
|
|
await writeFile(
|
|
piPath,
|
|
[
|
|
`#!${process.execPath}`,
|
|
"const fs = require('fs');",
|
|
`const capturePath = ${JSON.stringify(capturePath)};`,
|
|
"const obj = {",
|
|
" PI_SUBAGENTS_GITHUB_COPILOT_INITIATOR: process.env.PI_SUBAGENTS_GITHUB_COPILOT_INITIATOR || '',",
|
|
" PI_SUBAGENTS_CHILD: process.env.PI_SUBAGENTS_CHILD || '',",
|
|
" argv: process.argv.slice(2)",
|
|
"};",
|
|
"fs.writeFileSync(capturePath, JSON.stringify(obj), 'utf8');",
|
|
"console.log(JSON.stringify({type:'message_end',message:{role:'assistant',content:[{type:'text',text:'done'}],model:'github-copilot/gpt-4o',stopReason:'stop'}}));",
|
|
].join("\n"),
|
|
"utf8",
|
|
);
|
|
await chmod(piPath, 0o755);
|
|
|
|
await writeFile(
|
|
metaPath,
|
|
JSON.stringify(
|
|
{
|
|
runId: "run-1",
|
|
mode: "single",
|
|
task: "inspect auth",
|
|
cwd: dir,
|
|
requestedModel,
|
|
resolvedModel: resolved,
|
|
startedAt: "2026-04-09T00:00:00.000Z",
|
|
sessionPath: join(dir, "child-session.jsonl"),
|
|
eventsPath: join(dir, "events.jsonl"),
|
|
resultPath,
|
|
stdoutPath: join(dir, "stdout.log"),
|
|
stderrPath: join(dir, "stderr.log"),
|
|
transcriptPath: join(dir, "transcript.log"),
|
|
tools: ["read", "grep"],
|
|
systemPromptPath: join(dir, "system-prompt.md"),
|
|
},
|
|
null,
|
|
2,
|
|
),
|
|
"utf8",
|
|
);
|
|
|
|
const wrapperPath = join(dirname(fileURLToPath(import.meta.url)), "cli.mjs");
|
|
const child = spawn(process.execPath, [wrapperPath, metaPath], {
|
|
env: {
|
|
...process.env,
|
|
PATH: dir,
|
|
},
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
|
|
const exitCode = await waitForExit(child);
|
|
assert.equal(exitCode, 0);
|
|
|
|
const captureJson = JSON.parse(await readFile(capturePath, "utf8"));
|
|
return { flags: captureJson };
|
|
}
|
|
|
|
test("wrapper marks github-copilot child run as a subagent child", async () => {
|
|
const captured = await runWrapperWithFakePi("github-copilot/gpt-4o");
|
|
assert.equal(captured.flags.PI_SUBAGENTS_CHILD, "1");
|
|
});
|
|
|
|
test("wrapper marks anthropic child run as a subagent child", async () => {
|
|
const captured = await runWrapperWithFakePi("anthropic/claude-sonnet-4-5");
|
|
assert.equal(captured.flags.PI_SUBAGENTS_CHILD, "1");
|
|
});
|
|
|
|
test("wrapper respects tools metadata but ignores stale system prompt file", async () => {
|
|
const captured = await runWrapperWithFakePi("anthropic/claude-sonnet-4-5");
|
|
// tools metadata should be passed even if there are no tool binaries installed.
|
|
assert.equal(captured.flags.argv.includes("--tools"), true);
|
|
// system prompt file is not present in this test, so the flag should not be passed.
|
|
assert.equal(captured.flags.argv.includes("--append-system-prompt"), false);
|
|
});
|
|
|
|
test("wrapper passes --append-system-prompt when the prompt file exists", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "pi-subagents-wrapper-"));
|
|
const metaPath = join(dir, "meta.json");
|
|
const resultPath = join(dir, "result.json");
|
|
const capturePath = join(dir, "capture.json");
|
|
const piPath = join(dir, "pi");
|
|
const promptPath = join(dir, "system-prompt.md");
|
|
|
|
await writeFile(
|
|
piPath,
|
|
[
|
|
`#!${process.execPath}`,
|
|
"const fs = require('fs');",
|
|
`const capturePath = ${JSON.stringify(capturePath)};`,
|
|
"const obj = {",
|
|
" PI_SUBAGENTS_GITHUB_COPILOT_INITIATOR: process.env.PI_SUBAGENTS_GITHUB_COPILOT_INITIATOR || '',",
|
|
" PI_SUBAGENTS_CHILD: process.env.PI_SUBAGENTS_CHILD || '',",
|
|
" argv: process.argv.slice(2)",
|
|
"};",
|
|
"fs.writeFileSync(capturePath, JSON.stringify(obj), 'utf8');",
|
|
"console.log(JSON.stringify({type:'message_end',message:{role:'assistant',content:[{type:'text',text:'done'}],model:'github-copilot/gpt-4o',stopReason:'stop'}}));",
|
|
].join("\n"),
|
|
"utf8",
|
|
);
|
|
await chmod(piPath, 0o755);
|
|
|
|
// create the system prompt file so the wrapper will pass --append-system-prompt
|
|
await writeFile(promptPath, "System prompt here", "utf8");
|
|
|
|
await writeFile(
|
|
metaPath,
|
|
JSON.stringify(
|
|
{
|
|
runId: "run-1",
|
|
mode: "single",
|
|
task: "inspect auth",
|
|
cwd: dir,
|
|
requestedModel: "anthropic/claude-sonnet-4-5",
|
|
resolvedModel: "anthropic/claude-sonnet-4-5",
|
|
startedAt: "2026-04-09T00:00:00.000Z",
|
|
sessionPath: join(dir, "child-session.jsonl"),
|
|
eventsPath: join(dir, "events.jsonl"),
|
|
resultPath,
|
|
stdoutPath: join(dir, "stdout.log"),
|
|
stderrPath: join(dir, "stderr.log"),
|
|
transcriptPath: join(dir, "transcript.log"),
|
|
tools: ["read", "grep"],
|
|
systemPromptPath: promptPath,
|
|
},
|
|
null,
|
|
2,
|
|
),
|
|
"utf8",
|
|
);
|
|
|
|
const wrapperPath = join(dirname(fileURLToPath(import.meta.url)), "cli.mjs");
|
|
const child = spawn(process.execPath, [wrapperPath, metaPath], {
|
|
env: {
|
|
...process.env,
|
|
PATH: dir,
|
|
},
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
|
|
const exitCode = await waitForExit(child);
|
|
assert.equal(exitCode, 0);
|
|
|
|
const captureJson = JSON.parse(await readFile(capturePath, "utf8"));
|
|
const argv = captureJson.argv;
|
|
assert.equal(argv.includes("--tools"), true);
|
|
const toolsIndex = argv.indexOf("--tools");
|
|
assert.equal(argv[toolsIndex + 1], "read,grep");
|
|
assert.equal(argv.includes("--append-system-prompt"), true);
|
|
const idx = argv.indexOf("--append-system-prompt");
|
|
assert.equal(argv[idx + 1], promptPath);
|
|
});
|
|
|
|
test("wrapper marks github-copilot child runs as agent-initiated", async () => {
|
|
const captured = await runWrapperWithFakePi("github-copilot/gpt-4o");
|
|
assert.equal(captured.flags.PI_SUBAGENTS_GITHUB_COPILOT_INITIATOR, "agent");
|
|
assert.equal(captured.flags.PI_SUBAGENTS_CHILD, "1");
|
|
});
|
|
|
|
test("wrapper leaves non-copilot child runs unchanged", async () => {
|
|
const captured = await runWrapperWithFakePi("anthropic/claude-sonnet-4-5");
|
|
assert.equal(captured.flags.PI_SUBAGENTS_GITHUB_COPILOT_INITIATOR, "");
|
|
assert.equal(captured.flags.PI_SUBAGENTS_CHILD, "1");
|
|
});
|
|
|
|
test("wrapper uses effective model for both argv and env when requested/resolved differ", async () => {
|
|
const requested = "anthropic/claude-sonnet-4-5";
|
|
const resolved = "github-copilot/gpt-4o";
|
|
|
|
const captured = await runWrapperWithFakePi(requested, resolved);
|
|
|
|
assert.equal(captured.flags.PI_SUBAGENTS_GITHUB_COPILOT_INITIATOR, "agent");
|
|
assert.equal(captured.flags.PI_SUBAGENTS_CHILD, "1");
|
|
|
|
const argv = captured.flags.argv;
|
|
const modelIndex = argv.indexOf("--model");
|
|
assert.ok(modelIndex >= 0, "expected --model in argv");
|
|
assert.equal(argv[modelIndex + 1], resolved);
|
|
});
|
|
|
|
test("wrapper exits and writes result.json when the pi child cannot be spawned", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "pi-subagents-wrapper-"));
|
|
const metaPath = join(dir, "meta.json");
|
|
const resultPath = join(dir, "result.json");
|
|
|
|
await writeFile(
|
|
metaPath,
|
|
JSON.stringify(
|
|
{
|
|
runId: "run-1",
|
|
mode: "single",
|
|
task: "inspect auth",
|
|
cwd: dir,
|
|
requestedModel: "anthropic/claude-sonnet-4-5",
|
|
resolvedModel: "anthropic/claude-sonnet-4-5",
|
|
startedAt: "2026-04-09T00:00:00.000Z",
|
|
sessionPath: join(dir, "child-session.jsonl"),
|
|
eventsPath: join(dir, "events.jsonl"),
|
|
resultPath,
|
|
stdoutPath: join(dir, "stdout.log"),
|
|
stderrPath: join(dir, "stderr.log"),
|
|
transcriptPath: join(dir, "transcript.log"),
|
|
systemPromptPath: join(dir, "system-prompt.md"),
|
|
},
|
|
null,
|
|
2,
|
|
),
|
|
"utf8",
|
|
);
|
|
|
|
const wrapperPath = join(dirname(fileURLToPath(import.meta.url)), "cli.mjs");
|
|
const child = spawn(process.execPath, [wrapperPath, metaPath], {
|
|
env: {
|
|
...process.env,
|
|
PATH: dir,
|
|
},
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
|
|
const exitCode = await waitForExit(child);
|
|
assert.equal(exitCode, 1);
|
|
|
|
const result = JSON.parse(await readFile(resultPath, "utf8"));
|
|
assert.equal(result.runId, "run-1");
|
|
assert.equal(result.task, "inspect auth");
|
|
assert.equal("agent" in result, false);
|
|
assert.equal(result.exitCode, 1);
|
|
assert.match(result.errorMessage ?? "", /ENOENT|not found|spawn pi/i);
|
|
});
|
|
|
|
test("wrapper does not exit early on non-terminal toolUse assistant messages", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "pi-subagents-wrapper-"));
|
|
const metaPath = join(dir, "meta.json");
|
|
const resultPath = join(dir, "result.json");
|
|
const eventsPath = join(dir, "events.jsonl");
|
|
const piPath = join(dir, "pi");
|
|
|
|
await writeFile(
|
|
piPath,
|
|
[
|
|
`#!${process.execPath}`,
|
|
"console.log(JSON.stringify({type:'message_end',message:{role:'assistant',content:[{type:'text',text:'starting'}],model:'openai/gpt-5',stopReason:'toolUse'}}));",
|
|
"setTimeout(() => console.log(JSON.stringify({type:'tool_execution_start',toolName:'read',args:{path:'src/auth.ts'}})), 400);",
|
|
"setTimeout(() => console.log(JSON.stringify({type:'message_end',message:{role:'assistant',content:[{type:'text',text:'final'}],model:'openai/gpt-5',stopReason:'stop'}})), 700);",
|
|
"setTimeout(() => process.exit(0), 1200);",
|
|
].join("\n"),
|
|
"utf8",
|
|
);
|
|
await chmod(piPath, 0o755);
|
|
|
|
await writeFile(
|
|
metaPath,
|
|
JSON.stringify(
|
|
{
|
|
runId: "run-1",
|
|
mode: "single",
|
|
task: "inspect auth",
|
|
cwd: dir,
|
|
requestedModel: "openai/gpt-5",
|
|
resolvedModel: "openai/gpt-5",
|
|
startedAt: "2026-04-09T00:00:00.000Z",
|
|
sessionPath: join(dir, "child-session.jsonl"),
|
|
eventsPath,
|
|
resultPath,
|
|
stdoutPath: join(dir, "stdout.log"),
|
|
stderrPath: join(dir, "stderr.log"),
|
|
transcriptPath: join(dir, "transcript.log"),
|
|
systemPromptPath: join(dir, "system-prompt.md"),
|
|
},
|
|
null,
|
|
2,
|
|
),
|
|
"utf8",
|
|
);
|
|
|
|
const wrapperPath = join(dirname(fileURLToPath(import.meta.url)), "cli.mjs");
|
|
const child = spawn(process.execPath, [wrapperPath, metaPath], {
|
|
env: { ...process.env, PATH: dir },
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
|
|
const exitCode = await waitForExit(child, 8000);
|
|
assert.equal(exitCode, 0);
|
|
|
|
const result = JSON.parse(await readFile(resultPath, "utf8"));
|
|
assert.equal(result.exitCode, 0);
|
|
assert.equal(result.stopReason, "stop");
|
|
assert.equal(result.rawStopReason, "stop");
|
|
assert.equal(result.finalText, "final");
|
|
|
|
const eventsText = await readFile(eventsPath, "utf8");
|
|
assert.match(eventsText, /"type":"tool_call"/);
|
|
});
|
|
|
|
test("wrapper skips blank assistant transcript lines before later tool activity", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "pi-subagents-wrapper-"));
|
|
const metaPath = join(dir, "meta.json");
|
|
const resultPath = join(dir, "result.json");
|
|
const transcriptPath = join(dir, "transcript.log");
|
|
const piPath = join(dir, "pi");
|
|
|
|
await writeFile(
|
|
piPath,
|
|
[
|
|
`#!${process.execPath}`,
|
|
"console.log(JSON.stringify({type:'message_end',message:{role:'assistant',content:[{type:'text',text:' '}],model:'openai/gpt-5',stopReason:'toolUse'}}));",
|
|
"setTimeout(() => console.log(JSON.stringify({type:'tool_execution_start',toolName:'read',args:{path:'src/auth.ts'}})), 100);",
|
|
"setTimeout(() => console.log(JSON.stringify({type:'message_end',message:{role:'assistant',content:[{type:'text',text:'done'}],model:'openai/gpt-5',stopReason:'stop'}})), 200);",
|
|
"setTimeout(() => process.exit(0), 400);",
|
|
].join("\n"),
|
|
"utf8",
|
|
);
|
|
await chmod(piPath, 0o755);
|
|
|
|
await writeFile(
|
|
metaPath,
|
|
JSON.stringify(
|
|
{
|
|
runId: "run-1",
|
|
mode: "single",
|
|
task: "inspect auth",
|
|
cwd: dir,
|
|
requestedModel: "openai/gpt-5",
|
|
resolvedModel: "openai/gpt-5",
|
|
startedAt: "2026-04-09T00:00:00.000Z",
|
|
sessionPath: join(dir, "child-session.jsonl"),
|
|
eventsPath: join(dir, "events.jsonl"),
|
|
resultPath,
|
|
stdoutPath: join(dir, "stdout.log"),
|
|
stderrPath: join(dir, "stderr.log"),
|
|
transcriptPath,
|
|
systemPromptPath: join(dir, "system-prompt.md"),
|
|
},
|
|
null,
|
|
2,
|
|
),
|
|
"utf8",
|
|
);
|
|
|
|
const wrapperPath = join(dirname(fileURLToPath(import.meta.url)), "cli.mjs");
|
|
const child = spawn(process.execPath, [wrapperPath, metaPath], {
|
|
env: { ...process.env, PATH: dir },
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
|
|
const exitCode = await waitForExit(child, 8000);
|
|
assert.equal(exitCode, 0);
|
|
|
|
const transcript = await readFile(transcriptPath, "utf8");
|
|
assert.doesNotMatch(transcript, /\nnull\n/);
|
|
assert.match(transcript, /Reading src\/auth.ts/);
|
|
assert.match(transcript, /done/);
|
|
});
|
|
|
|
test("wrapper exits and writes result.json after terminal output even if the pi child lingers", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "pi-subagents-wrapper-"));
|
|
const metaPath = join(dir, "meta.json");
|
|
const resultPath = join(dir, "result.json");
|
|
const piPath = join(dir, "pi");
|
|
|
|
await writeFile(
|
|
piPath,
|
|
[
|
|
`#!${process.execPath}`,
|
|
"console.log(JSON.stringify({type:'message_end',message:{role:'assistant',content:[{type:'text',text:'done'}],model:'openai/gpt-5',stopReason:'stop'}}));",
|
|
"setTimeout(() => {}, 10_000);",
|
|
].join("\n"),
|
|
"utf8",
|
|
);
|
|
await chmod(piPath, 0o755);
|
|
|
|
await writeFile(
|
|
metaPath,
|
|
JSON.stringify(
|
|
{
|
|
runId: "run-1",
|
|
mode: "single",
|
|
task: "inspect auth",
|
|
cwd: dir,
|
|
requestedModel: "openai/gpt-5",
|
|
resolvedModel: "openai/gpt-5",
|
|
startedAt: "2026-04-09T00:00:00.000Z",
|
|
sessionPath: join(dir, "child-session.jsonl"),
|
|
eventsPath: join(dir, "events.jsonl"),
|
|
resultPath,
|
|
stdoutPath: join(dir, "stdout.log"),
|
|
stderrPath: join(dir, "stderr.log"),
|
|
transcriptPath: join(dir, "transcript.log"),
|
|
systemPromptPath: join(dir, "system-prompt.md"),
|
|
},
|
|
null,
|
|
2,
|
|
),
|
|
"utf8",
|
|
);
|
|
|
|
const wrapperPath = join(dirname(fileURLToPath(import.meta.url)), "cli.mjs");
|
|
const child = spawn(process.execPath, [wrapperPath, metaPath], {
|
|
env: { ...process.env, PATH: dir },
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
|
|
const exitCode = await waitForExit(child);
|
|
assert.equal(exitCode, 0);
|
|
|
|
const result = JSON.parse(await readFile(resultPath, "utf8"));
|
|
assert.equal(result.exitCode, 0);
|
|
assert.equal(result.stopReason, "stop");
|
|
assert.equal(result.finalText, "done");
|
|
assert.equal(result.resolvedModel, "openai/gpt-5");
|
|
});
|
|
|
|
test("wrapper still writes result.json when transcript/stdout artifact writes fail", async () => {
|
|
const dir = await mkdtemp(join(tmpdir(), "pi-subagents-wrapper-"));
|
|
const metaPath = join(dir, "meta.json");
|
|
const resultPath = join(dir, "result.json");
|
|
const piPath = join(dir, "pi");
|
|
const brokenArtifactPath = join(dir, "broken-artifact");
|
|
await mkdir(brokenArtifactPath);
|
|
|
|
await writeFile(
|
|
piPath,
|
|
[
|
|
`#!${process.execPath}`,
|
|
"console.log(JSON.stringify({type:'message_end',message:{role:'assistant',content:[{type:'text',text:'done'}],model:'openai/gpt-5',stopReason:'stop'}}));",
|
|
].join("\n"),
|
|
"utf8",
|
|
);
|
|
await chmod(piPath, 0o755);
|
|
|
|
await writeFile(
|
|
metaPath,
|
|
JSON.stringify(
|
|
{
|
|
runId: "run-1",
|
|
mode: "single",
|
|
task: "inspect auth",
|
|
cwd: dir,
|
|
requestedModel: "openai/gpt-5",
|
|
resolvedModel: "openai/gpt-5",
|
|
sessionPath: join(dir, "child-session.jsonl"),
|
|
eventsPath: join(dir, "events.jsonl"),
|
|
resultPath,
|
|
stdoutPath: brokenArtifactPath,
|
|
stderrPath: join(dir, "stderr.log"),
|
|
transcriptPath: brokenArtifactPath,
|
|
systemPromptPath: join(dir, "system-prompt.md"),
|
|
},
|
|
null,
|
|
2,
|
|
),
|
|
"utf8",
|
|
);
|
|
|
|
const wrapperPath = join(dirname(fileURLToPath(import.meta.url)), "cli.mjs");
|
|
const child = spawn(process.execPath, [wrapperPath, metaPath], {
|
|
env: { ...process.env, PATH: dir },
|
|
stdio: ["ignore", "pipe", "pipe"],
|
|
});
|
|
|
|
const exitCode = await waitForExit(child);
|
|
assert.equal(exitCode, 0);
|
|
|
|
const result = JSON.parse(await readFile(resultPath, "utf8"));
|
|
assert.equal(result.exitCode, 0);
|
|
assert.equal(result.finalText, "done");
|
|
});
|