feat: add pluggable multi-CLI backend system
Implement BackendAdapter interface with four CLI backends: - ClaudeCodeBackend (extracted from AgentRuntime) - CodexBackend (OpenAI Codex CLI) - GeminiBackend (Google Gemini CLI) - OpenCodeBackend (OpenCode CLI) Add BackendRegistry for resolution/creation via AGENT_BACKEND env var. Refactor AgentRuntime to delegate to BackendAdapter instead of hardcoding Claude CLI. Update GatewayConfig with new env vars (AGENT_BACKEND, BACKEND_CLI_PATH, BACKEND_MODEL, BACKEND_MAX_TURNS). Includes 10 property-based test files and unit tests for edge cases.
This commit is contained in:
163
tests/property/claude-backend.property.test.ts
Normal file
163
tests/property/claude-backend.property.test.ts
Normal file
@@ -0,0 +1,163 @@
|
||||
import { describe, it } from "vitest";
|
||||
import fc from "fast-check";
|
||||
import { ClaudeCodeBackend } from "../../src/backends/claude-backend.js";
|
||||
import type { BackendAdapterConfig } from "../../src/backends/types.js";
|
||||
|
||||
// Feature: multi-cli-backend, Property 1: Claude backend required flags
|
||||
// **Validates: Requirements 2.2, 2.5, 2.6**
|
||||
|
||||
/**
|
||||
* Arbitrary for non-empty strings that won't break CLI arg parsing.
|
||||
* Avoids empty strings since prompts/system prompts must be meaningful.
|
||||
*/
|
||||
const nonEmptyString = fc.string({ minLength: 1, maxLength: 200 });
|
||||
|
||||
/** Arbitrary for tool names (non-empty, no whitespace) */
|
||||
const toolName = fc.stringMatching(/^[A-Za-z][A-Za-z0-9_.-]{0,49}$/);
|
||||
|
||||
/** Arbitrary for a list of allowed tools */
|
||||
const toolsList = fc.array(toolName, { minLength: 0, maxLength: 10 });
|
||||
|
||||
/** Arbitrary for max turns (positive integer) */
|
||||
const maxTurns = fc.integer({ min: 1, max: 1000 });
|
||||
|
||||
function createBackend(allowedTools: string[], turns: number): ClaudeCodeBackend {
|
||||
const config: BackendAdapterConfig = {
|
||||
cliPath: "claude",
|
||||
workingDir: "/tmp",
|
||||
queryTimeoutMs: 60000,
|
||||
allowedTools,
|
||||
maxTurns: turns,
|
||||
};
|
||||
return new ClaudeCodeBackend(config);
|
||||
}
|
||||
|
||||
describe("Property 1: Claude backend required flags", () => {
|
||||
it("generated args always contain -p flag with the prompt", () => {
|
||||
fc.assert(
|
||||
fc.property(
|
||||
nonEmptyString,
|
||||
nonEmptyString,
|
||||
toolsList,
|
||||
maxTurns,
|
||||
(prompt, systemPromptFile, tools, turns) => {
|
||||
const backend = createBackend(tools, turns);
|
||||
const args = backend.buildArgs(prompt, systemPromptFile);
|
||||
|
||||
const pIndex = args.indexOf("-p");
|
||||
return pIndex !== -1 && args[pIndex + 1] === prompt;
|
||||
},
|
||||
),
|
||||
{ numRuns: 100 },
|
||||
);
|
||||
});
|
||||
|
||||
it("generated args always contain --output-format json", () => {
|
||||
fc.assert(
|
||||
fc.property(
|
||||
nonEmptyString,
|
||||
nonEmptyString,
|
||||
toolsList,
|
||||
maxTurns,
|
||||
(prompt, systemPromptFile, tools, turns) => {
|
||||
const backend = createBackend(tools, turns);
|
||||
const args = backend.buildArgs(prompt, systemPromptFile);
|
||||
|
||||
const idx = args.indexOf("--output-format");
|
||||
return idx !== -1 && args[idx + 1] === "json";
|
||||
},
|
||||
),
|
||||
{ numRuns: 100 },
|
||||
);
|
||||
});
|
||||
|
||||
it("generated args always contain --dangerously-skip-permissions", () => {
|
||||
fc.assert(
|
||||
fc.property(
|
||||
nonEmptyString,
|
||||
nonEmptyString,
|
||||
toolsList,
|
||||
maxTurns,
|
||||
(prompt, systemPromptFile, tools, turns) => {
|
||||
const backend = createBackend(tools, turns);
|
||||
const args = backend.buildArgs(prompt, systemPromptFile);
|
||||
|
||||
return args.includes("--dangerously-skip-permissions");
|
||||
},
|
||||
),
|
||||
{ numRuns: 100 },
|
||||
);
|
||||
});
|
||||
|
||||
it("generated args always contain --append-system-prompt-file with the file path", () => {
|
||||
fc.assert(
|
||||
fc.property(
|
||||
nonEmptyString,
|
||||
nonEmptyString,
|
||||
toolsList,
|
||||
maxTurns,
|
||||
(prompt, systemPromptFile, tools, turns) => {
|
||||
const backend = createBackend(tools, turns);
|
||||
const args = backend.buildArgs(prompt, systemPromptFile);
|
||||
|
||||
const idx = args.indexOf("--append-system-prompt-file");
|
||||
return idx !== -1 && args[idx + 1] === systemPromptFile;
|
||||
},
|
||||
),
|
||||
{ numRuns: 100 },
|
||||
);
|
||||
});
|
||||
|
||||
it("generated args always contain --max-turns with the configured value", () => {
|
||||
fc.assert(
|
||||
fc.property(
|
||||
nonEmptyString,
|
||||
nonEmptyString,
|
||||
toolsList,
|
||||
maxTurns,
|
||||
(prompt, systemPromptFile, tools, turns) => {
|
||||
const backend = createBackend(tools, turns);
|
||||
const args = backend.buildArgs(prompt, systemPromptFile);
|
||||
|
||||
const idx = args.indexOf("--max-turns");
|
||||
return idx !== -1 && args[idx + 1] === String(turns);
|
||||
},
|
||||
),
|
||||
{ numRuns: 100 },
|
||||
);
|
||||
});
|
||||
|
||||
it("generated args contain one --allowedTools entry per configured tool", () => {
|
||||
fc.assert(
|
||||
fc.property(
|
||||
nonEmptyString,
|
||||
nonEmptyString,
|
||||
toolsList,
|
||||
maxTurns,
|
||||
(prompt, systemPromptFile, tools, turns) => {
|
||||
const backend = createBackend(tools, turns);
|
||||
const args = backend.buildArgs(prompt, systemPromptFile);
|
||||
|
||||
// Collect all values following --allowedTools flags
|
||||
const allowedToolValues: string[] = [];
|
||||
for (let i = 0; i < args.length; i++) {
|
||||
if (args[i] === "--allowedTools") {
|
||||
allowedToolValues.push(args[i + 1]);
|
||||
}
|
||||
}
|
||||
|
||||
// Must have exactly one entry per configured tool
|
||||
if (allowedToolValues.length !== tools.length) return false;
|
||||
|
||||
// Each configured tool must appear
|
||||
for (const tool of tools) {
|
||||
if (!allowedToolValues.includes(tool)) return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
},
|
||||
),
|
||||
{ numRuns: 100 },
|
||||
);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user