Free Model Endpoints Make Good Fuzz Generators and Bad Oracles

// flagdemo.cpp
#include <iostream>
#include <string>
#include <vector>

int main(int argc, char** argv) {
    std::vector<std::string> args(argv + 1, argv + argc);
    std::string input;
    int count = 1;
    bool verbose = false;

    for (size_t i = 0; i < args.size(); ++i) {
        const std::string& a = args[i];
        if (a == "--verbose") {
            verbose = true;
        } else if (a == "--input") {
            if (i + 1 >= args.size()) return 2;
            input = args[++i];
        } else if (a == "--count") {
            if (i + 1 >= args.size()) return 2;
            try {
                count = std::stoi(args[++i]);
            } catch (...) {
                return 3;
            }
        } else {
            std::cerr << "unknown flag: " << a << "\n";
            return 4;
        }
    }

    std::cout << input << " " << count << " " << (verbose ? "1" : "0") << "\n";
    return 0;
}

# reference.py
def reference(args):
    input_ = ""
    count = 1
    verbose = False
    i = 0
    while i < len(args):
        a = args[i]
        if a == "--verbose":
            verbose = True
            i += 1
        elif a == "--input":
            if i + 1 >= len(args):
                return ("missing_value", 2)
            input_ = args[i + 1]
            i += 2
        elif a == "--count":
            if i + 1 >= len(args):
                return ("missing_value", 2)
            try:
                count = int(args[i + 1])
            except ValueError:
                return ("bad_count", 3)
            i += 2
        else:
            return ("unknown_flag", 4)
    return ("ok", 0, input_, count, verbose)

# fuzz.py
import json
import os
import subprocess
import urllib.request

# Provider-specific adapter: keep it thin and isolated.
def fetch_candidates(endpoint, prompt, auth_token):
    req = urllib.request.Request(
        endpoint,
        data=json.dumps({"prompt": prompt}).encode("utf-8"),
        headers={
            "Content-Type": "application/json",
            "Authorization": f"Bearer {auth_token}",
        },
    )
    with urllib.request.urlopen(req, timeout=30) as resp:
        payload = json.loads(resp.read().decode("utf-8"))
    return payload["cases"]

PROMPT = """
Generate 25 command-line argument arrays for a C++ program that accepts
--input, --verbose, and --count. Return only a JSON object with a "cases"
array of string arrays. Vary whitespace, Unicode, negative numbers, missing
values, repeated flags, huge values, and leading dashes. Do not include NUL
bytes or shell metacharacters.
"""

def run_binary(args):
    completed = subprocess.run(
        ["./flagdemo", *args],
        capture_output=True,
        timeout=2,
    )
    return completed.returncode, completed.stdout.decode(errors="replace")

def compare(case):
    expected = reference(case)
    actual_code, actual_stdout = run_binary(case)
    if expected[0] == "ok":
        expected_stdout = f"{expected[2]} {expected[3]} {int(expected[4])}\n"
        return actual_code == 0 and actual_stdout == expected_stdout, expected, actual_code, actual_stdout
    return actual_code == expected[1], expected, actual_code, actual_stdout

def main():
    endpoint = os.environ["MODEL_ENDPOINT"]
    token = os.environ["MODEL_TOKEN"]
    for case in fetch_candidates(endpoint, PROMPT, token):
        passed, expected, code, stdout = compare(case)
        if not passed:
            print("FAIL", json.dumps(case), expected, code, repr(stdout))

if __name__ == "__main__":
    main()

def record(case):
    corpus = json.load(open("corpus.json"))
    corpus.append(case)
    json.dump(corpus, open("corpus.json", "w"), indent=2)