feat(mockup): add OpenAI Codex (gpt-5.6-sol + built-in image_gen) raster mockup generator

This commit is contained in:
airent106 committed 2026-10-29 11:33:33 -06:00
1 parent 744d29c003
commit 8a25771839
6 files changed
+215 -24

No files matched your search

+2
View File
@@ -6,6 +6,8 @@ ENV NODE_ENV=production
ENV PORT=80 ENV PORT=80
ENV DATA_DIR=/app/store ENV DATA_DIR=/app/store
RUN npm install -g @openai/codex@latest
COPY package*.json ./ COPY package*.json ./
RUN npm install --omit=dev RUN npm install --omit=dev
+1
View File
@@ -7,6 +7,7 @@ services:
restart: unless-stopped restart: unless-stopped
volumes: volumes:
- /home/chrisf/docker/appdata/sparkvault/data:/app/store - /home/chrisf/docker/appdata/sparkvault/data:/app/store
- /home/chrisf/.codex:/root/.codex
networks: networks:
- t3_proxy - t3_proxy
ports: ports:
+1 -1
View File
@@ -1803,7 +1803,7 @@ function bindGlobalEvents() {
const editInstruction = document.getElementById("nb-edit-instruction-input").value.trim(); const editInstruction = document.getElementById("nb-edit-instruction-input").value.trim();
btn.disabled = true; btn.disabled = true;
statusLabel.textContent = "🍌 Calling Nano Banana 2.1 (gemini-nano-banana-2.1)..."; statusLabel.textContent = "🎨 Generating high-res AI mockup via OpenAI Codex (takes ~15–25s)...";
try { try {
const res = await apiFetch(`/api/ideas/${state.activeIdeaId}/mockups/nano-banana`, { const res = await apiFetch(`/api/ideas/${state.activeIdeaId}/mockups/nano-banana`, {
+4 -3
View File
@@ -534,7 +534,7 @@
<div class="studio-action-row"> <div class="studio-action-row">
<button type="button" id="btn-generate-nano-banana" class="btn btn-primary"> <button type="button" id="btn-generate-nano-banana" class="btn btn-primary">
🍌 Generate with Nano Banana 2.1 🎨 Generate AI Mockup (Codex / Nano Banana)
</button> </button>
<span id="nb-generation-status" class="muted-label"></span> <span id="nb-generation-status" class="muted-label"></span>
</div> </div>
@@ -684,9 +684,10 @@
<small id="settings-key-status" class="muted-label"></small> <small id="settings-key-status" class="muted-label"></small>
</div> </div>
<div class="field-group"> <div class="field-group">
<label for="settings-model-select">Nano Banana Model</label> <label for="settings-model-select">AI Mockup Engine / Model</label>
<select id="settings-model-select" class="filter-select full-width"> <select id="settings-model-select" class="filter-select full-width">
<option value="gemini-nano-banana-2.1">gemini-nano-banana-2.1 — Nano Banana 2.1 (Default High-Efficiency)</option> <option value="codex-image-gen">codex-image-gen — OpenAI Codex (gpt-5.6-sol + Built-in image_gen) [Recommended]</option>
<option value="gemini-nano-banana-2.1">gemini-nano-banana-2.1 — Nano Banana 2.1 (Google Gemini)</option>
<option value="gemini-3-pro-image">gemini-3-pro-image — Nano Banana Pro (High-Detail Studio)</option> <option value="gemini-3-pro-image">gemini-3-pro-image — Nano Banana Pro (High-Detail Studio)</option>
<option value="gemini-3.1-flash-lite-image">gemini-3.1-flash-lite-image — Nano Banana 2 Lite (Ultra-Fast)</option> <option value="gemini-3.1-flash-lite-image">gemini-3.1-flash-lite-image — Nano Banana 2 Lite (Ultra-Fast)</option>
</select> </select>
+23 -8
View File
@@ -14,7 +14,8 @@ import {
NANO_BANANA_MODELS, NANO_BANANA_MODELS,
SURFACE_PRESETS, SURFACE_PRESETS,
buildNanoBananaPrompt, buildNanoBananaPrompt,
generateWithNanoBanana generateWithNanoBanana,
isCodexConfigured
} from "./nano_banana.js"; } from "./nano_banana.js";
const __filename = fileURLToPath(import.meta.url); const __filename = fileURLToPath(import.meta.url);
@@ -328,7 +329,7 @@ export function createAppServer({
}); });
const storedKey = store.getSetting("gemini_api_key", ""); const storedKey = store.getSetting("gemini_api_key", "");
const selectedModel = body.model || store.getSetting("nano_banana_model", "gemini-nano-banana-2.1"); const selectedModel = body.model || store.getSetting("nano_banana_model", "codex-image-gen");
try { try {
const genResult = await generateWithNanoBanana({ const genResult = await generateWithNanoBanana({
@@ -479,12 +480,24 @@ export function createAppServer({
const savedKey = store.getSetting("gemini_api_key", ""); const savedKey = store.getSetting("gemini_api_key", "");
const envKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || ""; const envKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || "";
const activeKey = savedKey || envKey; const activeKey = savedKey || envKey;
const model = store.getSetting("nano_banana_model", "gemini-nano-banana-2.1"); const codexReady = isCodexConfigured();
const model = store.getSetting("nano_banana_model", "codex-image-gen");
return sendJson(res, 200, { return sendJson(res, 200, {
hasApiKey: Boolean(activeKey), hasApiKey: Boolean(activeKey) || codexReady,
apiKeySource: savedKey ? "settings" : envKey ? "environment" : "none", codexReady,
maskedKey: activeKey ? `${activeKey.slice(0, 4)}...${activeKey.slice(-4)}` : "", apiKeySource: codexReady
? "OpenAI Codex (OAuth)"
: savedKey
? "settings"
: envKey
? "environment"
: "none",
maskedKey: activeKey
? `${activeKey.slice(0, 4)}...${activeKey.slice(-4)}`
: codexReady
? "Authenticated via ~/.codex/auth.json"
: "",
nanoBananaModel: model, nanoBananaModel: model,
availableModels: NANO_BANANA_MODELS, availableModels: NANO_BANANA_MODELS,
surfacePresets: Object.entries(SURFACE_PRESETS).map(([id, val]) => ({ surfacePresets: Object.entries(SURFACE_PRESETS).map(([id, val]) => ({
@@ -504,10 +517,12 @@ export function createAppServer({
const savedKey = store.getSetting("gemini_api_key", ""); const savedKey = store.getSetting("gemini_api_key", "");
const envKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || ""; const envKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || "";
const activeKey = savedKey || envKey; const activeKey = savedKey || envKey;
const codexReady = isCodexConfigured();
return sendJson(res, 200, { return sendJson(res, 200, {
saved: true, saved: true,
hasApiKey: Boolean(activeKey), hasApiKey: Boolean(activeKey) || codexReady,
nanoBananaModel: store.getSetting("nano_banana_model", "gemini-nano-banana-2.1") codexReady,
nanoBananaModel: store.getSetting("nano_banana_model", "codex-image-gen")
}); });
} }
} }
+184 -12
View File
@@ -1,21 +1,28 @@
import * as fs from "node:fs"; import * as fs from "node:fs";
import * as path from "node:path"; import * as path from "node:path";
import * as os from "node:os";
import { spawn } from "node:child_process";
import { GoogleGenAI } from "@google/genai"; import { GoogleGenAI } from "@google/genai";
export const NANO_BANANA_MODELS = [ export const NANO_BANANA_MODELS = [
{
id: "codex-image-gen",
label: "OpenAI Codex (gpt-5.6-sol + Built-in image_gen) — Recommended",
description: "codex-image-gen — High-resolution photorealistic raster PNG product & apparel mockups via OpenAI Codex built-in image_gen"
},
{ {
id: "gemini-nano-banana-2.1", id: "gemini-nano-banana-2.1",
label: "Nano Banana 2.1 (Default High-Efficiency + Free-Tier Auto Fallback)", label: "Nano Banana 2.1 (Google Gemini + Free-Tier Auto Fallback)",
description: "gemini-nano-banana-2.1 — Uses native Nano Banana 2.1 on Tier 1 keys, with automatic Gemini 3.1 Flash-Lite Vector Mockup fallback on Free Tier keys" description: "gemini-nano-banana-2.1 — Uses native Nano Banana 2.1 on Tier 1 keys, with automatic Gemini 3.1 Flash-Lite Vector Mockup fallback on Free Tier keys"
}, },
{ {
id: "gemini-3-pro-image", id: "gemini-3-pro-image",
label: "Nano Banana Pro (Studio Quality)", label: "Nano Banana Pro (Gemini Studio Quality)",
description: "gemini-3-pro-image — High-detail studio mockup generation and multi-turn editing" description: "gemini-3-pro-image — High-detail studio mockup generation and multi-turn editing"
}, },
{ {
id: "gemini-3.1-flash-lite-image", id: "gemini-3.1-flash-lite-image",
label: "Nano Banana 2 Lite (Ultra-Fast)", label: "Nano Banana 2 Lite (Gemini Ultra-Fast)",
description: "gemini-3.1-flash-lite-image — Rapid concept sketching and quick iterations" description: "gemini-3.1-flash-lite-image — Rapid concept sketching and quick iterations"
} }
]; ];
@@ -88,6 +95,149 @@ export function buildNanoBananaPrompt({
return finalPrompt; return finalPrompt;
} }
export function isCodexConfigured() {
const codexHome = process.env.CODEX_HOME || path.join(os.homedir(), ".codex");
return fs.existsSync(path.join(codexHome, "auth.json"));
}
function findNewestGeneratedImage(generatedImagesRoot, startedAtMs) {
if (!fs.existsSync(generatedImagesRoot)) return null;
let newestFile = null;
let newestMtime = 0;
for (const entry of fs.readdirSync(generatedImagesRoot, { withFileTypes: true })) {
const fullEntry = path.join(generatedImagesRoot, entry.name);
if (entry.isDirectory()) {
for (const sub of fs.readdirSync(fullEntry, { withFileTypes: true })) {
if (sub.isFile() && /\.(png|jpg|jpeg|webp)$/i.test(sub.name)) {
const candidate = path.join(fullEntry, sub.name);
const stat = fs.statSync(candidate);
if (stat.mtimeMs >= startedAtMs - 2000 && stat.mtimeMs > newestMtime) {
newestMtime = stat.mtimeMs;
newestFile = candidate;
}
}
}
} else if (entry.isFile() && /\.(png|jpg|jpeg|webp)$/i.test(entry.name)) {
const stat = fs.statSync(fullEntry);
if (stat.mtimeMs >= startedAtMs - 2000 && stat.mtimeMs > newestMtime) {
newestMtime = stat.mtimeMs;
newestFile = fullEntry;
}
}
}
return newestFile;
}
/**
* Generates or edits a high-resolution raster PNG mockup using OpenAI Codex CLI
* (`codex exec -m gpt-5.6-sol`) and its built-in `image_gen` tool.
*/
export async function generateWithCodex({
prompt,
referenceImagePath = null,
outputDir
}) {
const codexHome = process.env.CODEX_HOME || path.join(os.homedir(), ".codex");
const generatedImagesRoot = path.join(codexHome, "generated_images");
const startedAtMs = Date.now();
const instruction = referenceImagePath && fs.existsSync(referenceImagePath)
? `Use the built-in image_gen tool right now to edit the attached reference image according to this instruction: ${prompt}. Do not run any shell commands, just call image_gen and reply DONE.`
: `Use the built-in image_gen tool right now to generate this image: ${prompt}. Do not run any shell commands, just call image_gen and reply DONE.`;
const args = [
"exec",
"-m",
process.env.CODEX_MODEL || "gpt-5.6-sol",
"--skip-git-repo-check",
"--json"
];
if (referenceImagePath && fs.existsSync(referenceImagePath) && !referenceImagePath.endsWith(".svg")) {
args.push("-i", referenceImagePath);
}
args.push(instruction);
const { stdout, stderr, exitCode } = await new Promise((resolve, reject) => {
const child = spawn("codex", args, {
stdio: ["ignore", "pipe", "pipe"],
env: { ...process.env }
});
let out = "";
let err = "";
child.stdout.on("data", (d) => {
out += d.toString();
});
child.stderr.on("data", (d) => {
err += d.toString();
});
child.on("error", reject);
child.on("close", (code) => {
resolve({ stdout: out, stderr: err, exitCode: code });
});
});
// Extract thread_id from JSONL stdout
let threadId = null;
let lastAgentText = null;
for (const line of stdout.split("\n")) {
const trimmed = line.trim();
if (!trimmed.startsWith("{")) continue;
try {
const evt = JSON.parse(trimmed);
if (evt.type === "thread.started" && evt.thread_id) {
threadId = evt.thread_id;
} else if (evt.type === "item.completed" && evt.item?.type === "agent_message") {
lastAgentText = evt.item.text;
}
} catch {
// ignore non-json lines
}
}
let sourceImagePath = null;
if (threadId) {
const threadDir = path.join(generatedImagesRoot, threadId);
if (fs.existsSync(threadDir)) {
const files = fs
.readdirSync(threadDir)
.filter((f) => /\.(png|jpg|jpeg|webp)$/i.test(f))
.map((f) => path.join(threadDir, f));
if (files.length > 0) {
sourceImagePath = files[files.length - 1];
}
}
}
if (!sourceImagePath) {
sourceImagePath = findNewestGeneratedImage(generatedImagesRoot, startedAtMs);
}
if (!sourceImagePath || !fs.existsSync(sourceImagePath)) {
throw new Error(
`Codex completed (exit ${exitCode}) without saving an image. ${stderr || lastAgentText || ""}`.trim()
);
}
const ext = path.extname(sourceImagePath).replace(/^\./, "") || "png";
const filename = `codex-${Date.now()}-${Math.random().toString(36).slice(2, 7)}.${ext}`;
const fullPath = path.join(outputDir, filename);
fs.mkdirSync(outputDir, { recursive: true });
fs.copyFileSync(sourceImagePath, fullPath);
return {
filename,
imageUrl: `/mockups/${filename}`,
mimeType: `image/${ext === "jpg" ? "jpeg" : ext}`,
modelUsed: "OpenAI Codex (gpt-5.6-sol / image_gen)",
interactionId: threadId,
textNotes: "Generated high-resolution raster PNG via OpenAI Codex built-in image_gen."
};
}
function extractSvgBlock(text) { function extractSvgBlock(text) {
if (!text) return null; if (!text) return null;
const match = String(text).match(/<svg[\s\S]*?<\/svg>/i); const match = String(text).match(/<svg[\s\S]*?<\/svg>/i);
@@ -124,7 +274,7 @@ async function generateFreeTierSvgMockup({
: "", : "",
"DESIGN RULES:", "DESIGN RULES:",
"1. Include a sleek studio backdrop (`#121621` radial gradient).", "1. Include a sleek studio backdrop (`#121621` radial gradient).",
"2. If the brief mentions a t-shirt, draw a realistic, well-proportioned crewneck t-shirt silhouette with collar ribbing, sleeve seams, subtle fabric shading, and the requested garment hex color. Place the custom screenprint illustration and vintage typography either full-front (`cx=400, cy=360`) or in the 5-inch left-chest pocket zone (`cx=490, cy=285`, scaled compactly) according to the brief.", "2. If the brief mentions a t-shirt, draw a realistic, well-proponed crewneck t-shirt silhouette with collar ribbing, sleeve seams, subtle fabric shading, and the requested garment hex color. Place the custom screenprint illustration and vintage typography either full-front (`cx=400, cy=360`) or in the 5-inch left-chest pocket zone (`cx=490, cy=285`, scaled compactly) according to the brief.",
"3. If the brief is a sticker, poster, or app UI, draw the realistic die-cut border, gallery frame, or device frame accordingly.", "3. If the brief is a sticker, poster, or app UI, draw the realistic die-cut border, gallery frame, or device frame accordingly.",
"4. Draw rich, multi-layered custom vector illustration paths, badges, emblems, stars/mountains/icons, and bold screenprint typography with high contrast against the garment/surface color.", "4. Draw rich, multi-layered custom vector illustration paths, badges, emblems, stars/mountains/icons, and bold screenprint typography with high contrast against the garment/surface color.",
"5. Return ONLY the raw `<svg xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"0 0 800 800\">...</svg>` code block with no markdown explanation." "5. Return ONLY the raw `<svg xmlns=\"http://www.w3.org/2000/svg\" viewBox=\"0 0 800 800\">...</svg>` code block with no markdown explanation."
@@ -153,29 +303,44 @@ async function generateFreeTierSvgMockup({
mimeType: "image/svg+xml", mimeType: "image/svg+xml",
modelUsed: "gemini-3.1-flash-lite (Free-Tier Vector Mockup)", modelUsed: "gemini-3.1-flash-lite (Free-Tier Vector Mockup)",
interactionId: interaction.id || null, interactionId: interaction.id || null,
textNotes: "Generated via Gemini 3.1 Flash-Lite Vector Studio (Free Tier key detected; enable Tier 1 billing in Google AI Studio for raster Nano Banana 2.1)." textNotes: "Generated via Gemini 3.1 Flash-Lite Vector Studio."
}; };
} }
/** /**
* Generates or edits an image using Google Gemini Nano Banana (`gemini-nano-banana-2.1`) * Generates or edits an image using:
* via the official `@google/genai` Interactions API (`ai.interactions.create`), * 1. OpenAI Codex (`codex-image-gen`) built-in `image_gen` tool (default when selected or when Codex auth is available), OR
* with automatic fallback to `gemini-3.1-flash-lite` vector mockup generation when * 2. Google Gemini Nano Banana (`gemini-nano-banana-2.1`) with automatic fallback to `Codex` or `gemini-3.1-flash-lite` SVG when Free Tier quota is 0.
* a Free Tier key hits `limit: 0`.
*/ */
export async function generateWithNanoBanana({ export async function generateWithNanoBanana({
apiKey, apiKey,
model = "gemini-nano-banana-2.1", model = "codex-image-gen",
prompt, prompt,
referenceImagePath = null, referenceImagePath = null,
referenceImageBase64 = null, referenceImageBase64 = null,
referenceMimeType = "image/png", referenceMimeType = "image/png",
outputDir outputDir
}) { }) {
// 1. Primary path: OpenAI Codex built-in image_gen
if (model === "codex-image-gen" || (!apiKey && isCodexConfigured())) {
return await generateWithCodex({
prompt,
referenceImagePath,
outputDir
});
}
const resolvedKey = apiKey || process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY; const resolvedKey = apiKey || process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY;
if (!resolvedKey) { if (!resolvedKey) {
if (isCodexConfigured()) {
return await generateWithCodex({
prompt,
referenceImagePath,
outputDir
});
}
const err = new Error( const err = new Error(
"No Gemini API key configured. Click '🍌 Nano Banana' in the top bar (or set GEMINI_API_KEY) to generate live mockups, or use the Interactive T-Shirt Canvas Compositor." "No AI image provider configured. Select 'OpenAI Codex (image_gen)' or configure a Gemini API key in Settings."
); );
err.code = "MISSING_API_KEY"; err.code = "MISSING_API_KEY";
throw err; throw err;
@@ -242,8 +407,15 @@ export async function generateWithNanoBanana({
}; };
} catch (err) { } catch (err) {
const msg = String(err?.message || ""); const msg = String(err?.message || "");
// If Free Tier quota is 0 for native raster image models, seamlessly fall back to Gemini 3.1 Flash-Lite SVG Vector Mockup!
if (msg.includes("429") || msg.includes("Free Tier") || msg.includes("limit: 0") || msg.includes("quota")) { if (msg.includes("429") || msg.includes("Free Tier") || msg.includes("limit: 0") || msg.includes("quota")) {
// If OpenAI Codex is authenticated on this host, prefer high-res raster generation via Codex!
if (isCodexConfigured()) {
return await generateWithCodex({
prompt,
referenceImagePath,
outputDir
});
}
return await generateFreeTierSvgMockup({ return await generateFreeTierSvgMockup({
ai, ai,
prompt, prompt,