From 8a25771839e2496c80bece8e6b660db5a0d6c1b5 Mon Sep 17 00:00:00 2001 From: airent106 Date: Thu, 29 Oct 2026 11:33:33 -0600 Subject: [PATCH] feat(mockup): add OpenAI Codex (gpt-5.6-sol + built-in image_gen) raster mockup generator --- Dockerfile | 2 + docker-compose.yml | 1 + public/app.js | 2 +- public/index.html | 7 +- server/index.js | 31 +++++-- server/nano_banana.js | 196 +++++++++++++++++++++++++++++++++++++++--- 6 files changed, 215 insertions(+), 24 deletions(-) diff --git a/Dockerfile b/Dockerfile index 24ebf47..afe9d41 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,8 @@ ENV NODE_ENV=production ENV PORT=80 ENV DATA_DIR=/app/store +RUN npm install -g @openai/codex@latest + COPY package*.json ./ RUN npm install --omit=dev diff --git a/docker-compose.yml b/docker-compose.yml index 6661b7c..bd4295c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ services: restart: unless-stopped volumes: - /home/chrisf/docker/appdata/sparkvault/data:/app/store + - /home/chrisf/.codex:/root/.codex networks: - t3_proxy ports: diff --git a/public/app.js b/public/app.js index aac1394..31c32e0 100644 --- a/public/app.js +++ b/public/app.js @@ -1803,7 +1803,7 @@ function bindGlobalEvents() { const editInstruction = document.getElementById("nb-edit-instruction-input").value.trim(); btn.disabled = true; - statusLabel.textContent = "🍌 Calling Nano Banana 2.1 (gemini-nano-banana-2.1)..."; + statusLabel.textContent = "🎨 Generating high-res AI mockup via OpenAI Codex (takes ~15–25s)..."; try { const res = await apiFetch(`/api/ideas/${state.activeIdeaId}/mockups/nano-banana`, { diff --git a/public/index.html b/public/index.html index 09a85c7..2f0be37 100644 --- a/public/index.html +++ b/public/index.html @@ -534,7 +534,7 @@
@@ -684,9 +684,10 @@
- + diff --git a/server/index.js b/server/index.js index 0729588..6edbe4f 100644 --- a/server/index.js +++ b/server/index.js @@ -14,7 +14,8 @@ import { NANO_BANANA_MODELS, SURFACE_PRESETS, buildNanoBananaPrompt, - generateWithNanoBanana + generateWithNanoBanana, + isCodexConfigured } from "./nano_banana.js"; const __filename = fileURLToPath(import.meta.url); @@ -328,7 +329,7 @@ export function createAppServer({ }); const storedKey = store.getSetting("gemini_api_key", ""); - const selectedModel = body.model || store.getSetting("nano_banana_model", "gemini-nano-banana-2.1"); + const selectedModel = body.model || store.getSetting("nano_banana_model", "codex-image-gen"); try { const genResult = await generateWithNanoBanana({ @@ -479,12 +480,24 @@ export function createAppServer({ const savedKey = store.getSetting("gemini_api_key", ""); const envKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || ""; const activeKey = savedKey || envKey; - const model = store.getSetting("nano_banana_model", "gemini-nano-banana-2.1"); + const codexReady = isCodexConfigured(); + const model = store.getSetting("nano_banana_model", "codex-image-gen"); return sendJson(res, 200, { - hasApiKey: Boolean(activeKey), - apiKeySource: savedKey ? "settings" : envKey ? "environment" : "none", - maskedKey: activeKey ? `${activeKey.slice(0, 4)}...${activeKey.slice(-4)}` : "", + hasApiKey: Boolean(activeKey) || codexReady, + codexReady, + apiKeySource: codexReady + ? "OpenAI Codex (OAuth)" + : savedKey + ? "settings" + : envKey + ? "environment" + : "none", + maskedKey: activeKey + ? `${activeKey.slice(0, 4)}...${activeKey.slice(-4)}` + : codexReady + ? "Authenticated via ~/.codex/auth.json" + : "", nanoBananaModel: model, availableModels: NANO_BANANA_MODELS, surfacePresets: Object.entries(SURFACE_PRESETS).map(([id, val]) => ({ @@ -504,10 +517,12 @@ export function createAppServer({ const savedKey = store.getSetting("gemini_api_key", ""); const envKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY || ""; const activeKey = savedKey || envKey; + const codexReady = isCodexConfigured(); return sendJson(res, 200, { saved: true, - hasApiKey: Boolean(activeKey), - nanoBananaModel: store.getSetting("nano_banana_model", "gemini-nano-banana-2.1") + hasApiKey: Boolean(activeKey) || codexReady, + codexReady, + nanoBananaModel: store.getSetting("nano_banana_model", "codex-image-gen") }); } } diff --git a/server/nano_banana.js b/server/nano_banana.js index 75a9ef1..103bcbb 100644 --- a/server/nano_banana.js +++ b/server/nano_banana.js @@ -1,21 +1,28 @@ import * as fs from "node:fs"; import * as path from "node:path"; +import * as os from "node:os"; +import { spawn } from "node:child_process"; import { GoogleGenAI } from "@google/genai"; export const NANO_BANANA_MODELS = [ + { + id: "codex-image-gen", + label: "OpenAI Codex (gpt-5.6-sol + Built-in image_gen) — Recommended", + description: "codex-image-gen — High-resolution photorealistic raster PNG product & apparel mockups via OpenAI Codex built-in image_gen" + }, { id: "gemini-nano-banana-2.1", - label: "Nano Banana 2.1 (Default High-Efficiency + Free-Tier Auto Fallback)", + label: "Nano Banana 2.1 (Google Gemini + Free-Tier Auto Fallback)", description: "gemini-nano-banana-2.1 — Uses native Nano Banana 2.1 on Tier 1 keys, with automatic Gemini 3.1 Flash-Lite Vector Mockup fallback on Free Tier keys" }, { id: "gemini-3-pro-image", - label: "Nano Banana Pro (Studio Quality)", + label: "Nano Banana Pro (Gemini Studio Quality)", description: "gemini-3-pro-image — High-detail studio mockup generation and multi-turn editing" }, { id: "gemini-3.1-flash-lite-image", - label: "Nano Banana 2 Lite (Ultra-Fast)", + label: "Nano Banana 2 Lite (Gemini Ultra-Fast)", description: "gemini-3.1-flash-lite-image — Rapid concept sketching and quick iterations" } ]; @@ -88,6 +95,149 @@ export function buildNanoBananaPrompt({ return finalPrompt; } +export function isCodexConfigured() { + const codexHome = process.env.CODEX_HOME || path.join(os.homedir(), ".codex"); + return fs.existsSync(path.join(codexHome, "auth.json")); +} + +function findNewestGeneratedImage(generatedImagesRoot, startedAtMs) { + if (!fs.existsSync(generatedImagesRoot)) return null; + let newestFile = null; + let newestMtime = 0; + + for (const entry of fs.readdirSync(generatedImagesRoot, { withFileTypes: true })) { + const fullEntry = path.join(generatedImagesRoot, entry.name); + if (entry.isDirectory()) { + for (const sub of fs.readdirSync(fullEntry, { withFileTypes: true })) { + if (sub.isFile() && /\.(png|jpg|jpeg|webp)$/i.test(sub.name)) { + const candidate = path.join(fullEntry, sub.name); + const stat = fs.statSync(candidate); + if (stat.mtimeMs >= startedAtMs - 2000 && stat.mtimeMs > newestMtime) { + newestMtime = stat.mtimeMs; + newestFile = candidate; + } + } + } + } else if (entry.isFile() && /\.(png|jpg|jpeg|webp)$/i.test(entry.name)) { + const stat = fs.statSync(fullEntry); + if (stat.mtimeMs >= startedAtMs - 2000 && stat.mtimeMs > newestMtime) { + newestMtime = stat.mtimeMs; + newestFile = fullEntry; + } + } + } + return newestFile; +} + +/** + * Generates or edits a high-resolution raster PNG mockup using OpenAI Codex CLI + * (`codex exec -m gpt-5.6-sol`) and its built-in `image_gen` tool. + */ +export async function generateWithCodex({ + prompt, + referenceImagePath = null, + outputDir +}) { + const codexHome = process.env.CODEX_HOME || path.join(os.homedir(), ".codex"); + const generatedImagesRoot = path.join(codexHome, "generated_images"); + const startedAtMs = Date.now(); + + const instruction = referenceImagePath && fs.existsSync(referenceImagePath) + ? `Use the built-in image_gen tool right now to edit the attached reference image according to this instruction: ${prompt}. Do not run any shell commands, just call image_gen and reply DONE.` + : `Use the built-in image_gen tool right now to generate this image: ${prompt}. Do not run any shell commands, just call image_gen and reply DONE.`; + + const args = [ + "exec", + "-m", + process.env.CODEX_MODEL || "gpt-5.6-sol", + "--skip-git-repo-check", + "--json" + ]; + + if (referenceImagePath && fs.existsSync(referenceImagePath) && !referenceImagePath.endsWith(".svg")) { + args.push("-i", referenceImagePath); + } + args.push(instruction); + + const { stdout, stderr, exitCode } = await new Promise((resolve, reject) => { + const child = spawn("codex", args, { + stdio: ["ignore", "pipe", "pipe"], + env: { ...process.env } + }); + + let out = ""; + let err = ""; + child.stdout.on("data", (d) => { + out += d.toString(); + }); + child.stderr.on("data", (d) => { + err += d.toString(); + }); + child.on("error", reject); + child.on("close", (code) => { + resolve({ stdout: out, stderr: err, exitCode: code }); + }); + }); + + // Extract thread_id from JSONL stdout + let threadId = null; + let lastAgentText = null; + for (const line of stdout.split("\n")) { + const trimmed = line.trim(); + if (!trimmed.startsWith("{")) continue; + try { + const evt = JSON.parse(trimmed); + if (evt.type === "thread.started" && evt.thread_id) { + threadId = evt.thread_id; + } else if (evt.type === "item.completed" && evt.item?.type === "agent_message") { + lastAgentText = evt.item.text; + } + } catch { + // ignore non-json lines + } + } + + let sourceImagePath = null; + if (threadId) { + const threadDir = path.join(generatedImagesRoot, threadId); + if (fs.existsSync(threadDir)) { + const files = fs + .readdirSync(threadDir) + .filter((f) => /\.(png|jpg|jpeg|webp)$/i.test(f)) + .map((f) => path.join(threadDir, f)); + if (files.length > 0) { + sourceImagePath = files[files.length - 1]; + } + } + } + + if (!sourceImagePath) { + sourceImagePath = findNewestGeneratedImage(generatedImagesRoot, startedAtMs); + } + + if (!sourceImagePath || !fs.existsSync(sourceImagePath)) { + throw new Error( + `Codex completed (exit ${exitCode}) without saving an image. ${stderr || lastAgentText || ""}`.trim() + ); + } + + const ext = path.extname(sourceImagePath).replace(/^\./, "") || "png"; + const filename = `codex-${Date.now()}-${Math.random().toString(36).slice(2, 7)}.${ext}`; + const fullPath = path.join(outputDir, filename); + + fs.mkdirSync(outputDir, { recursive: true }); + fs.copyFileSync(sourceImagePath, fullPath); + + return { + filename, + imageUrl: `/mockups/${filename}`, + mimeType: `image/${ext === "jpg" ? "jpeg" : ext}`, + modelUsed: "OpenAI Codex (gpt-5.6-sol / image_gen)", + interactionId: threadId, + textNotes: "Generated high-resolution raster PNG via OpenAI Codex built-in image_gen." + }; +} + function extractSvgBlock(text) { if (!text) return null; const match = String(text).match(//i); @@ -124,7 +274,7 @@ async function generateFreeTierSvgMockup({ : "", "DESIGN RULES:", "1. Include a sleek studio backdrop (`#121621` radial gradient).", - "2. If the brief mentions a t-shirt, draw a realistic, well-proportioned crewneck t-shirt silhouette with collar ribbing, sleeve seams, subtle fabric shading, and the requested garment hex color. Place the custom screenprint illustration and vintage typography either full-front (`cx=400, cy=360`) or in the 5-inch left-chest pocket zone (`cx=490, cy=285`, scaled compactly) according to the brief.", + "2. If the brief mentions a t-shirt, draw a realistic, well-proponed crewneck t-shirt silhouette with collar ribbing, sleeve seams, subtle fabric shading, and the requested garment hex color. Place the custom screenprint illustration and vintage typography either full-front (`cx=400, cy=360`) or in the 5-inch left-chest pocket zone (`cx=490, cy=285`, scaled compactly) according to the brief.", "3. If the brief is a sticker, poster, or app UI, draw the realistic die-cut border, gallery frame, or device frame accordingly.", "4. Draw rich, multi-layered custom vector illustration paths, badges, emblems, stars/mountains/icons, and bold screenprint typography with high contrast against the garment/surface color.", "5. Return ONLY the raw `...` code block with no markdown explanation." @@ -153,29 +303,44 @@ async function generateFreeTierSvgMockup({ mimeType: "image/svg+xml", modelUsed: "gemini-3.1-flash-lite (Free-Tier Vector Mockup)", interactionId: interaction.id || null, - textNotes: "Generated via Gemini 3.1 Flash-Lite Vector Studio (Free Tier key detected; enable Tier 1 billing in Google AI Studio for raster Nano Banana 2.1)." + textNotes: "Generated via Gemini 3.1 Flash-Lite Vector Studio." }; } /** - * Generates or edits an image using Google Gemini Nano Banana (`gemini-nano-banana-2.1`) - * via the official `@google/genai` Interactions API (`ai.interactions.create`), - * with automatic fallback to `gemini-3.1-flash-lite` vector mockup generation when - * a Free Tier key hits `limit: 0`. + * Generates or edits an image using: + * 1. OpenAI Codex (`codex-image-gen`) built-in `image_gen` tool (default when selected or when Codex auth is available), OR + * 2. Google Gemini Nano Banana (`gemini-nano-banana-2.1`) with automatic fallback to `Codex` or `gemini-3.1-flash-lite` SVG when Free Tier quota is 0. */ export async function generateWithNanoBanana({ apiKey, - model = "gemini-nano-banana-2.1", + model = "codex-image-gen", prompt, referenceImagePath = null, referenceImageBase64 = null, referenceMimeType = "image/png", outputDir }) { + // 1. Primary path: OpenAI Codex built-in image_gen + if (model === "codex-image-gen" || (!apiKey && isCodexConfigured())) { + return await generateWithCodex({ + prompt, + referenceImagePath, + outputDir + }); + } + const resolvedKey = apiKey || process.env.GEMINI_API_KEY || process.env.GOOGLE_API_KEY; if (!resolvedKey) { + if (isCodexConfigured()) { + return await generateWithCodex({ + prompt, + referenceImagePath, + outputDir + }); + } const err = new Error( - "No Gemini API key configured. Click '🍌 Nano Banana' in the top bar (or set GEMINI_API_KEY) to generate live mockups, or use the Interactive T-Shirt Canvas Compositor." + "No AI image provider configured. Select 'OpenAI Codex (image_gen)' or configure a Gemini API key in Settings." ); err.code = "MISSING_API_KEY"; throw err; @@ -242,8 +407,15 @@ export async function generateWithNanoBanana({ }; } catch (err) { const msg = String(err?.message || ""); - // If Free Tier quota is 0 for native raster image models, seamlessly fall back to Gemini 3.1 Flash-Lite SVG Vector Mockup! if (msg.includes("429") || msg.includes("Free Tier") || msg.includes("limit: 0") || msg.includes("quota")) { + // If OpenAI Codex is authenticated on this host, prefer high-res raster generation via Codex! + if (isCodexConfigured()) { + return await generateWithCodex({ + prompt, + referenceImagePath, + outputDir + }); + } return await generateFreeTierSvgMockup({ ai, prompt,