Rewrite validation step part 3

This commit is contained in:
2026-01-19 01:02:20 -05:00
parent d15360a7d4
commit 9ce84fe5b9
14 changed files with 1310 additions and 239 deletions

View File

@@ -960,7 +960,7 @@ router.post("/validate", async (req, res) => {
// - max_output_tokens: 20000 ensures space for large product batches
// Note: Responses API is the recommended endpoint for GPT-5 models
const completion = await createResponsesCompletion({
model: "gpt-5",
model: "gpt-5.2",
input: [
{
role: "developer",
@@ -978,7 +978,7 @@ router.post("/validate", async (req, res) => {
verbosity: "medium",
format: AI_VALIDATION_TEXT_FORMAT,
},
max_output_tokens: 20000,
max_output_tokens: 50000,
});
console.log("✅ Received response from OpenAI Responses API");
@@ -1480,6 +1480,7 @@ function normalizeJsonResponse(text) {
if (!text || typeof text !== 'string') return text;
let cleaned = text.trim();
// Remove markdown code fences if present
if (cleaned.startsWith('```')) {
const firstLineBreak = cleaned.indexOf('\n');
if (firstLineBreak !== -1) {
@@ -1496,5 +1497,86 @@ function normalizeJsonResponse(text) {
cleaned = cleaned.trim();
}
// Attempt to repair truncated JSON
// This handles cases where the AI response was cut off mid-response
cleaned = repairTruncatedJson(cleaned);
return cleaned;
}
/**
* Attempt to repair truncated JSON by adding missing closing brackets/braces
* This is a common issue when AI responses hit token limits
*/
function repairTruncatedJson(text) {
if (!text || typeof text !== 'string') return text;
// First, try parsing as-is
try {
JSON.parse(text);
return text; // Valid JSON, no repair needed
} catch (e) {
// JSON is invalid, try to repair
}
let repaired = text.trim();
// Count opening and closing brackets/braces
let braceCount = 0; // {}
let bracketCount = 0; // []
let inString = false;
let escapeNext = false;
for (let i = 0; i < repaired.length; i++) {
const char = repaired[i];
if (escapeNext) {
escapeNext = false;
continue;
}
if (char === '\\' && inString) {
escapeNext = true;
continue;
}
if (char === '"') {
inString = !inString;
continue;
}
if (!inString) {
if (char === '{') braceCount++;
else if (char === '}') braceCount--;
else if (char === '[') bracketCount++;
else if (char === ']') bracketCount--;
}
}
// If we're still inside a string, close it
if (inString) {
repaired += '"';
}
// Add missing closing brackets and braces
// Close arrays first, then objects (reverse of typical nesting)
while (bracketCount > 0) {
repaired += ']';
bracketCount--;
}
while (braceCount > 0) {
repaired += '}';
braceCount--;
}
// Try parsing the repaired JSON
try {
JSON.parse(repaired);
console.log('✅ Successfully repaired truncated JSON');
return repaired;
} catch (e) {
// Repair failed, return original and let the caller handle the error
console.log('⚠️ JSON repair attempt failed:', e.message);
return text;
}
}