Remove fallback and hardcoded prompts to rely on database prompts only for AI
This commit is contained in:
@@ -353,15 +353,14 @@ async function generateDebugResponse(productsToUse, res) {
|
||||
WHERE prompt_type = 'system'
|
||||
`);
|
||||
|
||||
// Get system prompt or use default
|
||||
let systemPrompt = null;
|
||||
if (systemPromptResult.rows.length > 0) {
|
||||
systemPrompt = systemPromptResult.rows[0];
|
||||
console.log("📝 Loaded system prompt from database, ID:", systemPrompt.id);
|
||||
} else {
|
||||
console.warn("⚠️ No system prompt found in database, will use default");
|
||||
if (systemPromptResult.rows.length === 0) {
|
||||
console.error("❌ No system prompt found in database");
|
||||
throw new Error("No system prompt found in database");
|
||||
}
|
||||
|
||||
const systemPrompt = systemPromptResult.rows[0];
|
||||
console.log("📝 Loaded system prompt from database, ID:", systemPrompt.id);
|
||||
|
||||
// Then, fetch the general prompt using the consolidated endpoint approach
|
||||
const generalPromptResult = await pool.query(`
|
||||
SELECT * FROM ai_prompts
|
||||
@@ -369,7 +368,7 @@ async function generateDebugResponse(productsToUse, res) {
|
||||
`);
|
||||
|
||||
if (generalPromptResult.rows.length === 0) {
|
||||
console.warn("⚠️ No general prompt found in database");
|
||||
console.error("❌ No general prompt found in database");
|
||||
throw new Error("No general prompt found in database");
|
||||
}
|
||||
|
||||
@@ -481,22 +480,15 @@ async function generateDebugResponse(productsToUse, res) {
|
||||
: null,
|
||||
}
|
||||
: null,
|
||||
basePrompt: systemPrompt ? systemPrompt.prompt_text + "\n\n" + generalPrompt.prompt_text : generalPrompt.prompt_text,
|
||||
basePrompt: systemPrompt.prompt_text + "\n\n" + generalPrompt.prompt_text,
|
||||
sampleFullPrompt: fullUserPrompt,
|
||||
promptLength: promptLength,
|
||||
apiFormat: apiMessages,
|
||||
promptSources: {
|
||||
...(systemPrompt ? {
|
||||
systemPrompt: {
|
||||
id: systemPrompt.id,
|
||||
prompt_text: systemPrompt.prompt_text
|
||||
}
|
||||
} : {
|
||||
systemPrompt: {
|
||||
id: 0,
|
||||
prompt_text: `You are a specialized e-commerce product data processor for a crafting supplies website tasked with providing complete, correct, appealing, and SEO-friendly product listings. You should write professionally, but in a friendly and engaging tone. You have meticulous attention to detail and are a master at your craft.`
|
||||
}
|
||||
}),
|
||||
systemPrompt: {
|
||||
id: systemPrompt.id,
|
||||
prompt_text: systemPrompt.prompt_text
|
||||
},
|
||||
generalPrompt: {
|
||||
id: generalPrompt.id,
|
||||
prompt_text: generalPrompt.prompt_text
|
||||
@@ -702,17 +694,14 @@ async function loadPrompt(connection, productsToValidate = null, appPool = null)
|
||||
WHERE prompt_type = 'system'
|
||||
`);
|
||||
|
||||
// Default system instructions in case the system prompt is not found
|
||||
let systemInstructions = `You are a specialized e-commerce product data processor for a crafting supplies website tasked with providing complete, correct, appealing, and SEO-friendly product listings. You should write professionally, but in a friendly and engaging tone. You have meticulous attention to detail and are a master at your craft.`;
|
||||
|
||||
// If system prompt exists in the database, use it
|
||||
if (systemPromptResult.rows.length > 0) {
|
||||
systemInstructions = systemPromptResult.rows[0].prompt_text;
|
||||
console.log("📝 Loaded system prompt from database");
|
||||
} else {
|
||||
console.warn("⚠️ No system prompt found in database, using default");
|
||||
if (systemPromptResult.rows.length === 0) {
|
||||
console.error("❌ No system prompt found in database");
|
||||
throw new Error("No system prompt found in database");
|
||||
}
|
||||
|
||||
const systemInstructions = systemPromptResult.rows[0].prompt_text;
|
||||
console.log("📝 Loaded system prompt from database");
|
||||
|
||||
// Fetch the general prompt using the consolidated endpoint approach
|
||||
const generalPromptResult = await pool.query(`
|
||||
SELECT * FROM ai_prompts
|
||||
@@ -720,7 +709,7 @@ async function loadPrompt(connection, productsToValidate = null, appPool = null)
|
||||
`);
|
||||
|
||||
if (generalPromptResult.rows.length === 0) {
|
||||
console.warn("⚠️ No general prompt found in database");
|
||||
console.error("❌ No general prompt found in database");
|
||||
throw new Error("No general prompt found in database");
|
||||
}
|
||||
|
||||
@@ -779,84 +768,87 @@ async function loadPrompt(connection, productsToValidate = null, appPool = null)
|
||||
combinedPrompt += "\n--- END COMPANY-SPECIFIC INSTRUCTIONS ---\n";
|
||||
}
|
||||
|
||||
// If we have products to validate, create a filtered prompt
|
||||
if (productsToValidate) {
|
||||
console.log("Creating filtered prompt for products:", productsToValidate);
|
||||
// Products are required for validation
|
||||
if (!productsToValidate || !Array.isArray(productsToValidate) || productsToValidate.length === 0) {
|
||||
throw new Error("Products are required for prompt generation");
|
||||
}
|
||||
|
||||
// Extract unique values from products for non-core attributes
|
||||
const uniqueValues = {
|
||||
supplierIds: new Set(),
|
||||
companyIds: new Set(),
|
||||
artistIds: new Set(),
|
||||
lineIds: new Set(),
|
||||
subLineIds: new Set(),
|
||||
};
|
||||
console.log("Creating filtered prompt for products:", productsToValidate);
|
||||
|
||||
// Collect any values that exist in the products
|
||||
productsToValidate.forEach((product) => {
|
||||
Object.entries(product).forEach(([key, value]) => {
|
||||
if (value === undefined || value === null) return;
|
||||
// Extract unique values from products for non-core attributes
|
||||
const uniqueValues = {
|
||||
supplierIds: new Set(),
|
||||
companyIds: new Set(),
|
||||
artistIds: new Set(),
|
||||
lineIds: new Set(),
|
||||
subLineIds: new Set(),
|
||||
};
|
||||
|
||||
// Map field names to their respective sets
|
||||
const fieldMap = {
|
||||
supplierid: "supplierIds",
|
||||
supplier: "supplierIds",
|
||||
company: "companyIds",
|
||||
artist: "artistIds",
|
||||
line: "lineIds",
|
||||
subline: "subLineIds",
|
||||
};
|
||||
// Collect any values that exist in the products
|
||||
productsToValidate.forEach((product) => {
|
||||
Object.entries(product).forEach(([key, value]) => {
|
||||
if (value === undefined || value === null) return;
|
||||
|
||||
if (fieldMap[key]) {
|
||||
uniqueValues[fieldMap[key]].add(Number(value));
|
||||
}
|
||||
});
|
||||
// Map field names to their respective sets
|
||||
const fieldMap = {
|
||||
supplierid: "supplierIds",
|
||||
supplier: "supplierIds",
|
||||
company: "companyIds",
|
||||
artist: "artistIds",
|
||||
line: "lineIds",
|
||||
subline: "subLineIds",
|
||||
};
|
||||
|
||||
if (fieldMap[key]) {
|
||||
uniqueValues[fieldMap[key]].add(Number(value));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
console.log("Unique values collected:", {
|
||||
suppliers: Array.from(uniqueValues.supplierIds),
|
||||
companies: Array.from(uniqueValues.companyIds),
|
||||
artists: Array.from(uniqueValues.artistIds),
|
||||
lines: Array.from(uniqueValues.lineIds),
|
||||
subLines: Array.from(uniqueValues.subLineIds),
|
||||
});
|
||||
console.log("Unique values collected:", {
|
||||
suppliers: Array.from(uniqueValues.supplierIds),
|
||||
companies: Array.from(uniqueValues.companyIds),
|
||||
artists: Array.from(uniqueValues.artistIds),
|
||||
lines: Array.from(uniqueValues.lineIds),
|
||||
subLines: Array.from(uniqueValues.subLineIds),
|
||||
});
|
||||
|
||||
// Create mixed taxonomy with filtered non-core data and full core data
|
||||
const mixedTaxonomy = {
|
||||
// Keep full data for core attributes
|
||||
categories: taxonomy.categories,
|
||||
themes: taxonomy.themes,
|
||||
colors: taxonomy.colors,
|
||||
taxCodes: taxonomy.taxCodes,
|
||||
sizeCategories: taxonomy.sizeCategories,
|
||||
// For non-core data, only include items that are actually used
|
||||
suppliers: taxonomy.suppliers.filter(([id]) =>
|
||||
uniqueValues.supplierIds.has(Number(id))
|
||||
),
|
||||
companies: taxonomy.companies.filter(([id]) =>
|
||||
uniqueValues.companyIds.has(Number(id))
|
||||
),
|
||||
artists: taxonomy.artists.filter(([id]) =>
|
||||
uniqueValues.artistIds.has(Number(id))
|
||||
),
|
||||
lines: taxonomy.lines.filter(([id]) =>
|
||||
uniqueValues.lineIds.has(Number(id))
|
||||
),
|
||||
subLines: taxonomy.subLines.filter(([id]) =>
|
||||
uniqueValues.subLineIds.has(Number(id))
|
||||
),
|
||||
};
|
||||
// Create mixed taxonomy with filtered non-core data and full core data
|
||||
const mixedTaxonomy = {
|
||||
// Keep full data for core attributes
|
||||
categories: taxonomy.categories,
|
||||
themes: taxonomy.themes,
|
||||
colors: taxonomy.colors,
|
||||
taxCodes: taxonomy.taxCodes,
|
||||
sizeCategories: taxonomy.sizeCategories,
|
||||
// For non-core data, only include items that are actually used
|
||||
suppliers: taxonomy.suppliers.filter(([id]) =>
|
||||
uniqueValues.supplierIds.has(Number(id))
|
||||
),
|
||||
companies: taxonomy.companies.filter(([id]) =>
|
||||
uniqueValues.companyIds.has(Number(id))
|
||||
),
|
||||
artists: taxonomy.artists.filter(([id]) =>
|
||||
uniqueValues.artistIds.has(Number(id))
|
||||
),
|
||||
lines: taxonomy.lines.filter(([id]) =>
|
||||
uniqueValues.lineIds.has(Number(id))
|
||||
),
|
||||
subLines: taxonomy.subLines.filter(([id]) =>
|
||||
uniqueValues.subLineIds.has(Number(id))
|
||||
),
|
||||
};
|
||||
|
||||
console.log("Filtered taxonomy counts:", {
|
||||
suppliers: mixedTaxonomy.suppliers.length,
|
||||
companies: mixedTaxonomy.companies.length,
|
||||
artists: mixedTaxonomy.artists.length,
|
||||
lines: mixedTaxonomy.lines.length,
|
||||
subLines: mixedTaxonomy.subLines.length,
|
||||
});
|
||||
console.log("Filtered taxonomy counts:", {
|
||||
suppliers: mixedTaxonomy.suppliers.length,
|
||||
companies: mixedTaxonomy.companies.length,
|
||||
artists: mixedTaxonomy.artists.length,
|
||||
lines: mixedTaxonomy.lines.length,
|
||||
subLines: mixedTaxonomy.subLines.length,
|
||||
});
|
||||
|
||||
// Format taxonomy data for the prompt, only including sections with values
|
||||
const taxonomySection = `
|
||||
// Format taxonomy data for the prompt, only including sections with values
|
||||
const taxonomySection = `
|
||||
All Available Categories:
|
||||
${JSON.stringify(mixedTaxonomy.categories)}
|
||||
|
||||
@@ -871,73 +863,38 @@ ${JSON.stringify(mixedTaxonomy.taxCodes)}
|
||||
|
||||
All Available Size Categories:
|
||||
${JSON.stringify(mixedTaxonomy.sizeCategories)}${
|
||||
mixedTaxonomy.suppliers.length
|
||||
? `\n\nSuppliers Used In This Data:\n${JSON.stringify(
|
||||
mixedTaxonomy.suppliers
|
||||
)}`
|
||||
: ""
|
||||
}${
|
||||
mixedTaxonomy.companies.length
|
||||
? `\n\nCompanies Used In This Data:\n${JSON.stringify(
|
||||
mixedTaxonomy.companies
|
||||
)}`
|
||||
: ""
|
||||
}${
|
||||
mixedTaxonomy.artists.length
|
||||
? `\n\nArtists Used In This Data:\n${JSON.stringify(
|
||||
mixedTaxonomy.artists
|
||||
)}`
|
||||
: ""
|
||||
}${
|
||||
mixedTaxonomy.lines.length
|
||||
? `\n\nLines Used In This Data:\n${JSON.stringify(
|
||||
mixedTaxonomy.lines
|
||||
)}`
|
||||
: ""
|
||||
}${
|
||||
mixedTaxonomy.subLines.length
|
||||
? `\n\nSub-Lines Used In This Data:\n${JSON.stringify(
|
||||
mixedTaxonomy.subLines
|
||||
)}`
|
||||
: ""
|
||||
}
|
||||
|
||||
----------Here is the product data to validate----------`;
|
||||
|
||||
// Return both system instructions and user content separately
|
||||
return {
|
||||
systemInstructions,
|
||||
userContent: combinedPrompt + "\n" + taxonomySection
|
||||
};
|
||||
mixedTaxonomy.suppliers.length
|
||||
? `\n\nSuppliers Used In This Data:\n${JSON.stringify(
|
||||
mixedTaxonomy.suppliers
|
||||
)}`
|
||||
: ""
|
||||
}${
|
||||
mixedTaxonomy.companies.length
|
||||
? `\n\nCompanies Used In This Data:\n${JSON.stringify(
|
||||
mixedTaxonomy.companies
|
||||
)}`
|
||||
: ""
|
||||
}${
|
||||
mixedTaxonomy.artists.length
|
||||
? `\n\nArtists Used In This Data:\n${JSON.stringify(
|
||||
mixedTaxonomy.artists
|
||||
)}`
|
||||
: ""
|
||||
}${
|
||||
mixedTaxonomy.lines.length
|
||||
? `\n\nLines Used In This Data:\n${JSON.stringify(
|
||||
mixedTaxonomy.lines
|
||||
)}`
|
||||
: ""
|
||||
}${
|
||||
mixedTaxonomy.subLines.length
|
||||
? `\n\nSub-Lines Used In This Data:\n${JSON.stringify(
|
||||
mixedTaxonomy.subLines
|
||||
)}`
|
||||
: ""
|
||||
}
|
||||
|
||||
// Generate the full unfiltered prompt for taxonomy section
|
||||
const taxonomySection = `
|
||||
Available Categories:
|
||||
${JSON.stringify(taxonomy.categories)}
|
||||
|
||||
Available Themes:
|
||||
${JSON.stringify(taxonomy.themes)}
|
||||
|
||||
Available Colors:
|
||||
${JSON.stringify(taxonomy.colors)}
|
||||
|
||||
Available Tax Codes:
|
||||
${JSON.stringify(taxonomy.taxCodes)}
|
||||
|
||||
Available Size Categories:
|
||||
${JSON.stringify(taxonomy.sizeCategories)}
|
||||
|
||||
Available Suppliers:
|
||||
${JSON.stringify(taxonomy.suppliers)}
|
||||
|
||||
Available Companies:
|
||||
${JSON.stringify(taxonomy.companies)}
|
||||
|
||||
Available Artists:
|
||||
${JSON.stringify(taxonomy.artists)}
|
||||
|
||||
Here is the product data to validate:`;
|
||||
----------Here is the product data to validate----------`;
|
||||
|
||||
// Return both system instructions and user content separately
|
||||
return {
|
||||
@@ -1007,7 +964,7 @@ router.post("/validate", async (req, res) => {
|
||||
input: [
|
||||
{
|
||||
role: "developer",
|
||||
content: `${promptData.systemInstructions}\n\nYou MUST respond with a single valid JSON object containing the following top-level keys: correctedData, changes, warnings, summary, metadata.\n- correctedData: array of product objects reflecting the updated data.\n- changes: array of human-readable bullet points summarizing the nature of updates.\n- warnings: array of caveats or risks that still require review.\n- summary: a concise paragraph (<=75 words) describing overall data quality and improvements.\n- metadata: object containing any supplemental machine-readable information (optional fields allowed).\nDo NOT include Markdown code fences or any text outside the JSON object.`,
|
||||
content: promptData.systemInstructions,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
@@ -1242,26 +1199,15 @@ router.post("/validate", async (req, res) => {
|
||||
}));
|
||||
|
||||
// Set prompt sources
|
||||
if (generalPromptResult.rows.length > 0) {
|
||||
if (generalPromptResult.rows.length > 0 && systemPromptResult.rows.length > 0) {
|
||||
const generalPrompt = generalPromptResult.rows[0];
|
||||
let systemPrompt = null;
|
||||
|
||||
if (systemPromptResult.rows.length > 0) {
|
||||
systemPrompt = systemPromptResult.rows[0];
|
||||
}
|
||||
const systemPrompt = systemPromptResult.rows[0];
|
||||
|
||||
promptSources = {
|
||||
...(systemPrompt ? {
|
||||
systemPrompt: {
|
||||
id: systemPrompt.id,
|
||||
prompt_text: systemPrompt.prompt_text
|
||||
}
|
||||
} : {
|
||||
systemPrompt: {
|
||||
id: 0,
|
||||
prompt_text: `You are a specialized e-commerce product data processor for a crafting supplies website tasked with providing complete, correct, appealing, and SEO-friendly product listings. You should write professionally, but in a friendly and engaging tone. You have meticulous attention to detail and are a master at your craft.`
|
||||
}
|
||||
}),
|
||||
systemPrompt: {
|
||||
id: systemPrompt.id,
|
||||
prompt_text: systemPrompt.prompt_text
|
||||
},
|
||||
generalPrompt: {
|
||||
id: generalPrompt.id,
|
||||
prompt_text: generalPrompt.prompt_text
|
||||
|
||||
Reference in New Issue
Block a user