matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add newsletter recommendations

Browse Source
This commit is contained in:
Matt
2026-01-31 22:04:49 -05:00
parent 4372dc5e26
commit 450fd96e19
11 changed files with 1169 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
inventory-server/scripts/import-from-prod.js
View File

@@ -6,6 +6,7 @@ const importCategories = require('./import/categories');
const { importProducts } = require('./import/products');
const importOrders = require('./import/orders');
const importPurchaseOrders = require('./import/purchase-orders');
const importDailyDeals = require('./import/daily-deals');
dotenv.config({ path: path.join(__dirname, "../.env") });
@@ -14,6 +15,7 @@ const IMPORT_CATEGORIES = true;
const IMPORT_PRODUCTS = true;
const IMPORT_ORDERS = true;
const IMPORT_PURCHASE_ORDERS = true;
const IMPORT_DAILY_DEALS = true;
// Add flag for incremental updates
const INCREMENTAL_UPDATE = process.env.INCREMENTAL_UPDATE !== 'false'; // Default to true unless explicitly set to false
@@ -78,7 +80,8 @@ async function main() {
IMPORT_CATEGORIES,
IMPORT_PRODUCTS,
IMPORT_ORDERS,
IMPORT_PURCHASE_ORDERS
IMPORT_PURCHASE_ORDERS,
IMPORT_DAILY_DEALS
].filter(Boolean).length;
try {
@@ -126,10 +129,11 @@ async function main() {
'categories_enabled', $2::boolean,
'products_enabled', $3::boolean,
'orders_enabled', $4::boolean,
'purchase_orders_enabled', $5::boolean
'purchase_orders_enabled', $5::boolean,
'daily_deals_enabled', $6::boolean
)
) RETURNING id
`, [INCREMENTAL_UPDATE, IMPORT_CATEGORIES, IMPORT_PRODUCTS, IMPORT_ORDERS, IMPORT_PURCHASE_ORDERS]);
`, [INCREMENTAL_UPDATE, IMPORT_CATEGORIES, IMPORT_PRODUCTS, IMPORT_ORDERS, IMPORT_PURCHASE_ORDERS, IMPORT_DAILY_DEALS]);
importHistoryId = historyResult.rows[0].id;
} catch (error) {
console.error("Error creating import history record:", error);
@@ -146,7 +150,8 @@ async function main() {
categories: null,
products: null,
orders: null,
purchaseOrders: null
purchaseOrders: null,
dailyDeals: null
};
let totalRecordsAdded = 0;
@@ -224,6 +229,34 @@ async function main() {
}
}
if (IMPORT_DAILY_DEALS) {
try {
const stepStart = Date.now();
results.dailyDeals = await importDailyDeals(prodConnection, localConnection);
stepTimings.dailyDeals = Math.round((Date.now() - stepStart) / 1000);
if (isImportCancelled) throw new Error("Import cancelled");
completedSteps++;
console.log('Daily deals import result:', results.dailyDeals);
if (results.dailyDeals?.status === 'error') {
console.error('Daily deals import had an error:', results.dailyDeals.error);
} else {
totalRecordsAdded += parseInt(results.dailyDeals?.recordsAdded || 0);
totalRecordsUpdated += parseInt(results.dailyDeals?.recordsUpdated || 0);
totalRecordsDeleted += parseInt(results.dailyDeals?.recordsDeleted || 0);
}
} catch (error) {
console.error('Error during daily deals import:', error);
results.dailyDeals = {
status: 'error',
error: error.message,
recordsAdded: 0,
recordsUpdated: 0
};
}
}
const endTime = Date.now();
const totalElapsedSeconds = Math.round((endTime - startTime) / 1000);
@@ -241,15 +274,17 @@ async function main() {
'products_enabled', $5::boolean,
'orders_enabled', $6::boolean,
'purchase_orders_enabled', $7::boolean,
'categories_result', COALESCE($8::jsonb, 'null'::jsonb),
'products_result', COALESCE($9::jsonb, 'null'::jsonb),
'orders_result', COALESCE($10::jsonb, 'null'::jsonb),
'purchase_orders_result', COALESCE($11::jsonb, 'null'::jsonb),
'total_deleted', $12::integer,
'total_skipped', $13::integer,
'step_timings', $14::jsonb
'daily_deals_enabled', $8::boolean,
'categories_result', COALESCE($9::jsonb, 'null'::jsonb),
'products_result', COALESCE($10::jsonb, 'null'::jsonb),
'orders_result', COALESCE($11::jsonb, 'null'::jsonb),
'purchase_orders_result', COALESCE($12::jsonb, 'null'::jsonb),
'daily_deals_result', COALESCE($13::jsonb, 'null'::jsonb),
'total_deleted', $14::integer,
'total_skipped', $15::integer,
'step_timings', $16::jsonb
)
WHERE id = $15
WHERE id = $17
`, [
totalElapsedSeconds,
parseInt(totalRecordsAdded),
@@ -258,10 +293,12 @@ async function main() {
IMPORT_PRODUCTS,
IMPORT_ORDERS,
IMPORT_PURCHASE_ORDERS,
IMPORT_DAILY_DEALS,
JSON.stringify(results.categories),
JSON.stringify(results.products),
JSON.stringify(results.orders),
JSON.stringify(results.purchaseOrders),
JSON.stringify(results.dailyDeals),
totalRecordsDeleted,
totalRecordsSkipped,
JSON.stringify(stepTimings),

167
inventory-server/scripts/import/daily-deals.js Normal file
View File

@@ -0,0 +1,167 @@
const { outputProgress, formatElapsedTime } = require('../metrics-new/utils/progress');
/**
* Import daily deals from production MySQL to local PostgreSQL.
*
* Production has two tables:
* - product_daily_deals (deal_id, deal_date, pid, price_id)
* - product_current_prices (price_id, pid, price_each, active, ...)
*
* We join them in the prod query to denormalize the deal price, avoiding
* the need to sync the full product_current_prices table.
*
* On each sync:
* 1. Fetch deals from the last 7 days (plus today) from production
* 2. Upsert into local table
* 3. Hard delete local deals older than 7 days past their deal_date
*/
async function importDailyDeals(prodConnection, localConnection) {
outputProgress({
operation: "Starting daily deals import",
status: "running",
});
const startTime = Date.now();
try {
await localConnection.query('BEGIN');
// Fetch recent daily deals from production (MySQL 5.7, no CTEs)
// Join product_current_prices to get the actual deal price
// Only grab last 7 days + today + tomorrow (for pre-scheduled deals)
const [deals] = await prodConnection.query(`
SELECT
pdd.deal_id,
pdd.deal_date,
pdd.pid,
pdd.price_id,
pcp.price_each as deal_price
FROM product_daily_deals pdd
LEFT JOIN product_current_prices pcp ON pcp.price_id = pdd.price_id
WHERE pdd.deal_date >= DATE_SUB(CURDATE(), INTERVAL 7 DAY)
AND pdd.deal_date <= DATE_ADD(CURDATE(), INTERVAL 1 DAY)
ORDER BY pdd.deal_date DESC, pdd.pid
`);
outputProgress({
status: "running",
operation: "Daily deals import",
message: `Fetched ${deals.length} deals from production`,
elapsed: formatElapsedTime(startTime),
});
let totalInserted = 0;
let totalUpdated = 0;
if (deals.length > 0) {
// Batch upsert — filter to only PIDs that exist locally
const pids = [...new Set(deals.map(d => d.pid))];
const existingResult = await localConnection.query(
`SELECT pid FROM products WHERE pid = ANY($1)`,
[pids]
);
const existingPids = new Set(
(Array.isArray(existingResult) ? existingResult[0] : existingResult)
.rows.map(r => Number(r.pid))
);
const validDeals = deals.filter(d => existingPids.has(Number(d.pid)));
if (validDeals.length > 0) {
// Build batch upsert
const values = validDeals.flatMap(d => [
d.deal_date,
d.pid,
d.price_id,
d.deal_price ?? null,
]);
const placeholders = validDeals
.map((_, i) => `($${i * 4 + 1}, $${i * 4 + 2}, $${i * 4 + 3}, $${i * 4 + 4})`)
.join(',');
const upsertQuery = `
WITH upserted AS (
INSERT INTO product_daily_deals (deal_date, pid, price_id, deal_price)
VALUES ${placeholders}
ON CONFLICT (deal_date, pid) DO UPDATE SET
price_id = EXCLUDED.price_id,
deal_price = EXCLUDED.deal_price
WHERE
product_daily_deals.price_id IS DISTINCT FROM EXCLUDED.price_id OR
product_daily_deals.deal_price IS DISTINCT FROM EXCLUDED.deal_price
RETURNING
CASE WHEN xmax = 0 THEN true ELSE false END as is_insert
)
SELECT
COUNT(*) FILTER (WHERE is_insert) as inserted,
COUNT(*) FILTER (WHERE NOT is_insert) as updated
FROM upserted
`;
const result = await localConnection.query(upsertQuery, values);
const queryResult = Array.isArray(result) ? result[0] : result;
totalInserted = parseInt(queryResult.rows[0].inserted) || 0;
totalUpdated = parseInt(queryResult.rows[0].updated) || 0;
}
const skipped = deals.length - validDeals.length;
if (skipped > 0) {
console.log(`Skipped ${skipped} deals (PIDs not in local products table)`);
}
}
// Hard delete deals older than 7 days past their deal_date
const deleteResult = await localConnection.query(`
DELETE FROM product_daily_deals
WHERE deal_date < CURRENT_DATE - INTERVAL '7 days'
`);
const deletedCount = deleteResult.rowCount ??
(Array.isArray(deleteResult) ? deleteResult[0]?.rowCount : 0) ?? 0;
// Update sync status
await localConnection.query(`
INSERT INTO sync_status (table_name, last_sync_timestamp)
VALUES ('product_daily_deals', NOW())
ON CONFLICT (table_name) DO UPDATE SET
last_sync_timestamp = NOW()
`);
await localConnection.query('COMMIT');
outputProgress({
status: "complete",
operation: "Daily deals import completed",
message: `Inserted ${totalInserted}, updated ${totalUpdated}, deleted ${deletedCount} expired`,
current: totalInserted + totalUpdated,
total: totalInserted + totalUpdated,
duration: formatElapsedTime(startTime),
});
return {
status: "complete",
recordsAdded: totalInserted,
recordsUpdated: totalUpdated,
recordsDeleted: deletedCount,
totalRecords: totalInserted + totalUpdated,
};
} catch (error) {
console.error("Error importing daily deals:", error);
try {
await localConnection.query('ROLLBACK');
} catch (rollbackError) {
console.error("Error during rollback:", rollbackError);
}
outputProgress({
status: "error",
operation: "Daily deals import failed",
error: error.message,
});
throw error;
}
}
module.exports = importDailyDeals;