diff --git a/inventory-server/db/schema.sql b/inventory-server/db/schema.sql index 458e35e..4b4068e 100644 --- a/inventory-server/db/schema.sql +++ b/inventory-server/db/schema.sql @@ -30,6 +30,67 @@ CREATE TABLE IF NOT EXISTS products ( INDEX idx_brand (brand) ); +-- New table for product metrics +CREATE TABLE IF NOT EXISTS product_metrics ( + product_id BIGINT NOT NULL, + last_calculated_at TIMESTAMP NOT NULL, + -- Sales velocity metrics + daily_sales_avg DECIMAL(10,3), + weekly_sales_avg DECIMAL(10,3), + monthly_sales_avg DECIMAL(10,3), + -- Stock metrics + days_of_inventory INT, + weeks_of_inventory INT, + reorder_point INT, + safety_stock INT, + -- Financial metrics + avg_margin_percent DECIMAL(10,3), + total_revenue DECIMAL(10,3), + -- Purchase metrics + avg_lead_time_days INT, + last_purchase_date DATE, + last_received_date DATE, + -- Classification + abc_class CHAR(1), + stock_status VARCHAR(20), + PRIMARY KEY (product_id), + FOREIGN KEY (product_id) REFERENCES products(product_id) ON DELETE CASCADE +); + +-- New table for time-based aggregates +CREATE TABLE IF NOT EXISTS product_time_aggregates ( + product_id BIGINT NOT NULL, + year INT NOT NULL, + month INT NOT NULL, + -- Sales metrics + total_quantity_sold INT DEFAULT 0, + total_revenue DECIMAL(10,3) DEFAULT 0, + total_cost DECIMAL(10,3) DEFAULT 0, + order_count INT DEFAULT 0, + -- Stock changes + stock_received INT DEFAULT 0, + stock_ordered INT DEFAULT 0, + -- Calculated fields + avg_price DECIMAL(10,3), + profit_margin DECIMAL(10,3), + PRIMARY KEY (product_id, year, month), + FOREIGN KEY (product_id) REFERENCES products(product_id) ON DELETE CASCADE, + INDEX idx_date (year, month) +); + +-- New table for vendor performance +CREATE TABLE IF NOT EXISTS vendor_metrics ( + vendor VARCHAR(100) NOT NULL, + last_calculated_at TIMESTAMP NOT NULL, + avg_lead_time_days DECIMAL(10,3), + on_time_delivery_rate DECIMAL(5,2), + order_fill_rate DECIMAL(5,2), + total_orders INT, + total_late_orders INT, + PRIMARY KEY (vendor), + FOREIGN KEY (vendor) REFERENCES products(vendor) ON DELETE CASCADE +); + CREATE TABLE IF NOT EXISTS orders ( id BIGINT AUTO_INCREMENT PRIMARY KEY, order_number VARCHAR(50) NOT NULL, @@ -97,4 +158,48 @@ CREATE TABLE IF NOT EXISTS product_categories ( FOREIGN KEY (category_id) REFERENCES categories(id) ON DELETE CASCADE, INDEX idx_category (category_id), INDEX idx_product (product_id) -); \ No newline at end of file +); + +-- Create views for common calculations +CREATE OR REPLACE VIEW product_sales_trends AS +SELECT + p.product_id, + p.SKU, + p.title, + COALESCE(SUM(o.quantity), 0) as total_sold, + COALESCE(AVG(o.quantity), 0) as avg_quantity_per_order, + COALESCE(COUNT(DISTINCT o.order_number), 0) as number_of_orders, + MIN(o.date) as first_sale_date, + MAX(o.date) as last_sale_date +FROM + products p +LEFT JOIN + orders o ON p.product_id = o.product_id +WHERE + o.canceled = false +GROUP BY + p.product_id, p.SKU, p.title; + +-- Create view for inventory health +CREATE OR REPLACE VIEW inventory_health AS +SELECT + p.product_id, + p.SKU, + p.title, + p.stock_quantity, + pm.daily_sales_avg, + pm.days_of_inventory, + pm.reorder_point, + pm.safety_stock, + CASE + WHEN p.stock_quantity <= pm.safety_stock THEN 'Critical' + WHEN p.stock_quantity <= pm.reorder_point THEN 'Reorder' + WHEN p.stock_quantity > (pm.daily_sales_avg * 90) THEN 'Overstocked' + ELSE 'Healthy' + END as stock_status +FROM + products p +LEFT JOIN + product_metrics pm ON p.product_id = pm.product_id +WHERE + p.managing_stock = true; \ No newline at end of file diff --git a/inventory-server/scripts/create-test-snapshot.js b/inventory-server/scripts/create-test-snapshot.js new file mode 100644 index 0000000..57f8587 --- /dev/null +++ b/inventory-server/scripts/create-test-snapshot.js @@ -0,0 +1,115 @@ +const fs = require('fs'); +const path = require('path'); +const mysql = require('mysql2/promise'); +const dotenv = require('dotenv'); + +dotenv.config({ path: path.join(__dirname, '../.env') }); + +const dbConfig = { + host: process.env.DB_HOST, + user: process.env.DB_USER, + password: process.env.DB_PASSWORD, + database: process.env.DB_NAME, + multipleStatements: true +}; + +const SNAPSHOTS_DIR = path.join(__dirname, '../snapshots'); + +async function createSnapshot() { + console.log('Creating test database snapshot...'); + const pool = mysql.createPool(dbConfig); + + try { + const connection = await pool.getConnection(); + try { + // Create snapshots directory if it doesn't exist + if (!fs.existsSync(SNAPSHOTS_DIR)) { + fs.mkdirSync(SNAPSHOTS_DIR, { recursive: true }); + } + + // Get categories + const [categories] = await connection.query(` + SELECT id, name, created_at + FROM categories + LIMIT 10 + `); + + // Get a diverse set of ~100 products + const [products] = await connection.query(` + SELECT p.* + FROM products p + WHERE EXISTS ( + SELECT 1 FROM orders o WHERE o.product_id = p.product_id + UNION + SELECT 1 FROM purchase_orders po WHERE po.product_id = p.product_id + ) + ORDER BY RAND() + LIMIT 100 + `); + + // Get product_categories for selected products + const [product_categories] = await connection.query(` + SELECT pc.product_id, pc.category_id + FROM product_categories pc + WHERE pc.product_id IN (?) + `, [products.map(p => p.product_id)]); + + // Get orders for selected products (last 6 months) + const [orders] = await connection.query(` + SELECT o.* + FROM orders o + WHERE o.product_id IN (?) + AND o.date >= DATE_SUB(NOW(), INTERVAL 6 MONTH) + ORDER BY o.date DESC + `, [products.map(p => p.product_id)]); + + // Get purchase orders for selected products (last 6 months) + const [purchase_orders] = await connection.query(` + SELECT po.* + FROM purchase_orders po + WHERE po.product_id IN (?) + AND po.date >= DATE_SUB(NOW(), INTERVAL 6 MONTH) + ORDER BY po.date DESC + `, [products.map(p => p.product_id)]); + + // Create snapshot object + const snapshot = { + metadata: { + created_at: new Date().toISOString(), + description: 'Test snapshot with ~100 diverse products and their related data' + }, + categories, + products, + product_categories, + orders, + purchase_orders + }; + + // Save snapshot + const snapshotPath = path.join(SNAPSHOTS_DIR, 'test_snapshot.json'); + fs.writeFileSync(snapshotPath, JSON.stringify(snapshot, null, 2)); + + console.log('Snapshot created successfully:'); + console.log('Products:', products.length); + console.log('Orders:', orders.length); + console.log('Purchase Orders:', purchase_orders.length); + console.log('Categories:', categories.length); + console.log('Saved to:', snapshotPath); + + } finally { + connection.release(); + } + } catch (error) { + console.error('Error creating snapshot:', error); + throw error; + } finally { + await pool.end(); + } +} + +// Create snapshot if run directly +if (require.main === module) { + createSnapshot().catch(console.error); +} + +module.exports = { createSnapshot }; \ No newline at end of file diff --git a/inventory-server/scripts/import-csv.js b/inventory-server/scripts/import-csv.js index 6aa9c3f..862b9e6 100644 --- a/inventory-server/scripts/import-csv.js +++ b/inventory-server/scripts/import-csv.js @@ -23,6 +23,37 @@ const dbConfig = { namedPlaceholders: true }; +// Set up logging +const LOG_DIR = path.join(__dirname, '../logs'); +const ERROR_LOG = path.join(LOG_DIR, 'import-errors.log'); +const IMPORT_LOG = path.join(LOG_DIR, 'import.log'); + +// Ensure log directory exists +if (!fs.existsSync(LOG_DIR)) { + fs.mkdirSync(LOG_DIR, { recursive: true }); +} + +// Helper function to log errors +function logError(error, context = '') { + const timestamp = new Date().toISOString(); + const errorMessage = `[${timestamp}] ${context}\nError: ${error.message}\nStack: ${error.stack}\n\n`; + + // Log to error file + fs.appendFileSync(ERROR_LOG, errorMessage); + + // Also log to console + console.error(`\n${context}\nError: ${error.message}`); +} + +// Helper function to log import progress +function logImport(message) { + const timestamp = new Date().toISOString(); + const logMessage = `[${timestamp}] ${message}\n`; + + // Log to import file + fs.appendFileSync(IMPORT_LOG, logMessage); +} + // Helper function to output progress in JSON format function outputProgress(data) { if (!data.status) { @@ -31,6 +62,11 @@ function outputProgress(data) { ...data }; } + + // Log progress to import log + logImport(JSON.stringify(data)); + + // Output to console console.log(JSON.stringify(data)); } @@ -137,6 +173,306 @@ async function handleCategories(connection, productId, categoriesStr) { } } +// Helper function to calculate sales velocity metrics +async function calculateSalesVelocity(connection, productId) { + const [rows] = await connection.query(` + SELECT + COALESCE(COUNT(*) / NULLIF(DATEDIFF(MAX(date), MIN(date)), 0), 0) as daily_sales_avg, + COALESCE(SUM(quantity) / NULLIF(DATEDIFF(MAX(date), MIN(date)), 0) * 7, 0) as weekly_sales_avg, + COALESCE(SUM(quantity) / NULLIF(DATEDIFF(MAX(date), MIN(date)), 0) * 30, 0) as monthly_sales_avg + FROM orders + WHERE product_id = ? AND canceled = false + GROUP BY product_id + `, [productId]); + + return rows[0] || { daily_sales_avg: 0, weekly_sales_avg: 0, monthly_sales_avg: 0 }; +} + +// Helper function to calculate stock metrics +async function calculateStockMetrics(connection, productId, dailySalesAvg) { + const [product] = await connection.query( + 'SELECT stock_quantity FROM products WHERE product_id = ?', + [productId] + ); + + if (!product[0]) return null; + + const stockQty = product[0].stock_quantity; + const daysOfInventory = dailySalesAvg > 0 ? Math.floor(stockQty / dailySalesAvg) : 999; + const weeksOfInventory = Math.floor(daysOfInventory / 7); + + // Calculate safety stock (2 weeks of average sales) + const safetyStock = Math.ceil(dailySalesAvg * 14); + + // Calculate reorder point (safety stock + 1 week of sales) + const reorderPoint = Math.ceil(safetyStock + (dailySalesAvg * 7)); + + return { + days_of_inventory: daysOfInventory, + weeks_of_inventory: weeksOfInventory, + safety_stock: safetyStock, + reorder_point: reorderPoint + }; +} + +// Helper function to calculate financial metrics +async function calculateFinancialMetrics(connection, productId) { + const [rows] = await connection.query(` + SELECT + SUM(o.price * o.quantity) as total_revenue, + AVG((o.price - p.cost_price) / o.price * 100) as avg_margin_percent + FROM orders o + JOIN products p ON o.product_id = p.product_id + WHERE o.product_id = ? AND o.canceled = false + GROUP BY o.product_id + `, [productId]); + + return rows[0] || { total_revenue: 0, avg_margin_percent: 0 }; +} + +// Helper function to calculate purchase metrics +async function calculatePurchaseMetrics(connection, productId) { + const [rows] = await connection.query(` + SELECT + AVG(DATEDIFF(received_date, date)) as avg_lead_time_days, + MAX(date) as last_purchase_date, + MAX(received_date) as last_received_date + FROM purchase_orders + WHERE product_id = ? AND status = 'closed' + GROUP BY product_id + `, [productId]); + + return rows[0] || { + avg_lead_time_days: 0, + last_purchase_date: null, + last_received_date: null + }; +} + +// Helper function to calculate ABC classification +async function calculateABCClass(connection, productId) { + // Get total revenue for this product + const [productRevenue] = await connection.query(` + SELECT SUM(price * quantity) as revenue + FROM orders + WHERE product_id = ? AND canceled = false + `, [productId]); + + // Get total revenue across all products + const [totalRevenue] = await connection.query(` + SELECT SUM(price * quantity) as total + FROM orders + WHERE canceled = false + `); + + const revenue = productRevenue[0]?.revenue || 0; + const total = totalRevenue[0]?.total || 0; + + if (total === 0) return 'C'; + + const percentage = (revenue / total) * 100; + + // A: Top 20% of revenue + // B: Next 30% of revenue + // C: Remaining 50% of revenue + if (percentage >= 20) return 'A'; + if (percentage >= 5) return 'B'; + return 'C'; +} + +// Helper function to calculate time-based aggregates +async function calculateTimeAggregates(connection, productId) { + await connection.query(` + INSERT INTO product_time_aggregates ( + product_id, year, month, + total_quantity_sold, total_revenue, total_cost, + order_count, stock_received, stock_ordered, + avg_price, profit_margin + ) + SELECT + o.product_id, + YEAR(o.date) as year, + MONTH(o.date) as month, + SUM(o.quantity) as total_quantity_sold, + SUM(o.price * o.quantity) as total_revenue, + SUM(p.cost_price * o.quantity) as total_cost, + COUNT(DISTINCT o.order_number) as order_count, + COALESCE(SUM(po.received), 0) as stock_received, + COALESCE(SUM(po.ordered), 0) as stock_ordered, + AVG(o.price) as avg_price, + CASE + WHEN SUM(o.price * o.quantity) = 0 THEN 0 + ELSE ((SUM(o.price * o.quantity) - COALESCE(SUM(p.cost_price * o.quantity), 0)) / + NULLIF(SUM(o.price * o.quantity), 0) * 100) + END as profit_margin + FROM orders o + JOIN products p ON o.product_id = p.product_id + LEFT JOIN purchase_orders po ON o.product_id = po.product_id + AND YEAR(o.date) = YEAR(po.date) + AND MONTH(o.date) = MONTH(po.date) + WHERE o.product_id = ? AND o.canceled = false + GROUP BY o.product_id, YEAR(o.date), MONTH(o.date) + ON DUPLICATE KEY UPDATE + total_quantity_sold = VALUES(total_quantity_sold), + total_revenue = VALUES(total_revenue), + total_cost = VALUES(total_cost), + order_count = VALUES(order_count), + stock_received = VALUES(stock_received), + stock_ordered = VALUES(stock_ordered), + avg_price = VALUES(avg_price), + profit_margin = VALUES(profit_margin) + `, [productId]); +} + +// Helper function to calculate vendor metrics +async function calculateVendorMetrics(connection) { + try { + // Get list of vendors + const [vendors] = await connection.query('SELECT DISTINCT vendor FROM products WHERE vendor IS NOT NULL'); + const startTime = Date.now(); + let current = 0; + const total = vendors.length; + + outputProgress({ + operation: 'Calculating vendor metrics', + current: 0, + total, + percentage: '0.0' + }); + + for (const { vendor } of vendors) { + // Calculate average lead time + const [leadTimeResult] = await connection.query(` + SELECT + AVG(DATEDIFF(received_date, date)) as avg_lead_time, + COUNT(*) as total_orders, + SUM(CASE WHEN ordered = received THEN 1 ELSE 0 END) as fulfilled_orders + FROM purchase_orders + WHERE vendor = ? AND status = 'closed' + GROUP BY vendor + `, [vendor]); + + const metrics = leadTimeResult[0] || { + avg_lead_time: 0, + total_orders: 0, + fulfilled_orders: 0 + }; + + // Calculate fill rate + const fillRate = metrics.total_orders > 0 ? + (metrics.fulfilled_orders / metrics.total_orders * 100) : 0; + + // Update vendor metrics + await connection.query(` + INSERT INTO vendor_metrics ( + vendor, + avg_lead_time_days, + total_orders, + fulfilled_orders, + fill_rate + ) VALUES (?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + avg_lead_time_days = VALUES(avg_lead_time_days), + total_orders = VALUES(total_orders), + fulfilled_orders = VALUES(fulfilled_orders), + fill_rate = VALUES(fill_rate) + `, [ + vendor, + metrics.avg_lead_time || 0, + metrics.total_orders, + metrics.fulfilled_orders, + fillRate + ]); + + current++; + updateProgress(current, total, 'Calculating vendor metrics', startTime); + } + + outputProgress({ + status: 'complete', + operation: 'Vendor metrics calculation completed', + current: total, + total, + percentage: '100.0' + }); + } catch (error) { + logError(error, 'Error calculating vendor metrics'); + throw error; + } +} + +// Helper function to update product metrics +async function updateProductMetrics(connection, productId, startTime, current, total) { + try { + // Calculate sales velocity metrics + const velocityMetrics = await calculateSalesVelocity(connection, productId); + + // Calculate stock metrics + const stockMetrics = await calculateStockMetrics(connection, productId, velocityMetrics.daily_sales_avg); + + // Calculate financial metrics + const financialMetrics = await calculateFinancialMetrics(connection, productId); + + // Calculate purchase metrics + const purchaseMetrics = await calculatePurchaseMetrics(connection, productId); + + // Update metrics in database + await connection.query(` + INSERT INTO product_metrics ( + product_id, + daily_sales_avg, + weekly_sales_avg, + monthly_sales_avg, + days_of_inventory, + weeks_of_inventory, + safety_stock, + reorder_point, + total_revenue, + avg_margin_percent, + avg_lead_time_days, + last_purchase_date, + last_received_date + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE + daily_sales_avg = VALUES(daily_sales_avg), + weekly_sales_avg = VALUES(weekly_sales_avg), + monthly_sales_avg = VALUES(monthly_sales_avg), + days_of_inventory = VALUES(days_of_inventory), + weeks_of_inventory = VALUES(weeks_of_inventory), + safety_stock = VALUES(safety_stock), + reorder_point = VALUES(reorder_point), + total_revenue = VALUES(total_revenue), + avg_margin_percent = VALUES(avg_margin_percent), + avg_lead_time_days = VALUES(avg_lead_time_days), + last_purchase_date = VALUES(last_purchase_date), + last_received_date = VALUES(last_received_date) + `, [ + productId, + velocityMetrics.daily_sales_avg, + velocityMetrics.weekly_sales_avg, + velocityMetrics.monthly_sales_avg, + stockMetrics?.days_of_inventory || 0, + stockMetrics?.weeks_of_inventory || 0, + stockMetrics?.safety_stock || 0, + stockMetrics?.reorder_point || 0, + financialMetrics.total_revenue, + financialMetrics.avg_margin_percent, + purchaseMetrics.avg_lead_time_days, + purchaseMetrics.last_purchase_date, + purchaseMetrics.last_received_date + ]); + + // Output progress every 5 products or every second + if (current % 5 === 0 || Date.now() - startTime > 1000) { + updateProgress(current, total, 'Calculating product metrics', startTime); + startTime = Date.now(); + } + } catch (error) { + logError(error, `Error updating metrics for product ${productId}`); + throw error; + } +} + async function importProducts(pool, filePath) { const parser = fs.createReadStream(filePath).pipe(csv.parse({ columns: true, trim: true })); const totalRows = PRODUCTS_TEST_LIMIT > 0 ? Math.min(await countRows(filePath), PRODUCTS_TEST_LIMIT) : await countRows(filePath); @@ -172,91 +508,101 @@ async function importProducts(pool, filePath) { let batch = []; let categoryUpdates = new Map(); // Store category updates for batch processing - for await (const record of parser) { - if (PRODUCTS_TEST_LIMIT > 0 && rowCount >= PRODUCTS_TEST_LIMIT) { - // Process remaining batch - if (batch.length > 0) { - await processBatch(batch, categoryUpdates); - } - outputProgress({ - operation: 'Products import', - message: `Reached test limit of ${PRODUCTS_TEST_LIMIT.toLocaleString()} rows`, - current: rowCount, - total: totalRows - }); - break; - } - rowCount++; - - // Update progress every 100ms to avoid console flooding - const now = Date.now(); - if (now - lastUpdate > 100) { - updateProgress(rowCount, totalRows, 'Products import', startTime); - lastUpdate = now; - } - - // Add to batch - batch.push({ - product_id: record.product_id, - title: record.title, - SKU: record.SKU, - created_at: convertDate(record.created_at), - stock_quantity: parseInt(record.stock_quantity) || 0, - price: parseFloat(record.price) || 0, - regular_price: parseFloat(record.regular_price) || 0, - cost_price: parseFloat(record.cost_price) || null, - landing_cost_price: parseFloat(record.landing_cost_price) || null, - barcode: record.barcode, - updated_at: convertDate(record.updated_at), - visible: record.visible === '1', - managing_stock: record.managing_stock === '1', - replenishable: record.replenishable === '1', - vendor: record.vendor, - vendor_reference: record.vendor_reference, - permalink: record.permalink, - categories: record.categories, - image: record.image, - brand: record.brand, - options: record.options, - tags: record.tags, - moq: parseInt(record.moq) || 1, - uom: parseInt(record.uom) || 1 - }); - - // Store category updates for later - if (record.categories) { - categoryUpdates.set(record.product_id, record.categories); - } - - // Process batch if it reaches BATCH_SIZE - if (batch.length >= BATCH_SIZE) { - await processBatch(batch, categoryUpdates); - batch = []; - categoryUpdates.clear(); - } - } - - // Process any remaining records in the final batch - if (batch.length > 0) { - await processBatch(batch, categoryUpdates); - } + // Get a connection from the pool that we'll reuse + const connection = await pool.getConnection(); - outputProgress({ - status: 'running', - operation: 'Products import completed', - current: rowCount, - total: totalRows, - added, - updated, - duration: formatDuration((Date.now() - startTime) / 1000), - percentage: '100' - }); + try { + for await (const record of parser) { + if (PRODUCTS_TEST_LIMIT > 0 && rowCount >= PRODUCTS_TEST_LIMIT) { + // Process remaining batch + if (batch.length > 0) { + await processBatch(batch, categoryUpdates); + } + outputProgress({ + operation: 'Products import', + message: `Reached test limit of ${PRODUCTS_TEST_LIMIT.toLocaleString()} rows`, + current: rowCount, + total: totalRows + }); + break; + } + rowCount++; + + // Update progress every 100ms to avoid console flooding + const now = Date.now(); + if (now - lastUpdate > 100) { + updateProgress(rowCount, totalRows, 'Products import', startTime); + lastUpdate = now; + } + + // Add to batch + batch.push({ + product_id: record.product_id, + title: record.title, + SKU: record.SKU, + created_at: convertDate(record.created_at), + stock_quantity: parseInt(record.stock_quantity) || 0, + price: parseFloat(record.price) || 0, + regular_price: parseFloat(record.regular_price) || 0, + cost_price: parseFloat(record.cost_price) || null, + landing_cost_price: parseFloat(record.landing_cost_price) || null, + barcode: record.barcode, + updated_at: convertDate(record.updated_at), + visible: record.visible === '1', + managing_stock: record.managing_stock === '1', + replenishable: record.replenishable === '1', + vendor: record.vendor, + vendor_reference: record.vendor_reference, + permalink: record.permalink, + categories: record.categories, + image: record.image, + brand: record.brand, + options: record.options, + tags: record.tags, + moq: parseInt(record.moq) || 1, + uom: parseInt(record.uom) || 1 + }); + + if (record.categories) { + categoryUpdates.set(record.product_id, record.categories); + } + + // Process batch if it reaches BATCH_SIZE + if (batch.length >= BATCH_SIZE) { + await processBatch(batch, categoryUpdates); + batch = []; + categoryUpdates.clear(); + } + } + + // Process any remaining records in the final batch + if (batch.length > 0) { + await processBatch(batch, categoryUpdates); + } + + outputProgress({ + status: 'running', + operation: 'Products import completed', + current: rowCount, + total: totalRows, + added, + updated, + duration: formatDuration((Date.now() - startTime) / 1000), + percentage: '100' + }); + } catch (error) { + console.error('Error during products import:', error); + throw error; + } finally { + if (connection) { + connection.release(); + } + } // Helper function to process a batch of records async function processBatch(records, categoryUpdates) { if (records.length === 0) return; - const connection = await pool.getConnection(); try { await connection.beginTransaction(); try { @@ -301,12 +647,6 @@ async function importProducts(pool, filePath) { // Update stats if (result.affectedRows > 0) { - // For INSERT ... ON DUPLICATE KEY UPDATE: - // - If a row is inserted, affectedRows = 1 - // - If a row is updated, affectedRows = 2 - // So we can calculate: - // - Number of inserts = number of rows where affectedRows = 1 - // - Number of updates = number of rows where affectedRows = 2 const insertCount = result.affectedRows - result.changedRows; const updateCount = result.changedRows; added += insertCount; @@ -321,13 +661,12 @@ async function importProducts(pool, filePath) { await connection.commit(); } catch (error) { await connection.rollback(); + logError(error, `Error processing batch of ${records.length} records`); throw error; } } catch (error) { - console.error(`\nError processing batch:`, error.message); + logError(error, `Error in batch processing:\nFirst record: ${JSON.stringify(records[0])}`); // Continue with next batch instead of failing completely - } finally { - connection.release(); } } } @@ -674,9 +1013,11 @@ async function main() { }); const startTime = Date.now(); - const pool = mysql.createPool(dbConfig); + let pool; try { + pool = mysql.createPool(dbConfig); + // Check if tables exist, if not create them outputProgress({ operation: 'Checking database schema', @@ -686,19 +1027,79 @@ async function main() { const schemaSQL = fs.readFileSync(path.join(__dirname, '../db/schema.sql'), 'utf8'); await pool.query(schemaSQL); - // Import products first since they're referenced by other tables - await importProducts(pool, path.join(__dirname, '../csv/39f2x83-products.csv')); + // Step 1: Import all data first + try { + // Import products first since they're referenced by other tables + await importProducts(pool, path.join(__dirname, '../csv/39f2x83-products.csv')); - // Process orders and purchase orders in parallel - outputProgress({ - operation: 'Starting parallel import', - message: 'Processing orders and purchase orders simultaneously...' - }); + // Process orders and purchase orders in parallel + outputProgress({ + operation: 'Starting parallel import', + message: 'Processing orders and purchase orders simultaneously...' + }); - await Promise.all([ - importOrders(pool, path.join(__dirname, '../csv/39f2x83-orders.csv')), - importPurchaseOrders(pool, path.join(__dirname, '../csv/39f2x83-purchase_orders.csv')) - ]); + await Promise.all([ + importOrders(pool, path.join(__dirname, '../csv/39f2x83-orders.csv')), + importPurchaseOrders(pool, path.join(__dirname, '../csv/39f2x83-purchase_orders.csv')) + ]); + + // Step 2: Calculate all metrics after imports are complete + outputProgress({ + operation: 'Starting metrics calculation', + message: 'Calculating metrics for all products and vendors...' + }); + + const connection = await pool.getConnection(); + try { + // Calculate product metrics + const [products] = await connection.query('SELECT DISTINCT product_id FROM products'); + const totalProducts = products.length; + let processedProducts = 0; + const metricsStartTime = Date.now(); + + outputProgress({ + operation: 'Starting product metrics calculation', + message: `Calculating metrics for ${totalProducts} products...`, + current: 0, + total: totalProducts, + percentage: '0' + }); + + for (const product of products) { + try { + // Update progress every 5 products or 1 second + if (processedProducts % 5 === 0 || (Date.now() - lastUpdate) > 1000) { + updateProgress(processedProducts, totalProducts, 'Calculating product metrics', metricsStartTime); + lastUpdate = Date.now(); + } + + await updateProductMetrics(connection, product.product_id, metricsStartTime, processedProducts, totalProducts); + processedProducts++; + } catch (error) { + logError(error, `Error calculating metrics for product ${product.product_id}`); + // Continue with next product instead of failing completely + } + } + + outputProgress({ + operation: 'Product metrics calculation completed', + current: processedProducts, + total: totalProducts, + duration: formatDuration((Date.now() - metricsStartTime) / 1000), + percentage: '100' + }); + + // Calculate vendor metrics + await calculateVendorMetrics(connection); + + } finally { + connection.release(); + } + + } catch (error) { + logError(error, 'Error during import/metrics calculation'); + throw error; + } outputProgress({ status: 'complete', @@ -706,15 +1107,21 @@ async function main() { duration: formatDuration((Date.now() - startTime) / 1000) }); } catch (error) { + logError(error, 'Fatal error during import process'); outputProgress({ status: 'error', error: error.message }); process.exit(1); } finally { - await pool.end(); + if (pool) { + await pool.end(); + } } } // Run the import -main(); \ No newline at end of file +main().catch(error => { + logError(error, 'Unhandled error in main process'); + process.exit(1); +}); \ No newline at end of file diff --git a/inventory-server/scripts/restore-test-snapshot.js b/inventory-server/scripts/restore-test-snapshot.js new file mode 100644 index 0000000..acf2d3d --- /dev/null +++ b/inventory-server/scripts/restore-test-snapshot.js @@ -0,0 +1,129 @@ +const fs = require('fs'); +const path = require('path'); +const mysql = require('mysql2/promise'); +const dotenv = require('dotenv'); + +dotenv.config({ path: path.join(__dirname, '../.env') }); + +const dbConfig = { + host: process.env.DB_HOST, + user: process.env.DB_USER, + password: process.env.DB_PASSWORD, + database: process.env.DB_NAME, + multipleStatements: true +}; + +const SNAPSHOTS_DIR = path.join(__dirname, '../snapshots'); + +async function restoreSnapshot() { + console.log('Restoring test database from snapshot...'); + const pool = mysql.createPool(dbConfig); + + try { + // Read snapshot + const snapshotPath = path.join(SNAPSHOTS_DIR, 'test_snapshot.json'); + if (!fs.existsSync(snapshotPath)) { + throw new Error('Snapshot file not found. Run create-test-snapshot.js first.'); + } + + const snapshot = JSON.parse(fs.readFileSync(snapshotPath, 'utf8')); + + // First, create schema (this will drop existing tables) + const schemaSQL = fs.readFileSync(path.join(__dirname, '../db/schema.sql'), 'utf8'); + await pool.query(schemaSQL); + + const connection = await pool.getConnection(); + try { + await connection.beginTransaction(); + + // Insert categories first (they're referenced by product_categories) + if (snapshot.categories.length > 0) { + const categoryValues = snapshot.categories.map(c => [c.id, c.name, c.created_at]); + await connection.query( + 'INSERT INTO categories (id, name, created_at) VALUES ?', + [categoryValues] + ); + } + + // Insert products + if (snapshot.products.length > 0) { + const productValues = snapshot.products.map(p => [ + p.product_id, p.title, p.SKU, p.created_at, p.stock_quantity, + p.price, p.regular_price, p.cost_price, p.landing_cost_price, + p.barcode, p.updated_at, p.visible, p.managing_stock, + p.replenishable, p.vendor, p.vendor_reference, p.permalink, + p.categories, p.image, p.brand, p.options, p.tags, p.moq, p.uom + ]); + await connection.query( + 'INSERT INTO products VALUES ?', + [productValues] + ); + } + + // Insert product_categories relationships + if (snapshot.product_categories.length > 0) { + const pcValues = snapshot.product_categories.map(pc => [ + pc.product_id, pc.category_id + ]); + await connection.query( + 'INSERT INTO product_categories (product_id, category_id) VALUES ?', + [pcValues] + ); + } + + // Insert orders + if (snapshot.orders.length > 0) { + const orderValues = snapshot.orders.map(o => [ + o.id, o.order_number, o.product_id, o.SKU, o.date, + o.price, o.quantity, o.discount, o.tax, o.tax_included, + o.shipping, o.customer, o.status, o.payment_method, + o.shipping_method, o.shipping_address, o.billing_address, + o.canceled + ]); + await connection.query( + 'INSERT INTO orders VALUES ?', + [orderValues] + ); + } + + // Insert purchase orders + if (snapshot.purchase_orders.length > 0) { + const poValues = snapshot.purchase_orders.map(po => [ + po.id, po.po_id, po.vendor, po.date, po.expected_date, + po.product_id, po.sku, po.cost_price, po.status, po.notes, + po.ordered, po.received, po.received_date + ]); + await connection.query( + 'INSERT INTO purchase_orders VALUES ?', + [poValues] + ); + } + + await connection.commit(); + + console.log('Snapshot restored successfully:'); + console.log('Products:', snapshot.products.length); + console.log('Orders:', snapshot.orders.length); + console.log('Purchase Orders:', snapshot.purchase_orders.length); + console.log('Categories:', snapshot.categories.length); + + } catch (error) { + await connection.rollback(); + throw error; + } finally { + connection.release(); + } + } catch (error) { + console.error('Error restoring snapshot:', error); + throw error; + } finally { + await pool.end(); + } +} + +// Restore snapshot if run directly +if (require.main === module) { + restoreSnapshot().catch(console.error); +} + +module.exports = { restoreSnapshot }; \ No newline at end of file diff --git a/inventory/src/pages/Settings.tsx b/inventory/src/pages/Settings.tsx index 0a2b851..0c8d20b 100644 --- a/inventory/src/pages/Settings.tsx +++ b/inventory/src/pages/Settings.tsx @@ -58,6 +58,9 @@ export function Settings() { orders: 0, purchaseOrders: 0 }); + const [isCreatingSnapshot, setIsCreatingSnapshot] = useState(false); + const [isRestoringSnapshot, setIsRestoringSnapshot] = useState(false); + const [snapshotProgress, setSnapshotProgress] = useState(null); // Helper function to update progress state const updateProgressState = (progressData: any) => { @@ -87,11 +90,15 @@ export function Settings() { setImportProgress(prev => ({ ...prev, ...progressUpdate })); } else if (operation.includes('purchase orders import')) { setPurchaseOrdersProgress(prev => ({ ...prev, ...progressUpdate })); + } else if (operation.includes('metrics') || operation.includes('vendor metrics')) { + setImportProgress(prev => ({ ...prev, ...progressUpdate })); + } else if (operation.includes('snapshot')) { + setSnapshotProgress(prev => ({ ...prev, ...progressUpdate })); } }; // Helper to connect to event source - const connectToEventSource = useCallback((type: 'update' | 'import' | 'reset') => { + const connectToEventSource = useCallback((type: 'update' | 'import' | 'reset' | 'snapshot') => { if (eventSource) { eventSource.close(); } @@ -389,6 +396,63 @@ export function Settings() { } }; + // Add handlers for snapshot operations + const handleCreateSnapshot = async () => { + try { + setIsCreatingSnapshot(true); + setSnapshotProgress({ status: 'running', operation: 'Creating test data snapshot' }); + + // Connect to SSE for progress updates + connectToEventSource('snapshot'); + + const response = await fetch(`${config.apiUrl}/snapshot/create`, { + method: 'POST', + credentials: 'include' + }); + + if (!response.ok) { + throw new Error('Failed to create snapshot'); + } + } catch (error) { + console.error('Error creating snapshot:', error); + if (eventSource) { + eventSource.close(); + setEventSource(null); + } + setIsCreatingSnapshot(false); + setSnapshotProgress(null); + toast.error('Failed to create snapshot'); + } + }; + + const handleRestoreSnapshot = async () => { + try { + setIsRestoringSnapshot(true); + setSnapshotProgress({ status: 'running', operation: 'Restoring test data snapshot' }); + + // Connect to SSE for progress updates + connectToEventSource('snapshot'); + + const response = await fetch(`${config.apiUrl}/snapshot/restore`, { + method: 'POST', + credentials: 'include' + }); + + if (!response.ok) { + throw new Error('Failed to restore snapshot'); + } + } catch (error) { + console.error('Error restoring snapshot:', error); + if (eventSource) { + eventSource.close(); + setEventSource(null); + } + setIsRestoringSnapshot(false); + setSnapshotProgress(null); + toast.error('Failed to restore snapshot'); + } + }; + // Cleanup on unmount useEffect(() => { return () => { @@ -625,6 +689,30 @@ export function Settings() { {renderProgress(purchaseOrdersProgress)} )} + {/* Show metrics calculation progress */} + {importProgress?.operation?.toLowerCase().includes('metrics') && ( +
+ +

+ {importProgress.message || importProgress.operation || 'Calculating metrics...'} + {importProgress.current && importProgress.total && ( + <> ({importProgress.current} of {importProgress.total}) + )} +

+ {importProgress.elapsed && ( +

+ Elapsed: {importProgress.elapsed} + {importProgress.remaining && <> • Remaining: {importProgress.remaining}} +

+ )} +
+ )} + {/* Show vendor metrics progress */} + {importProgress?.operation?.toLowerCase().includes('vendor metrics') && ( +
+ {renderProgress(importProgress)} +
+ )} )} @@ -637,36 +725,104 @@ export function Settings() { Drop all tables and recreate the database schema. This will delete ALL data. -
+ + + + + + + Are you absolutely sure? + + This action cannot be undone. This will permanently delete all data from the database. + + + + Cancel + Continue + + + + {resetProgress && ( +
+ +

+ {resetProgress.message || 'Resetting database...'} +

+
+ )} + + + + {/* Test Data Snapshots Card */} + + + Test Data Snapshots + Create and restore test data snapshots for development and testing. + + +
+ - Are you absolutely sure? + Restore test data snapshot? - This action cannot be undone. This will permanently delete all data - from the database and reset it to its initial state. + This will replace your current database with the test data snapshot. Any unsaved changes will be lost. Cancel - - Reset Database - + Continue
- - {isResetting && renderProgress(resetProgress)} + {snapshotProgress && ( +
+ +

+ {snapshotProgress.message || 'Processing snapshot...'} +

+
+ )} +
+

The test data snapshot includes:

+
    +
  • ~100 diverse products with associated data
    • +
  • Orders from the last 6 months
    • +
  • Purchase orders from the last 6 months
    • +
  • Categories and product relationships
    • +
+