matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

More import optimizations + reset optimizations

Browse Source
This commit is contained in:
Matt
2025-01-11 00:18:03 -05:00
parent 7ce5092b69
commit 0bc86a3fee
3 changed files with 183 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
inventory-server/scripts/import-csv.js
View File

@@ -124,7 +124,14 @@ async function importProducts(pool, filePath) {
});
function convertDate(dateStr) {
if (!dateStr) return null;
if (!dateStr) {
// Default to current date for missing dates
const now = new Date();
const year = now.getFullYear();
const month = String(now.getMonth() + 1).padStart(2, '0');
const day = String(now.getDate()).padStart(2, '0');
return `${year}-${month}-${day}`;
}
const [day, month, year] = dateStr.split('-');
return `${year}-${month}-${day}`;
}
@@ -268,8 +275,16 @@ async function importProducts(pool, filePath) {
// Update stats
if (result.affectedRows > 0) {
updated += result.affectedRows - result.insertId;
added += result.insertId;
// For INSERT ... ON DUPLICATE KEY UPDATE:
// - If a row is inserted, affectedRows = 1
// - If a row is updated, affectedRows = 2
// So we can calculate:
// - Number of inserts = number of rows where affectedRows = 1
// - Number of updates = number of rows where affectedRows = 2
const insertCount = result.affectedRows - result.changedRows;
const updateCount = result.changedRows;
added += insertCount;
updated += updateCount;
}
// Process categories within the same transaction
@@ -304,7 +319,14 @@ async function importOrders(pool, filePath) {
});
function convertDate(dateStr) {
if (!dateStr) return null;
if (!dateStr) {
// Default to current date for missing dates
const now = new Date();
const year = now.getFullYear();
const month = String(now.getMonth() + 1).padStart(2, '0');
const day = String(now.getDate()).padStart(2, '0');
return `${year}-${month}-${day}`;
}
const [day, month, year] = dateStr.split('-');
return `${year}-${month}-${day}`;
}
@@ -430,8 +452,16 @@ async function importOrders(pool, filePath) {
// Update stats
if (result.affectedRows > 0) {
updated += result.affectedRows - result.insertId;
added += result.insertId;
// For INSERT ... ON DUPLICATE KEY UPDATE:
// - If a row is inserted, affectedRows = 1
// - If a row is updated, affectedRows = 2
// So we can calculate:
// - Number of inserts = number of rows where affectedRows = 1
// - Number of updates = number of rows where affectedRows = 2
const insertCount = result.affectedRows - result.changedRows;
const updateCount = result.changedRows;
added += insertCount;
updated += updateCount;
}
} catch (error) {
console.error(`\nError processing batch:`, error.message);
@@ -456,7 +486,14 @@ async function importPurchaseOrders(pool, filePath) {
});
function convertDate(dateStr) {
if (!dateStr) return null;
if (!dateStr) {
// Default to current date for missing dates
const now = new Date();
const year = now.getFullYear();
const month = String(now.getMonth() + 1).padStart(2, '0');
const day = String(now.getDate()).padStart(2, '0');
return `${year}-${month}-${day}`;
}
const [day, month, year] = dateStr.split('-');
return `${year}-${month}-${day}`;
}
@@ -583,8 +620,16 @@ async function importPurchaseOrders(pool, filePath) {
// Update stats
if (result.affectedRows > 0) {
updated += result.affectedRows - result.insertId;
added += result.insertId;
// For INSERT ... ON DUPLICATE KEY UPDATE:
// - If a row is inserted, affectedRows = 1
// - If a row is updated, affectedRows = 2
// So we can calculate:
// - Number of inserts = number of rows where affectedRows = 1
// - Number of updates = number of rows where affectedRows = 2
const insertCount = result.affectedRows - result.changedRows;
const updateCount = result.changedRows;
added += insertCount;
updated += updateCount;
}
} catch (error) {
console.error(`\nError processing batch:`, error.message);
@@ -617,8 +662,17 @@ async function main() {
// Import products first since they're referenced by other tables
await importProducts(pool, path.join(__dirname, '../csv/39f2x83-products.csv'));
await importOrders(pool, path.join(__dirname, '../csv/39f2x83-orders.csv'));
await importPurchaseOrders(pool, path.join(__dirname, '../csv/39f2x83-purchase_orders.csv'));
// Process orders and purchase orders in parallel
outputProgress({
operation: 'Starting parallel import',
message: 'Processing orders and purchase orders simultaneously...'
});
await Promise.all([
importOrders(pool, path.join(__dirname, '../csv/39f2x83-orders.csv')),
importPurchaseOrders(pool, path.join(__dirname, '../csv/39f2x83-purchase_orders.csv'))
]);
outputProgress({
status: 'complete',

67
inventory-server/scripts/reset-db.js
View File

@@ -1,7 +1,7 @@
const mysql = require('mysql2/promise');
const path = require('path');
const dotenv = require('dotenv');
const { spawn } = require('child_process');
const fs = require('fs');
dotenv.config({ path: path.join(__dirname, '../.env') });
@@ -33,77 +33,48 @@ async function resetDatabase() {
const connection = await mysql.createConnection(dbConfig);
try {
// Get list of all tables
// Get list of all tables efficiently
outputProgress({
operation: 'Getting table list',
message: 'Retrieving all table names...'
});
const [tables] = await connection.query(
'SELECT table_name FROM information_schema.tables WHERE table_schema = ?',
[dbConfig.database]
// More efficient query to get table names
const [tables] = await connection.query(`
SELECT GROUP_CONCAT(table_name) as tables
FROM information_schema.tables
WHERE table_schema = DATABASE()`
);
if (tables.length === 0) {
if (!tables[0].tables) {
outputProgress({
operation: 'No tables found',
message: 'Database is already empty'
});
} else {
// Disable foreign key checks to allow dropping tables with dependencies
await connection.query('SET FOREIGN_KEY_CHECKS = 0');
// Drop each table
for (let i = 0; i < tables.length; i++) {
const tableName = tables[i].TABLE_NAME;
// Disable foreign key checks and drop all tables in one query
outputProgress({
operation: 'Dropping tables',
message: `Dropping table: ${tableName}`,
current: i + 1,
total: tables.length,
percentage: (((i + 1) / tables.length) * 100).toFixed(1)
message: 'Dropping all tables...'
});
await connection.query(`DROP TABLE IF EXISTS \`${tableName}\``);
}
// Re-enable foreign key checks
await connection.query('SET FOREIGN_KEY_CHECKS = 0');
// Create DROP TABLE statements for all tables at once
const dropQuery = `DROP TABLE IF EXISTS ${tables[0].tables.split(',').map(table => '`' + table + '`').join(', ')}`;
await connection.query(dropQuery);
await connection.query('SET FOREIGN_KEY_CHECKS = 1');
}
// Run setup-db.js
// Read and execute schema directly instead of spawning a process
outputProgress({
operation: 'Running database setup',
message: 'Creating new tables...'
});
const setupScript = path.join(__dirname, 'setup-db.js');
const setupProcess = spawn('node', [setupScript]);
setupProcess.stdout.on('data', (data) => {
const output = data.toString().trim();
outputProgress({
operation: 'Database setup',
message: output
});
});
setupProcess.stderr.on('data', (data) => {
const error = data.toString().trim();
outputProgress({
status: 'error',
error
});
});
await new Promise((resolve, reject) => {
setupProcess.on('close', (code) => {
if (code === 0) {
resolve();
} else {
reject(new Error(`Setup process exited with code ${code}`));
}
});
});
const schemaSQL = fs.readFileSync(path.join(__dirname, '../db/schema.sql'), 'utf8');
await connection.query(schemaSQL);
outputProgress({
status: 'complete',

173
inventory/src/pages/Settings.tsx
View File

@@ -50,6 +50,7 @@ export function Settings() {
const [isResetting, setIsResetting] = useState(false);
const [updateProgress, setUpdateProgress] = useState<ImportProgress | null>(null);
const [importProgress, setImportProgress] = useState<ImportProgress | null>(null);
const [purchaseOrdersProgress, setPurchaseOrdersProgress] = useState<ImportProgress | null>(null);
const [resetProgress, setResetProgress] = useState<ImportProgress | null>(null);
const [eventSource, setEventSource] = useState<EventSource | null>(null);
const [limits, setLimits] = useState<ImportLimits>({
@@ -58,72 +59,10 @@ export function Settings() {
purchaseOrders: 0
});
// Helper to connect to event source
const connectToEventSource = useCallback((type: 'update' | 'import' | 'reset') => {
if (eventSource) {
eventSource.close();
}
console.log('Connecting to event source:', type); // Debug log
const source = new EventSource(`${config.apiUrl}/csv/${type}/progress`, {
withCredentials: true
});
setEventSource(source);
source.onopen = () => {
console.log('Event source connected:', type); // Debug log
};
source.onerror = () => {
console.log('Event source error:', type, source.readyState); // Debug log
if (source.readyState === EventSource.CLOSED) {
source.close();
setEventSource(null);
// Only reset states if we're not in a completed state
const progress = type === 'update' ? updateProgress :
type === 'import' ? importProgress :
resetProgress;
if (progress?.status !== 'complete') {
// Reset the appropriate state based on type
if (type === 'update') {
setIsUpdating(false);
setUpdateProgress(null);
} else if (type === 'import') {
setIsImporting(false);
setImportProgress(null);
} else if (type === 'reset') {
setIsResetting(false);
setResetProgress(null);
}
}
}
};
source.onmessage = (event) => {
try {
const data = JSON.parse(event.data);
let progressData = data.progress ?
(typeof data.progress === 'string' ? JSON.parse(data.progress) : data.progress)
: data;
// Update the appropriate progress state based on type
const setProgress = type === 'update' ? setUpdateProgress :
type === 'import' ? setImportProgress :
setResetProgress;
// Also set the operation state if we're getting progress
if (progressData.status === 'running') {
if (type === 'update') setIsUpdating(true);
else if (type === 'import') setIsImporting(true);
else if (type === 'reset') setIsResetting(true);
}
setProgress(prev => {
// If we're getting a new operation, clear out old messages
if (progressData.operation && progressData.operation !== prev?.operation) {
return {
// Helper function to update progress state
const updateProgressState = (progressData: any) => {
const operation = progressData.operation?.toLowerCase() || '';
const progressUpdate = {
status: progressData.status || 'running',
operation: progressData.operation,
current: progressData.current !== undefined ? Number(progressData.current) : undefined,
@@ -132,20 +71,55 @@ export function Settings() {
percentage: progressData.percentage,
elapsed: progressData.elapsed,
remaining: progressData.remaining,
message: progressData.message,
error: progressData.error,
message: progressData.message,
added: progressData.added,
updated: progressData.updated,
skipped: progressData.skipped,
duration: progressData.duration
};
if (operation.includes('products import completed')) {
setImportProgress(null);
} else if (operation.includes('products import')) {
setImportProgress(prev => ({ ...prev, ...progressUpdate }));
} else if (operation.includes('orders import') && !operation.includes('purchase')) {
setImportProgress(prev => ({ ...prev, ...progressUpdate }));
} else if (operation.includes('purchase orders import')) {
setPurchaseOrdersProgress(prev => ({ ...prev, ...progressUpdate }));
}
};
// Helper to connect to event source
const connectToEventSource = useCallback((type: 'update' | 'import' | 'reset') => {
if (eventSource) {
eventSource.close();
}
// Otherwise update existing state
return {
const source = new EventSource(`${config.apiUrl}/csv/${type}/progress`, {
withCredentials: true
});
setEventSource(source);
source.onmessage = (event) => {
try {
const data = JSON.parse(event.data);
let progressData = data.progress ?
(typeof data.progress === 'string' ? JSON.parse(data.progress) : data.progress)
: data;
// Handle different types of progress
if (type === 'import') {
updateProgressState(progressData);
} else {
// For non-import operations, use the existing logic
const setProgress = type === 'update' ? setUpdateProgress :
type === 'reset' ? setResetProgress :
setImportProgress;
setProgress(prev => ({
...prev,
status: progressData.status || prev?.status || 'running',
operation: progressData.operation || prev?.operation,
status: progressData.status || 'running',
operation: progressData.operation,
current: progressData.current !== undefined ? Number(progressData.current) : prev?.current,
total: progressData.total !== undefined ? Number(progressData.total) : prev?.total,
rate: progressData.rate !== undefined ? Number(progressData.rate) : prev?.rate,
@@ -158,8 +132,15 @@ export function Settings() {
updated: progressData.updated !== undefined ? progressData.updated : prev?.updated,
skipped: progressData.skipped !== undefined ? progressData.skipped : prev?.skipped,
duration: progressData.duration || prev?.duration
};
});
}));
}
// Set operation state if we're getting progress
if (progressData.status === 'running') {
if (type === 'update') setIsUpdating(true);
else if (type === 'import') setIsImporting(true);
else if (type === 'reset') setIsResetting(true);
}
if (progressData.status === 'complete') {
source.close();
@@ -172,6 +153,7 @@ export function Settings() {
} else if (type === 'import') {
setIsImporting(false);
setImportProgress(null);
setPurchaseOrdersProgress(null);
} else if (type === 'reset') {
setIsResetting(false);
setResetProgress(null);
@@ -196,7 +178,7 @@ export function Settings() {
handleError(`${type.charAt(0).toUpperCase() + type.slice(1)}`, progressData.error || 'Unknown error');
}
} catch (error) {
console.error('Error parsing event data:', error); // Debug log
console.error('Error parsing event data:', error);
}
};
}, []); // Remove dependencies that might prevent initial connection
@@ -339,7 +321,7 @@ export function Settings() {
setImportProgress({ status: 'running', operation: 'Starting import process' });
try {
// Connect to SSE for progress updates
// Connect to SSE for progress updates first
connectToEventSource('import');
// Make the import request
@@ -354,21 +336,24 @@ export function Settings() {
if (!response.ok) {
const data = await response.json().catch(() => ({}));
if (data.error === 'Import already in progress') {
// If there's already an import, just let the SSE connection handle showing progress
return;
}
// Only handle error if we don't have any progress yet
if (!importProgress?.current && !purchaseOrdersProgress?.current) {
throw new Error(data.error || 'Failed to start CSV import');
}
}
} catch (error) {
// Only clean up if we don't have any progress
if (!importProgress?.current && !purchaseOrdersProgress?.current) {
if (eventSource) {
eventSource.close();
setEventSource(null);
}
setIsImporting(false);
setImportProgress(null);
setPurchaseOrdersProgress(null);
handleError('Data import', error instanceof Error ? error.message : 'Unknown error');
}
}
};
const handleResetDB = async () => {
@@ -479,7 +464,11 @@ export function Settings() {
if (error.includes('cancelled') ||
error.includes('Process exited with code 143') ||
error.includes('Operation cancelled') ||
error.includes('500 Internal Server Error')) {
error.includes('500 Internal Server Error') ||
// Skip "Failed to start" errors if we have active progress
(error.includes('Failed to start CSV import') && (importProgress || purchaseOrdersProgress)) ||
// Skip connection errors if we have active progress
(error.includes('Failed to fetch') && (importProgress || purchaseOrdersProgress))) {
return;
}
toast.error(`${operation} failed: ${error}`);
@@ -615,7 +604,29 @@ export function Settings() {
)}
</div>
{isImporting && renderProgress(importProgress)}
{isImporting && (
<div className="space-y-4">
{/* Show products progress */}
{importProgress?.operation?.toLowerCase().includes('products') && (
<div>
{renderProgress(importProgress)}
</div>
)}
{/* Show orders progress */}
{importProgress?.operation?.toLowerCase().includes('orders import') &&
!importProgress.operation.toLowerCase().includes('purchase') && (
<div>
{renderProgress(importProgress)}
</div>
)}
{/* Show purchase orders progress */}
{purchaseOrdersProgress && (
<div>
{renderProgress(purchaseOrdersProgress)}
</div>
)}
</div>
)}
</CardContent>
</Card>