305 lines
10 KiB
JavaScript
305 lines
10 KiB
JavaScript
const dotenv = require("dotenv");
|
|
const path = require("path");
|
|
const { outputProgress, formatElapsedTime } = require('./metrics/utils/progress');
|
|
const { setupConnections, closeConnections } = require('./import/utils');
|
|
const importCategories = require('./import/categories');
|
|
const { importProducts } = require('./import/products');
|
|
const importOrders = require('./import/orders');
|
|
const importPurchaseOrders = require('./import/purchase-orders');
|
|
|
|
dotenv.config({ path: path.join(__dirname, "../.env") });
|
|
|
|
// Constants to control which imports run
|
|
const IMPORT_CATEGORIES = false;
|
|
const IMPORT_PRODUCTS = false;
|
|
const IMPORT_ORDERS = true;
|
|
const IMPORT_PURCHASE_ORDERS = false;
|
|
|
|
// Add flag for incremental updates
|
|
const INCREMENTAL_UPDATE = process.env.INCREMENTAL_UPDATE !== 'false'; // Default to true unless explicitly set to false
|
|
|
|
// SSH configuration
|
|
const sshConfig = {
|
|
ssh: {
|
|
host: process.env.PROD_SSH_HOST,
|
|
port: process.env.PROD_SSH_PORT || 22,
|
|
username: process.env.PROD_SSH_USER,
|
|
privateKey: process.env.PROD_SSH_KEY_PATH
|
|
? require("fs").readFileSync(process.env.PROD_SSH_KEY_PATH)
|
|
: undefined,
|
|
compress: true, // Enable SSH compression
|
|
},
|
|
prodDbConfig: {
|
|
// MySQL config for production
|
|
host: process.env.PROD_DB_HOST || "localhost",
|
|
user: process.env.PROD_DB_USER,
|
|
password: process.env.PROD_DB_PASSWORD,
|
|
database: process.env.PROD_DB_NAME,
|
|
port: process.env.PROD_DB_PORT || 3306,
|
|
timezone: 'Z',
|
|
},
|
|
localDbConfig: {
|
|
// PostgreSQL config for local
|
|
host: process.env.DB_HOST,
|
|
user: process.env.DB_USER,
|
|
password: process.env.DB_PASSWORD,
|
|
database: process.env.DB_NAME,
|
|
port: process.env.DB_PORT || 5432,
|
|
ssl: process.env.DB_SSL === 'true',
|
|
connectionTimeoutMillis: 60000,
|
|
idleTimeoutMillis: 30000,
|
|
max: 10 // connection pool max size
|
|
}
|
|
};
|
|
|
|
let isImportCancelled = false;
|
|
|
|
// Add cancel function
|
|
function cancelImport() {
|
|
isImportCancelled = true;
|
|
outputProgress({
|
|
status: 'cancelled',
|
|
operation: 'Import process',
|
|
message: 'Import cancelled by user',
|
|
current: 0,
|
|
total: 0,
|
|
elapsed: null,
|
|
remaining: null,
|
|
rate: 0
|
|
});
|
|
}
|
|
|
|
async function main() {
|
|
const startTime = Date.now();
|
|
let connections;
|
|
let completedSteps = 0;
|
|
let importHistoryId;
|
|
const totalSteps = [
|
|
IMPORT_CATEGORIES,
|
|
IMPORT_PRODUCTS,
|
|
IMPORT_ORDERS,
|
|
IMPORT_PURCHASE_ORDERS
|
|
].filter(Boolean).length;
|
|
|
|
try {
|
|
// Initial progress update
|
|
outputProgress({
|
|
status: "running",
|
|
operation: "Import process",
|
|
message: `Initializing SSH tunnel for ${INCREMENTAL_UPDATE ? 'incremental' : 'full'} import...`,
|
|
current: completedSteps,
|
|
total: totalSteps,
|
|
elapsed: formatElapsedTime(startTime)
|
|
});
|
|
|
|
connections = await setupConnections(sshConfig);
|
|
const { prodConnection, localConnection } = connections;
|
|
|
|
if (isImportCancelled) throw new Error("Import cancelled");
|
|
|
|
// Clean up any previously running imports that weren't completed
|
|
await localConnection.query(`
|
|
UPDATE import_history
|
|
SET
|
|
status = 'cancelled',
|
|
end_time = NOW(),
|
|
duration_seconds = EXTRACT(EPOCH FROM (NOW() - start_time))::INTEGER,
|
|
error_message = 'Previous import was not completed properly'
|
|
WHERE status = 'running'
|
|
`);
|
|
|
|
// Initialize sync_status table if it doesn't exist
|
|
await localConnection.query(`
|
|
CREATE TABLE IF NOT EXISTS sync_status (
|
|
table_name VARCHAR(50) PRIMARY KEY,
|
|
last_sync_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
last_sync_id BIGINT
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_last_sync ON sync_status (last_sync_timestamp);
|
|
`);
|
|
|
|
// Create import history record for the overall session
|
|
const [historyResult] = await localConnection.query(`
|
|
INSERT INTO import_history (
|
|
table_name,
|
|
start_time,
|
|
is_incremental,
|
|
status,
|
|
additional_info
|
|
) VALUES (
|
|
'all_tables',
|
|
NOW(),
|
|
$1::boolean,
|
|
'running',
|
|
jsonb_build_object(
|
|
'categories_enabled', $2::boolean,
|
|
'products_enabled', $3::boolean,
|
|
'orders_enabled', $4::boolean,
|
|
'purchase_orders_enabled', $5::boolean
|
|
)
|
|
) RETURNING id
|
|
`, [INCREMENTAL_UPDATE, IMPORT_CATEGORIES, IMPORT_PRODUCTS, IMPORT_ORDERS, IMPORT_PURCHASE_ORDERS]);
|
|
importHistoryId = historyResult.rows[0].id;
|
|
|
|
const results = {
|
|
categories: null,
|
|
products: null,
|
|
orders: null,
|
|
purchaseOrders: null
|
|
};
|
|
|
|
let totalRecordsAdded = 0;
|
|
let totalRecordsUpdated = 0;
|
|
|
|
// Run each import based on constants
|
|
if (IMPORT_CATEGORIES) {
|
|
results.categories = await importCategories(prodConnection, localConnection);
|
|
if (isImportCancelled) throw new Error("Import cancelled");
|
|
completedSteps++;
|
|
console.log('Categories import result:', results.categories);
|
|
totalRecordsAdded += results.categories?.recordsAdded || 0;
|
|
totalRecordsUpdated += results.categories?.recordsUpdated || 0;
|
|
}
|
|
|
|
if (IMPORT_PRODUCTS) {
|
|
results.products = await importProducts(prodConnection, localConnection, INCREMENTAL_UPDATE);
|
|
if (isImportCancelled) throw new Error("Import cancelled");
|
|
completedSteps++;
|
|
console.log('Products import result:', results.products);
|
|
totalRecordsAdded += results.products?.recordsAdded || 0;
|
|
totalRecordsUpdated += results.products?.recordsUpdated || 0;
|
|
}
|
|
|
|
if (IMPORT_ORDERS) {
|
|
results.orders = await importOrders(prodConnection, localConnection, INCREMENTAL_UPDATE);
|
|
if (isImportCancelled) throw new Error("Import cancelled");
|
|
completedSteps++;
|
|
console.log('Orders import result:', results.orders);
|
|
totalRecordsAdded += results.orders?.recordsAdded || 0;
|
|
totalRecordsUpdated += results.orders?.recordsUpdated || 0;
|
|
}
|
|
|
|
if (IMPORT_PURCHASE_ORDERS) {
|
|
results.purchaseOrders = await importPurchaseOrders(prodConnection, localConnection, INCREMENTAL_UPDATE);
|
|
if (isImportCancelled) throw new Error("Import cancelled");
|
|
completedSteps++;
|
|
console.log('Purchase orders import result:', results.purchaseOrders);
|
|
totalRecordsAdded += results.purchaseOrders?.recordsAdded || 0;
|
|
totalRecordsUpdated += results.purchaseOrders?.recordsUpdated || 0;
|
|
}
|
|
|
|
const endTime = Date.now();
|
|
const totalElapsedSeconds = Math.round((endTime - startTime) / 1000);
|
|
|
|
// Update import history with final stats
|
|
await localConnection.query(`
|
|
UPDATE import_history
|
|
SET
|
|
end_time = NOW(),
|
|
duration_seconds = $1,
|
|
records_added = $2,
|
|
records_updated = $3,
|
|
status = 'completed',
|
|
additional_info = jsonb_build_object(
|
|
'categories_enabled', $4::boolean,
|
|
'products_enabled', $5::boolean,
|
|
'orders_enabled', $6::boolean,
|
|
'purchase_orders_enabled', $7::boolean,
|
|
'categories_result', COALESCE($8::jsonb, 'null'::jsonb),
|
|
'products_result', COALESCE($9::jsonb, 'null'::jsonb),
|
|
'orders_result', COALESCE($10::jsonb, 'null'::jsonb),
|
|
'purchase_orders_result', COALESCE($11::jsonb, 'null'::jsonb)
|
|
)
|
|
WHERE id = $12
|
|
`, [
|
|
totalElapsedSeconds,
|
|
totalRecordsAdded,
|
|
totalRecordsUpdated,
|
|
IMPORT_CATEGORIES,
|
|
IMPORT_PRODUCTS,
|
|
IMPORT_ORDERS,
|
|
IMPORT_PURCHASE_ORDERS,
|
|
JSON.stringify(results.categories),
|
|
JSON.stringify(results.products),
|
|
JSON.stringify(results.orders),
|
|
JSON.stringify(results.purchaseOrders),
|
|
importHistoryId
|
|
]);
|
|
|
|
outputProgress({
|
|
status: "complete",
|
|
operation: "Import process",
|
|
message: `${INCREMENTAL_UPDATE ? 'Incremental' : 'Full'} import completed successfully in ${formatElapsedTime(totalElapsedSeconds)}`,
|
|
current: completedSteps,
|
|
total: totalSteps,
|
|
elapsed: formatElapsedTime(startTime),
|
|
timing: {
|
|
start_time: new Date(startTime).toISOString(),
|
|
end_time: new Date(endTime).toISOString(),
|
|
elapsed_time: formatElapsedTime(startTime),
|
|
elapsed_seconds: totalElapsedSeconds,
|
|
total_duration: formatElapsedTime(totalElapsedSeconds)
|
|
},
|
|
results
|
|
});
|
|
|
|
return results;
|
|
} catch (error) {
|
|
const endTime = Date.now();
|
|
const totalElapsedSeconds = Math.round((endTime - startTime) / 1000);
|
|
|
|
// Update import history with error
|
|
if (importHistoryId && connections?.localConnection) {
|
|
await connections.localConnection.query(`
|
|
UPDATE import_history
|
|
SET
|
|
end_time = NOW(),
|
|
duration_seconds = $1,
|
|
status = $2,
|
|
error_message = $3
|
|
WHERE id = $4
|
|
`, [totalElapsedSeconds, error.message === "Import cancelled" ? 'cancelled' : 'failed', error.message, importHistoryId]);
|
|
}
|
|
|
|
console.error("Error during import process:", error);
|
|
outputProgress({
|
|
status: error.message === "Import cancelled" ? "cancelled" : "error",
|
|
operation: "Import process",
|
|
message: error.message === "Import cancelled"
|
|
? `${INCREMENTAL_UPDATE ? 'Incremental' : 'Full'} import cancelled by user after ${formatElapsedTime(totalElapsedSeconds)}`
|
|
: `${INCREMENTAL_UPDATE ? 'Incremental' : 'Full'} import failed after ${formatElapsedTime(totalElapsedSeconds)}`,
|
|
error: error.message,
|
|
current: completedSteps,
|
|
total: totalSteps,
|
|
elapsed: formatElapsedTime(startTime),
|
|
timing: {
|
|
start_time: new Date(startTime).toISOString(),
|
|
end_time: new Date(endTime).toISOString(),
|
|
elapsed_time: formatElapsedTime(startTime),
|
|
elapsed_seconds: totalElapsedSeconds,
|
|
total_duration: formatElapsedTime(totalElapsedSeconds)
|
|
}
|
|
});
|
|
throw error;
|
|
} finally {
|
|
if (connections) {
|
|
await closeConnections(connections);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Run the import only if this is the main module
|
|
if (require.main === module) {
|
|
main().catch((error) => {
|
|
console.error("Unhandled error in main process:", error);
|
|
process.exit(1);
|
|
});
|
|
}
|
|
|
|
// Export the functions needed by the route
|
|
module.exports = {
|
|
main,
|
|
cancelImport,
|
|
};
|