matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Try to synchronize time zones across import

Browse Source
This commit is contained in:
Matt
2025-04-05 16:20:43 -04:00
parent c9b656d34b
commit a4c1a19d2e
4 changed files with 252 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
inventory-server/db/schema.sql
View File

@@ -213,55 +213,55 @@ SET session_replication_role = 'origin'; -- Re-enable foreign key checks
-- Create views for common calculations -- Create views for common calculations
-- product_sales_trends view moved to metrics-schema.sql -- product_sales_trends view moved to metrics-schema.sql
-- Historical data tables imported from production -- -- Historical data tables imported from production
CREATE TABLE imported_product_current_prices ( -- CREATE TABLE imported_product_current_prices (
price_id BIGSERIAL PRIMARY KEY, -- price_id BIGSERIAL PRIMARY KEY,
pid BIGINT NOT NULL, -- pid BIGINT NOT NULL,
qty_buy SMALLINT NOT NULL, -- qty_buy SMALLINT NOT NULL,
is_min_qty_buy BOOLEAN NOT NULL, -- is_min_qty_buy BOOLEAN NOT NULL,
price_each NUMERIC(10,3) NOT NULL, -- price_each NUMERIC(10,3) NOT NULL,
qty_limit SMALLINT NOT NULL, -- qty_limit SMALLINT NOT NULL,
no_promo BOOLEAN NOT NULL, -- no_promo BOOLEAN NOT NULL,
checkout_offer BOOLEAN NOT NULL, -- checkout_offer BOOLEAN NOT NULL,
active BOOLEAN NOT NULL, -- active BOOLEAN NOT NULL,
date_active TIMESTAMP WITH TIME ZONE, -- date_active TIMESTAMP WITH TIME ZONE,
date_deactive TIMESTAMP WITH TIME ZONE, -- date_deactive TIMESTAMP WITH TIME ZONE,
updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP -- updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
); -- );
CREATE INDEX idx_imported_product_current_prices_pid ON imported_product_current_prices(pid, active, qty_buy); -- CREATE INDEX idx_imported_product_current_prices_pid ON imported_product_current_prices(pid, active, qty_buy);
CREATE INDEX idx_imported_product_current_prices_checkout ON imported_product_current_prices(checkout_offer, active); -- CREATE INDEX idx_imported_product_current_prices_checkout ON imported_product_current_prices(checkout_offer, active);
CREATE INDEX idx_imported_product_current_prices_deactive ON imported_product_current_prices(date_deactive, active); -- CREATE INDEX idx_imported_product_current_prices_deactive ON imported_product_current_prices(date_deactive, active);
CREATE INDEX idx_imported_product_current_prices_active ON imported_product_current_prices(date_active, active); -- CREATE INDEX idx_imported_product_current_prices_active ON imported_product_current_prices(date_active, active);
CREATE TABLE imported_daily_inventory ( -- CREATE TABLE imported_daily_inventory (
date DATE NOT NULL, -- date DATE NOT NULL,
pid BIGINT NOT NULL, -- pid BIGINT NOT NULL,
amountsold SMALLINT NOT NULL DEFAULT 0, -- amountsold SMALLINT NOT NULL DEFAULT 0,
times_sold SMALLINT NOT NULL DEFAULT 0, -- times_sold SMALLINT NOT NULL DEFAULT 0,
qtyreceived SMALLINT NOT NULL DEFAULT 0, -- qtyreceived SMALLINT NOT NULL DEFAULT 0,
price NUMERIC(7,2) NOT NULL DEFAULT 0, -- price NUMERIC(7,2) NOT NULL DEFAULT 0,
costeach NUMERIC(7,2) NOT NULL DEFAULT 0, -- costeach NUMERIC(7,2) NOT NULL DEFAULT 0,
stamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, -- stamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, -- updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (date, pid) -- PRIMARY KEY (date, pid)
); -- );
CREATE INDEX idx_imported_daily_inventory_pid ON imported_daily_inventory(pid); -- CREATE INDEX idx_imported_daily_inventory_pid ON imported_daily_inventory(pid);
CREATE TABLE imported_product_stat_history ( -- CREATE TABLE imported_product_stat_history (
pid BIGINT NOT NULL, -- pid BIGINT NOT NULL,
date DATE NOT NULL, -- date DATE NOT NULL,
score NUMERIC(10,2) NOT NULL, -- score NUMERIC(10,2) NOT NULL,
score2 NUMERIC(10,2) NOT NULL, -- score2 NUMERIC(10,2) NOT NULL,
qty_in_baskets SMALLINT NOT NULL, -- qty_in_baskets SMALLINT NOT NULL,
qty_sold SMALLINT NOT NULL, -- qty_sold SMALLINT NOT NULL,
notifies_set SMALLINT NOT NULL, -- notifies_set SMALLINT NOT NULL,
visibility_score NUMERIC(10,2) NOT NULL, -- visibility_score NUMERIC(10,2) NOT NULL,
health_score VARCHAR(5) NOT NULL, -- health_score VARCHAR(5) NOT NULL,
sold_view_score NUMERIC(6,3) NOT NULL, -- sold_view_score NUMERIC(6,3) NOT NULL,
updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, -- updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (pid, date) -- PRIMARY KEY (pid, date)
); -- );
CREATE INDEX idx_imported_product_stat_history_date ON imported_product_stat_history(date); -- CREATE INDEX idx_imported_product_stat_history_date ON imported_product_stat_history(date);

2
inventory-server/scripts/import-from-prod.js
View File

@@ -38,7 +38,7 @@ const sshConfig = {
password: process.env.PROD_DB_PASSWORD, password: process.env.PROD_DB_PASSWORD,
database: process.env.PROD_DB_NAME, database: process.env.PROD_DB_NAME,
port: process.env.PROD_DB_PORT || 3306, port: process.env.PROD_DB_PORT || 3306,
timezone: 'Z', timezone: '-05:00', // Production DB always stores times in EST (UTC-5) regardless of DST
}, },
localDbConfig: { localDbConfig: {
// PostgreSQL config for local // PostgreSQL config for local

309
inventory-server/scripts/reset-db.js
View File

@@ -13,6 +13,22 @@ const dbConfig = {
port: process.env.DB_PORT || 5432 port: process.env.DB_PORT || 5432
}; };
// Tables to always protect from being dropped
const PROTECTED_TABLES = [
'users',
'permissions',
'user_permissions',
'calculate_history',
'import_history',
'ai_prompts',
'ai_validation_performance',
'templates',
'reusable_images',
'imported_daily_inventory',
'imported_product_stat_history',
'imported_product_current_prices'
];
// Helper function to output progress in JSON format // Helper function to output progress in JSON format
function outputProgress(data) { function outputProgress(data) {
if (!data.status) { if (!data.status) {
@@ -33,17 +49,6 @@ const CORE_TABLES = [
'product_categories' 'product_categories'
]; ];
// Config tables that must be created
const CONFIG_TABLES = [
'stock_thresholds',
'lead_time_thresholds',
'sales_velocity_config',
'abc_classification_config',
'safety_stock_config',
'sales_seasonality',
'turnover_config'
];
// Split SQL into individual statements // Split SQL into individual statements
function splitSQLStatements(sql) { function splitSQLStatements(sql) {
// First, normalize line endings // First, normalize line endings
@@ -184,8 +189,8 @@ async function resetDatabase() {
SELECT string_agg(tablename, ', ') as tables SELECT string_agg(tablename, ', ') as tables
FROM pg_tables FROM pg_tables
WHERE schemaname = 'public' WHERE schemaname = 'public'
AND tablename NOT IN ('users', 'permissions', 'user_permissions', 'calculate_history', 'import_history', 'ai_prompts', 'ai_validation_performance', 'templates', 'reusable_images'); AND tablename NOT IN (SELECT unnest($1::text[]));
`); `, [PROTECTED_TABLES]);
if (!tablesResult.rows[0].tables) { if (!tablesResult.rows[0].tables) {
outputProgress({ outputProgress({
@@ -204,7 +209,7 @@ async function resetDatabase() {
// Drop all tables except users // Drop all tables except users
const tables = tablesResult.rows[0].tables.split(', '); const tables = tablesResult.rows[0].tables.split(', ');
for (const table of tables) { for (const table of tables) {
if (!['users', 'reusable_images'].includes(table)) { if (!PROTECTED_TABLES.includes(table)) {
await client.query(`DROP TABLE IF EXISTS "${table}" CASCADE`); await client.query(`DROP TABLE IF EXISTS "${table}" CASCADE`);
} }
} }
@@ -259,7 +264,9 @@ async function resetDatabase() {
'category_metrics', 'category_metrics',
'brand_metrics', 'brand_metrics',
'sales_forecasts', 'sales_forecasts',
'abc_classification' 'abc_classification',
'daily_snapshots',
'periodic_metrics'
) )
`); `);
} }
@@ -301,51 +308,67 @@ async function resetDatabase() {
} }
}); });
for (let i = 0; i < statements.length; i++) { // Start a transaction for better error handling
const stmt = statements[i]; await client.query('BEGIN');
try { try {
const result = await client.query(stmt); for (let i = 0; i < statements.length; i++) {
const stmt = statements[i];
// Verify if table was created (if this was a CREATE TABLE statement) try {
if (stmt.trim().toLowerCase().startsWith('create table')) { const result = await client.query(stmt);
const tableName = stmt.match(/create\s+table\s+(?:if\s+not\s+exists\s+)?["]?(\w+)["]?/i)?.[1];
if (tableName) { // Verify if table was created (if this was a CREATE TABLE statement)
const tableExists = await client.query(` if (stmt.trim().toLowerCase().startsWith('create table')) {
SELECT COUNT(*) as count const tableName = stmt.match(/create\s+table\s+(?:if\s+not\s+exists\s+)?["]?(\w+)["]?/i)?.[1];
FROM information_schema.tables if (tableName) {
WHERE table_schema = 'public' const tableExists = await client.query(`
AND table_name = $1 SELECT COUNT(*) as count
`, [tableName]); FROM information_schema.tables
WHERE table_schema = 'public'
outputProgress({ AND table_name = $1
operation: 'Table Creation Verification', `, [tableName]);
message: {
table: tableName, outputProgress({
exists: tableExists.rows[0].count > 0 operation: 'Table Creation Verification',
} message: {
}); table: tableName,
exists: tableExists.rows[0].count > 0
}
});
}
} }
outputProgress({
operation: 'SQL Progress',
message: {
statement: i + 1,
total: statements.length,
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : ''),
rowCount: result.rowCount
}
});
// Commit in chunks of 10 statements to avoid long-running transactions
if (i > 0 && i % 10 === 0) {
await client.query('COMMIT');
await client.query('BEGIN');
}
} catch (sqlError) {
await client.query('ROLLBACK');
outputProgress({
status: 'error',
operation: 'SQL Error',
error: sqlError.message,
statement: stmt,
statementNumber: i + 1
});
throw sqlError;
} }
outputProgress({
operation: 'SQL Progress',
message: {
statement: i + 1,
total: statements.length,
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : ''),
rowCount: result.rowCount
}
});
} catch (sqlError) {
outputProgress({
status: 'error',
operation: 'SQL Error',
error: sqlError.message,
statement: stmt,
statementNumber: i + 1
});
throw sqlError;
} }
// Commit the final transaction
await client.query('COMMIT');
} catch (error) {
await client.query('ROLLBACK');
throw error;
} }
// Verify core tables were created // Verify core tables were created
@@ -383,11 +406,25 @@ async function resetDatabase() {
operation: 'Running config setup', operation: 'Running config setup',
message: 'Creating configuration tables...' message: 'Creating configuration tables...'
}); });
const configSchemaSQL = fs.readFileSync( const configSchemaPath = path.join(__dirname, '../db/config-schema-new.sql');
path.join(__dirname, '../db/config-schema-new.sql'),
'utf8'
);
// Verify file exists
if (!fs.existsSync(configSchemaPath)) {
throw new Error(`Config schema file not found at: ${configSchemaPath}`);
}
const configSchemaSQL = fs.readFileSync(configSchemaPath, 'utf8');
outputProgress({
operation: 'Config Schema file',
message: {
path: configSchemaPath,
exists: fs.existsSync(configSchemaPath),
size: fs.statSync(configSchemaPath).size,
firstFewLines: configSchemaSQL.split('\n').slice(0, 5).join('\n')
}
});
// Execute config schema statements one at a time // Execute config schema statements one at a time
const configStatements = splitSQLStatements(configSchemaSQL); const configStatements = splitSQLStatements(configSchemaSQL);
outputProgress({ outputProgress({
@@ -401,30 +438,46 @@ async function resetDatabase() {
} }
}); });
for (let i = 0; i < configStatements.length; i++) { // Start a transaction for better error handling
const stmt = configStatements[i]; await client.query('BEGIN');
try { try {
const result = await client.query(stmt); for (let i = 0; i < configStatements.length; i++) {
const stmt = configStatements[i];
outputProgress({ try {
operation: 'Config SQL Progress', const result = await client.query(stmt);
message: {
statement: i + 1, outputProgress({
total: configStatements.length, operation: 'Config SQL Progress',
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : ''), message: {
rowCount: result.rowCount statement: i + 1,
total: configStatements.length,
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : ''),
rowCount: result.rowCount
}
});
// Commit in chunks of 10 statements to avoid long-running transactions
if (i > 0 && i % 10 === 0) {
await client.query('COMMIT');
await client.query('BEGIN');
} }
}); } catch (sqlError) {
} catch (sqlError) { await client.query('ROLLBACK');
outputProgress({ outputProgress({
status: 'error', status: 'error',
operation: 'Config SQL Error', operation: 'Config SQL Error',
error: sqlError.message, error: sqlError.message,
statement: stmt, statement: stmt,
statementNumber: i + 1 statementNumber: i + 1
}); });
throw sqlError; throw sqlError;
}
} }
// Commit the final transaction
await client.query('COMMIT');
} catch (error) {
await client.query('ROLLBACK');
throw error;
} }
// Read and execute metrics schema (metrics tables) // Read and execute metrics schema (metrics tables)
@@ -432,11 +485,25 @@ async function resetDatabase() {
operation: 'Running metrics setup', operation: 'Running metrics setup',
message: 'Creating metrics tables...' message: 'Creating metrics tables...'
}); });
const metricsSchemaSQL = fs.readFileSync( const metricsSchemaPath = path.join(__dirname, '../db/metrics-schema-new.sql');
path.join(__dirname, '../db/metrics-schema-new.sql'),
'utf8'
);
// Verify file exists
if (!fs.existsSync(metricsSchemaPath)) {
throw new Error(`Metrics schema file not found at: ${metricsSchemaPath}`);
}
const metricsSchemaSQL = fs.readFileSync(metricsSchemaPath, 'utf8');
outputProgress({
operation: 'Metrics Schema file',
message: {
path: metricsSchemaPath,
exists: fs.existsSync(metricsSchemaPath),
size: fs.statSync(metricsSchemaPath).size,
firstFewLines: metricsSchemaSQL.split('\n').slice(0, 5).join('\n')
}
});
// Execute metrics schema statements one at a time // Execute metrics schema statements one at a time
const metricsStatements = splitSQLStatements(metricsSchemaSQL); const metricsStatements = splitSQLStatements(metricsSchemaSQL);
outputProgress({ outputProgress({
@@ -450,30 +517,46 @@ async function resetDatabase() {
} }
}); });
for (let i = 0; i < metricsStatements.length; i++) { // Start a transaction for better error handling
const stmt = metricsStatements[i]; await client.query('BEGIN');
try { try {
const result = await client.query(stmt); for (let i = 0; i < metricsStatements.length; i++) {
const stmt = metricsStatements[i];
outputProgress({ try {
operation: 'Metrics SQL Progress', const result = await client.query(stmt);
message: {
statement: i + 1, outputProgress({
total: metricsStatements.length, operation: 'Metrics SQL Progress',
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : ''), message: {
rowCount: result.rowCount statement: i + 1,
total: metricsStatements.length,
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : ''),
rowCount: result.rowCount
}
});
// Commit in chunks of 10 statements to avoid long-running transactions
if (i > 0 && i % 10 === 0) {
await client.query('COMMIT');
await client.query('BEGIN');
} }
}); } catch (sqlError) {
} catch (sqlError) { await client.query('ROLLBACK');
outputProgress({ outputProgress({
status: 'error', status: 'error',
operation: 'Metrics SQL Error', operation: 'Metrics SQL Error',
error: sqlError.message, error: sqlError.message,
statement: stmt, statement: stmt,
statementNumber: i + 1 statementNumber: i + 1
}); });
throw sqlError; throw sqlError;
}
} }
// Commit the final transaction
await client.query('COMMIT');
} catch (error) {
await client.query('ROLLBACK');
throw error;
} }
outputProgress({ outputProgress({
@@ -490,6 +573,14 @@ async function resetDatabase() {
}); });
process.exit(1); process.exit(1);
} finally { } finally {
// Make sure to re-enable foreign key checks if they were disabled
try {
await client.query('SET session_replication_role = \'origin\'');
} catch (e) {
console.error('Error re-enabling foreign key checks:', e.message);
}
// Close the database connection
await client.end(); await client.end();
} }
} }

5
inventory-server/scripts/reset-metrics-new.js
View File

@@ -31,7 +31,10 @@ const PROTECTED_TABLES = [
'ai_prompts', 'ai_prompts',
'ai_validation_performance', 'ai_validation_performance',
'templates', 'templates',
'reusable_images' 'reusable_images',
'imported_daily_inventory',
'imported_product_stat_history',
'imported_product_current_prices'
]; ];
// Split SQL into individual statements // Split SQL into individual statements