matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update import tables schema with minor changes, add new metrics schema

Browse Source
This commit is contained in:
Matt
2025-03-29 16:46:31 -04:00
parent 0796518e26
commit f4854423ab
8 changed files with 380 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
inventory-server/scripts/import-from-prod.js
View File

@@ -10,9 +10,9 @@ const importPurchaseOrders = require('./import/purchase-orders');
dotenv.config({ path: path.join(__dirname, "../.env") });
// Constants to control which imports run
const IMPORT_CATEGORIES = true;
const IMPORT_PRODUCTS = true;
const IMPORT_ORDERS = true;
const IMPORT_CATEGORIES = false;
const IMPORT_PRODUCTS = false;
const IMPORT_ORDERS = false;
const IMPORT_PURCHASE_ORDERS = true;
// Add flag for incremental updates
@@ -111,8 +111,8 @@ async function main() {
// Initialize sync_status table if it doesn't exist
await localConnection.query(`
CREATE TABLE IF NOT EXISTS sync_status (
table_name VARCHAR(50) PRIMARY KEY,
last_sync_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
table_name TEXT PRIMARY KEY,
last_sync_timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
last_sync_id BIGINT
);

46
inventory-server/scripts/import/orders.js
View File

@@ -117,43 +117,43 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
CREATE TEMP TABLE temp_order_items (
order_id INTEGER NOT NULL,
pid INTEGER NOT NULL,
SKU VARCHAR(50) NOT NULL,
price DECIMAL(10,2) NOT NULL,
sku TEXT NOT NULL,
price NUMERIC(14, 4) NOT NULL,
quantity INTEGER NOT NULL,
base_discount DECIMAL(10,2) DEFAULT 0,
base_discount NUMERIC(14, 4) DEFAULT 0,
PRIMARY KEY (order_id, pid)
);
CREATE TEMP TABLE temp_order_meta (
order_id INTEGER NOT NULL,
date DATE NOT NULL,
customer VARCHAR(100) NOT NULL,
customer_name VARCHAR(150) NOT NULL,
status INTEGER,
date TIMESTAMP WITH TIME ZONE NOT NULL,
customer TEXT NOT NULL,
customer_name TEXT NOT NULL,
status TEXT,
canceled BOOLEAN,
summary_discount DECIMAL(10,2) DEFAULT 0.00,
summary_subtotal DECIMAL(10,2) DEFAULT 0.00,
summary_discount NUMERIC(14, 4) DEFAULT 0.0000,
summary_subtotal NUMERIC(14, 4) DEFAULT 0.0000,
PRIMARY KEY (order_id)
);
CREATE TEMP TABLE temp_order_discounts (
order_id INTEGER NOT NULL,
pid INTEGER NOT NULL,
discount DECIMAL(10,2) NOT NULL,
discount NUMERIC(14, 4) NOT NULL,
PRIMARY KEY (order_id, pid)
);
CREATE TEMP TABLE temp_order_taxes (
order_id INTEGER NOT NULL,
pid INTEGER NOT NULL,
tax DECIMAL(10,2) NOT NULL,
tax NUMERIC(14, 4) NOT NULL,
PRIMARY KEY (order_id, pid)
);
CREATE TEMP TABLE temp_order_costs (
order_id INTEGER NOT NULL,
pid INTEGER NOT NULL,
costeach DECIMAL(10,3) DEFAULT 0.000,
costeach NUMERIC(14, 4) DEFAULT 0.0000,
PRIMARY KEY (order_id, pid)
);
@@ -172,10 +172,10 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
]);
await localConnection.query(`
INSERT INTO temp_order_items (order_id, pid, SKU, price, quantity, base_discount)
INSERT INTO temp_order_items (order_id, pid, sku, price, quantity, base_discount)
VALUES ${placeholders}
ON CONFLICT (order_id, pid) DO UPDATE SET
SKU = EXCLUDED.SKU,
sku = EXCLUDED.sku,
price = EXCLUDED.price,
quantity = EXCLUDED.quantity,
base_discount = EXCLUDED.base_discount
@@ -241,10 +241,10 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
const values = subBatch.flatMap(order => [
order.order_id,
order.date,
new Date(order.date), // Convert to TIMESTAMP WITH TIME ZONE
order.customer,
toTitleCase(order.customer_name) || '',
order.status,
order.status.toString(), // Convert status to TEXT
order.canceled,
order.summary_discount || 0,
order.summary_subtotal || 0
@@ -447,7 +447,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
SELECT
oi.order_id as order_number,
oi.pid::bigint as pid,
oi.SKU as sku,
oi.sku,
om.date,
oi.price,
oi.quantity,
@@ -457,18 +457,18 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
WHEN om.summary_discount > 0 AND om.summary_subtotal > 0 THEN
ROUND((om.summary_discount * (oi.price * oi.quantity)) / NULLIF(om.summary_subtotal, 0), 2)
ELSE 0
END)::DECIMAL(10,2) as discount,
COALESCE(ot.total_tax, 0)::DECIMAL(10,2) as tax,
END)::NUMERIC(14, 4) as discount,
COALESCE(ot.total_tax, 0)::NUMERIC(14, 4) as tax,
false as tax_included,
0 as shipping,
om.customer,
om.customer_name,
om.status,
om.canceled,
COALESCE(ot.costeach, oi.price * 0.5)::DECIMAL(10,3) as costeach
COALESCE(ot.costeach, oi.price * 0.5)::NUMERIC(14, 4) as costeach
FROM (
SELECT DISTINCT ON (order_id, pid)
order_id, pid, SKU, price, quantity, base_discount
order_id, pid, sku, price, quantity, base_discount
FROM temp_order_items
WHERE order_id = ANY($1)
ORDER BY order_id, pid
@@ -508,7 +508,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
o.order_number,
o.pid,
o.sku || 'NO-SKU',
o.date,
o.date, // This is now a TIMESTAMP WITH TIME ZONE
o.price,
o.quantity,
o.discount,
@@ -517,7 +517,7 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
o.shipping,
o.customer,
o.customer_name,
o.status,
o.status.toString(), // Convert status to TEXT
o.canceled,
o.costeach
]);

60
inventory-server/scripts/import/products.js
View File

@@ -57,50 +57,50 @@ async function setupTemporaryTables(connection) {
await connection.query(`
CREATE TEMP TABLE temp_products (
pid BIGINT NOT NULL,
title VARCHAR(255),
title TEXT,
description TEXT,
sku VARCHAR(50),
sku TEXT,
stock_quantity INTEGER DEFAULT 0,
preorder_count INTEGER DEFAULT 0,
notions_inv_count INTEGER DEFAULT 0,
price DECIMAL(10,3) NOT NULL DEFAULT 0,
regular_price DECIMAL(10,3) NOT NULL DEFAULT 0,
cost_price DECIMAL(10,3),
vendor VARCHAR(100),
vendor_reference VARCHAR(100),
notions_reference VARCHAR(100),
brand VARCHAR(100),
line VARCHAR(100),
subline VARCHAR(100),
artist VARCHAR(100),
price NUMERIC(14, 4) NOT NULL DEFAULT 0,
regular_price NUMERIC(14, 4) NOT NULL DEFAULT 0,
cost_price NUMERIC(14, 4),
vendor TEXT,
vendor_reference TEXT,
notions_reference TEXT,
brand TEXT,
line TEXT,
subline TEXT,
artist TEXT,
categories TEXT,
created_at TIMESTAMP,
first_received TIMESTAMP,
landing_cost_price DECIMAL(10,3),
barcode VARCHAR(50),
harmonized_tariff_code VARCHAR(50),
updated_at TIMESTAMP,
created_at TIMESTAMP WITH TIME ZONE,
first_received TIMESTAMP WITH TIME ZONE,
landing_cost_price NUMERIC(14, 4),
barcode TEXT,
harmonized_tariff_code TEXT,
updated_at TIMESTAMP WITH TIME ZONE,
visible BOOLEAN,
managing_stock BOOLEAN DEFAULT true,
replenishable BOOLEAN,
permalink VARCHAR(255),
permalink TEXT,
moq INTEGER DEFAULT 1,
uom INTEGER DEFAULT 1,
rating DECIMAL(10,2),
rating NUMERIC(14, 4),
reviews INTEGER,
weight DECIMAL(10,3),
length DECIMAL(10,3),
width DECIMAL(10,3),
height DECIMAL(10,3),
country_of_origin VARCHAR(100),
location VARCHAR(100),
weight NUMERIC(14, 4),
length NUMERIC(14, 4),
width NUMERIC(14, 4),
height NUMERIC(14, 4),
country_of_origin TEXT,
location TEXT,
total_sold INTEGER,
baskets INTEGER,
notifies INTEGER,
date_last_sold TIMESTAMP,
image VARCHAR(255),
image_175 VARCHAR(255),
image_full VARCHAR(255),
date_last_sold TIMESTAMP WITH TIME ZONE,
image TEXT,
image_175 TEXT,
image_full TEXT,
options TEXT,
tags TEXT,
needs_update BOOLEAN DEFAULT TRUE,

104
inventory-server/scripts/import/purchase-orders.js
View File

@@ -73,19 +73,18 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
-- Temporary table for purchase orders
CREATE TEMP TABLE temp_purchase_orders (
po_id VARCHAR(50) NOT NULL,
po_id TEXT NOT NULL,
pid BIGINT NOT NULL,
sku VARCHAR(50),
name VARCHAR(255),
vendor VARCHAR(255),
sku TEXT,
name TEXT,
vendor TEXT,
date TIMESTAMP WITH TIME ZONE,
expected_date DATE,
status INTEGER,
status_text VARCHAR(50),
status TEXT,
notes TEXT,
long_note TEXT,
ordered INTEGER,
po_cost_price DECIMAL(10,3),
po_cost_price NUMERIC(14, 4),
supplier_id INTEGER,
date_created TIMESTAMP WITH TIME ZONE,
date_ordered TIMESTAMP WITH TIME ZONE,
@@ -94,27 +93,26 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
-- Temporary table for receivings
CREATE TEMP TABLE temp_receivings (
receiving_id VARCHAR(50) NOT NULL,
po_id VARCHAR(50),
receiving_id TEXT NOT NULL,
po_id TEXT,
pid BIGINT NOT NULL,
qty_each INTEGER,
cost_each DECIMAL(10,5),
cost_each NUMERIC(14, 4),
received_by INTEGER,
received_date TIMESTAMP WITH TIME ZONE,
receiving_created_date TIMESTAMP WITH TIME ZONE,
supplier_id INTEGER,
status INTEGER,
status_text VARCHAR(50),
status TEXT,
PRIMARY KEY (receiving_id, pid)
);
-- Temporary table for tracking FIFO allocations
CREATE TEMP TABLE temp_receiving_allocations (
po_id VARCHAR(50) NOT NULL,
po_id TEXT NOT NULL,
pid BIGINT NOT NULL,
receiving_id VARCHAR(50) NOT NULL,
receiving_id TEXT NOT NULL,
allocated_qty INTEGER NOT NULL,
cost_each DECIMAL(10,5) NOT NULL,
cost_each NUMERIC(14, 4) NOT NULL,
received_date TIMESTAMP WITH TIME ZONE NOT NULL,
received_by INTEGER,
PRIMARY KEY (po_id, pid, receiving_id)
@@ -123,8 +121,8 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
-- Temporary table for employee names
CREATE TEMP TABLE employee_names (
employeeid INTEGER PRIMARY KEY,
firstname VARCHAR(100),
lastname VARCHAR(100)
firstname TEXT,
lastname TEXT
);
-- Create indexes for efficient joins
@@ -135,22 +133,22 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
// Map status codes to text values
const poStatusMap = {
0: 'Canceled',
1: 'Created',
10: 'Ready ESend',
11: 'Ordered',
12: 'Preordered',
13: 'Electronically Sent',
15: 'Receiving Started',
50: 'Done'
0: 'canceled',
1: 'created',
10: 'electronically_ready_send',
11: 'ordered',
12: 'preordered',
13: 'electronically_sent',
15: 'receiving_started',
50: 'done'
};
const receivingStatusMap = {
0: 'Canceled',
1: 'Created',
30: 'Partial Received',
40: 'Full Received',
50: 'Paid'
0: 'canceled',
1: 'created',
30: 'partial_received',
40: 'full_received',
50: 'paid'
};
// Get time window for data retrieval
@@ -281,8 +279,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
vendor: po.vendor || 'Unknown Vendor',
date: validateDate(po.date_ordered) || validateDate(po.date_created),
expected_date: validateDate(po.date_estin),
status: po.status,
status_text: poStatusMap[po.status] || '',
status: poStatusMap[po.status] || 'created',
notes: po.notes || '',
long_note: po.long_note || '',
ordered: product.qty_each,
@@ -298,8 +295,8 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
const batch = completePOs.slice(i, i + INSERT_BATCH_SIZE);
const placeholders = batch.map((_, idx) => {
const base = idx * 16;
return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6}, $${base + 7}, $${base + 8}, $${base + 9}, $${base + 10}, $${base + 11}, $${base + 12}, $${base + 13}, $${base + 14}, $${base + 15}, $${base + 16})`;
const base = idx * 15;
return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6}, $${base + 7}, $${base + 8}, $${base + 9}, $${base + 10}, $${base + 11}, $${base + 12}, $${base + 13}, $${base + 14}, $${base + 15})`;
}).join(',');
const values = batch.flatMap(po => [
@@ -311,7 +308,6 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
po.date,
po.expected_date,
po.status,
po.status_text,
po.notes,
po.long_note,
po.ordered,
@@ -323,8 +319,8 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
await localConnection.query(`
INSERT INTO temp_purchase_orders (
po_id, pid, sku, name, vendor, date, expected_date, status, status_text,
notes, long_note, ordered, po_cost_price, supplier_id, date_created, date_ordered
po_id, pid, sku, name, vendor, date, expected_date, status, notes, long_note,
ordered, po_cost_price, supplier_id, date_created, date_ordered
)
VALUES ${placeholders}
ON CONFLICT (po_id, pid) DO UPDATE SET
@@ -334,7 +330,6 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
date = EXCLUDED.date,
expected_date = EXCLUDED.expected_date,
status = EXCLUDED.status,
status_text = EXCLUDED.status_text,
notes = EXCLUDED.notes,
long_note = EXCLUDED.long_note,
ordered = EXCLUDED.ordered,
@@ -448,9 +443,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
received_date: validateDate(product.received_date) || validateDate(product.receiving_created_date),
receiving_created_date: validateDate(product.receiving_created_date),
supplier_id: receiving.supplier_id,
status: receiving.status,
status_text: receivingStatusMap[receiving.status] || '',
receiving_created_date: validateDate(product.receiving_created_date)
status: receivingStatusMap[receiving.status] || 'created'
});
}
@@ -459,8 +452,8 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
const batch = completeReceivings.slice(i, i + INSERT_BATCH_SIZE);
const placeholders = batch.map((_, idx) => {
const base = idx * 11;
return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6}, $${base + 7}, $${base + 8}, $${base + 9}, $${base + 10}, $${base + 11})`;
const base = idx * 10;
return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6}, $${base + 7}, $${base + 8}, $${base + 9}, $${base + 10})`;
}).join(',');
const values = batch.flatMap(r => [
@@ -473,14 +466,13 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
r.received_date,
r.receiving_created_date,
r.supplier_id,
r.status,
r.status_text
r.status
]);
await localConnection.query(`
INSERT INTO temp_receivings (
receiving_id, po_id, pid, qty_each, cost_each, received_by,
received_date, receiving_created_date, supplier_id, status, status_text
received_date, receiving_created_date, supplier_id, status
)
VALUES ${placeholders}
ON CONFLICT (receiving_id, pid) DO UPDATE SET
@@ -491,8 +483,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
received_date = EXCLUDED.received_date,
receiving_created_date = EXCLUDED.receiving_created_date,
supplier_id = EXCLUDED.supplier_id,
status = EXCLUDED.status,
status_text = EXCLUDED.status_text
status = EXCLUDED.status
`, values);
}
@@ -586,7 +577,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
name: "Handling standalone receivings",
query: `
INSERT INTO temp_purchase_orders (
po_id, pid, sku, name, vendor, date, status, status_text,
po_id, pid, sku, name, vendor, date, status,
ordered, po_cost_price, supplier_id, date_created, date_ordered
)
SELECT
@@ -600,8 +591,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
'Unknown Vendor'
) as vendor,
COALESCE(r.received_date, r.receiving_created_date) as date,
NULL as status,
NULL as status_text,
'created' as status,
NULL as ordered,
r.cost_each as po_cost_price,
r.supplier_id,
@@ -872,13 +862,13 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
po.name,
COALESCE(ca.avg_cost, po.po_cost_price) as cost_price,
po.po_cost_price,
CASE WHEN po.status IS NULL THEN 1 ELSE po.status END as status,
COALESCE(po.status, 'created'),
CASE
WHEN rs.total_received IS NULL THEN 1
WHEN rs.total_received = 0 THEN 1
WHEN rs.total_received < po.ordered THEN 30
WHEN rs.total_received >= po.ordered THEN 40
ELSE 1
WHEN rs.total_received IS NULL THEN 'created'
WHEN rs.total_received = 0 THEN 'created'
WHEN rs.total_received < po.ordered THEN 'partial_received'
WHEN rs.total_received >= po.ordered THEN 'full_received'
ELSE 'created'
END as receiving_status,
po.notes,
po.long_note,