Compare commits
307 Commits
Improve-da
...
6aefc1b40d
| Author | SHA1 | Date | |
|---|---|---|---|
| 6aefc1b40d | |||
| 7c41a7f799 | |||
| 12cc7a4639 | |||
| 9b2f9016f6 | |||
| 8044771301 | |||
| b5469440bf | |||
| fd14af0f9e | |||
| a703019b0b | |||
| 2744e82264 | |||
| 450fd96e19 | |||
| 4372dc5e26 | |||
| dd0e989669 | |||
| 89d518b57f | |||
| ac39257a51 | |||
| 003e1ddd61 | |||
| 2dc8152b53 | |||
| 01d4097030 | |||
| f9e8c9265e | |||
| ee2f314775 | |||
| 11d0555eeb | |||
| ec8ab17d3f | |||
| 100e398aae | |||
| aec02e490a | |||
| 3831cef234 | |||
| 1866cbae7e | |||
| 3d1e8862f9 | |||
| 1dcb47cfc5 | |||
| 167c13c572 | |||
| 7218e7cc3f | |||
| 43d76e011d | |||
| 9ce84fe5b9 | |||
| d15360a7d4 | |||
| 630945e901 | |||
| 54ddaa0492 | |||
| 262890a7be | |||
| ef50aec33c | |||
| 0ffd02e22e | |||
| 738ed94ad5 | |||
| f5b2b4e421 | |||
| b81dfb9649 | |||
| 9be0f34f07 | |||
| ad5b797ce6 | |||
| 78932360d1 | |||
| 217abd41af | |||
| d56beb5143 | |||
| 0b5f3162c7 | |||
| 72930bbc73 | |||
| 0ceef144d7 | |||
| f0e2023803 | |||
| 0a20d74bb6 | |||
| 9761c29934 | |||
| e84c7e568f | |||
| 4953355b91 | |||
| dadcf3b6c6 | |||
| 920c33d119 | |||
| 451d5f0b3b | |||
| dd79298b94 | |||
| 7b7274f72c | |||
| 60875c25a6 | |||
| e10df632d8 | |||
| 945e4a8cc3 | |||
| c6e4fc9cff | |||
| ff17b290aa | |||
| 6bffcfb0a4 | |||
| 2c5255cd13 | |||
| 1696ecf591 | |||
| dc774862a7 | |||
| d3e3cba087 | |||
| 4ea3a4aec3 | |||
| a161f4533d | |||
| 6e30ba60ff | |||
| 138251cf86 | |||
| 24aee1db90 | |||
| 2fe7fd5b2f | |||
| d8b39979cd | |||
| 4776a112b6 | |||
| 2ff325a132 | |||
| 5d46a2a7e5 | |||
| 512b351429 | |||
| 3991341376 | |||
| 5833779c10 | |||
| c61115f665 | |||
| 7da2b304b4 | |||
| 4ccda8ad49 | |||
| 88f703ec70 | |||
| ab998fb7c4 | |||
| faaa8cc47a | |||
| 459c5092d2 | |||
| 6c9fd062e9 | |||
| 5d7d7a8671 | |||
| 54f55b06a1 | |||
| 4935cfe3bb | |||
| 5e2ee73e2d | |||
| 4dfe85231a | |||
| 9e7aac836e | |||
| d35c7dd6cf | |||
| ad1ebeefe1 | |||
| a0c442d1af | |||
| 7938c50762 | |||
| 5dcd19e7f3 | |||
| 075e7253a0 | |||
| 763aa4f74b | |||
| 520ff5bd74 | |||
| 8496bbc4ee | |||
| 38f6688f10 | |||
| fcfe7e2fab | |||
| 2e3e81a02b | |||
| 8606a90e34 | |||
| a97819f4a6 | |||
| dd82c624d8 | |||
| 7999e1e64a | |||
| 12a0f540b3 | |||
| e793cb0cc5 | |||
| b2330dee22 | |||
| 00501704df | |||
| 4cb41a7e4c | |||
| d05d27494d | |||
| 4ed734e5c0 | |||
| 1e3be5d4cb | |||
| 8dd852dd6a | |||
| eeff5817ea | |||
| 1b19feb172 | |||
| 80ff8124ec | |||
| 8508bfac93 | |||
| ac14179bd2 | |||
| 00249f7c33 | |||
| f271f3aae4 | |||
| 43f76e4ac0 | |||
| 92ff80fba2 | |||
| a4c1a19d2e | |||
| c9b656d34b | |||
| d081a60662 | |||
| 4021fe487d | |||
| 4552fa4862 | |||
| 2601a04211 | |||
| 6051b849d6 | |||
| dbd0232285 | |||
| 1b9f01d101 | |||
| a9dbbbf824 | |||
| 97296946f1 | |||
| 5035dda733 | |||
| 796a2e5d1f | |||
| 047122a620 | |||
| 4c4359908c | |||
| 54cc4be1e3 | |||
| f4854423ab | |||
| 0796518e26 | |||
| 7aa494aaad | |||
| 1e0be3f86e | |||
| a068a253cd | |||
| 087ec710f6 | |||
| 957c7b5eb1 | |||
| 8b8845b423 | |||
| e5c4f617c5 | |||
| 8e19e6cd74 | |||
| 749907bd30 | |||
| 108181c63d | |||
| 5dd779cb4a | |||
| 7b0e792d03 | |||
| 517bbe72f4 | |||
| 87d4b9e804 | |||
| 75da2c6772 | |||
| 00a02aa788 | |||
| 114018080a | |||
| 228ae8b2a9 | |||
| dd4b3f7145 | |||
| 7eb4077224 | |||
| d60a8cbc6e | |||
| 1fcbf54989 | |||
| ce75496770 | |||
| 7eae4a0b29 | |||
| f421154c1d | |||
| 03dc119a15 | |||
| 1963bee00c | |||
| 387e7e5e73 | |||
| a51a48ce89 | |||
| aacb3a2fd0 | |||
| 35d2f0df7c | |||
| 7d46ebd6ba | |||
| 1496aa57b1 | |||
| fc9ef2f0d7 | |||
| af067f7360 | |||
| 949b543d1f | |||
| 8fdb68fb19 | |||
| 136f767309 | |||
| aa9664c459 | |||
| f60f0b1b5c | |||
| 676cd44d9d | |||
| 1d081bb218 | |||
| 52ae7e10aa | |||
| 153bbecc44 | |||
| cb46970808 | |||
| 97fa7f3495 | |||
| a88dbb8486 | |||
| d0a83c04ca | |||
| f95c1f2d43 | |||
| 0ef27a3229 | |||
| 0f89373d11 | |||
| f55d35e301 | |||
| 1aee18a025 | |||
| 0068d77ad9 | |||
| b69182e2c7 | |||
| 1c8709f520 | |||
| de1408bd58 | |||
| c295c330ff | |||
| 7cc723ce83 | |||
| c3c48669ad | |||
| 78a0018940 | |||
| 851cc3c4cc | |||
| 74454cdc7f | |||
| 31c838197a | |||
| 45fa583ce8 | |||
| c96f514bcd | |||
| 6a5e6d2bfb | |||
| 875d0b8f55 | |||
| b15387041b | |||
| 60cdb1cee3 | |||
| 52fd47a921 | |||
| b723ec3c0f | |||
| 68ca7e93a1 | |||
| bc5607f48c | |||
| 36a5186c17 | |||
| 05bac73c45 | |||
| 7a43428e76 | |||
| e21da8330e | |||
| 56c3f0534d | |||
| 98e3b89d46 | |||
| 8271c9f95a | |||
| f7bdefb0a3 | |||
| e0a7787139 | |||
| c1159f518c | |||
| a19a8ba412 | |||
| bb455b3c37 | |||
| ca35a67e9f | |||
| 88f1853b09 | |||
| 3ca72674af | |||
| c185d4e3ca | |||
| 2d62cac5f7 | |||
| e3361cf098 | |||
| 41f7f33746 | |||
| 8141fafb34 | |||
| 42af434bd7 | |||
| fbb200c4ee | |||
| b96a9f412a | |||
| 6b101a91f6 | |||
| 2df5428712 | |||
| 5d7e05172d | |||
| 41058ff5c6 | |||
| 54a87ca3dc | |||
| 6bf93d33ea | |||
| 441a2c74ad | |||
| f628774267 | |||
| 3f16413769 | |||
| 959a64aebc | |||
| 694014934c | |||
| cff176e7a3 | |||
| 7f7e6fdd1f | |||
| 45a52cbc33 | |||
| bba7362641 | |||
| 468f85c45d | |||
| 24e2d01ccc | |||
| 43d7775d08 | |||
| 527dec4d49 | |||
| fe70b56d24 | |||
| ed62f03ba0 | |||
| e034e83198 | |||
| 110f4ec332 | |||
| 5bf265ed46 | |||
| 528fe7c024 | |||
| 08be0658cb | |||
| f823841b15 | |||
| 9ce3793067 | |||
| 89d4605577 | |||
| 675a0fc374 | |||
| ca2653ea1a | |||
| a8d3fd8033 | |||
| 702b956ff1 | |||
| 9b8577f258 | |||
| 9623681a15 | |||
| cc22fd8c35 | |||
| 0ef1b6100e | |||
| a519746ccb | |||
| f29dd8ef8b | |||
| f2a5c06005 | |||
| fb9f959fe5 | |||
| 169407a729 | |||
| 302172c537 | |||
| 4fdaab9e87 | |||
| 4dcc1f9e90 | |||
| 67d57c8872 | |||
| d7bf79dec9 | |||
| d90e9b51dc | |||
| 98e2e4073a | |||
| 23c2085f1c | |||
| 2a6a0d0a87 | |||
| ebffb8f912 | |||
| 5676e9094d | |||
| b926aba9ff | |||
| e62c6ac8ee | |||
| 18f4970059 | |||
| 12cab7473a | |||
| 06b0f1251e | |||
| 8a43da502a | |||
| bd5bcdd548 | |||
| 0a51328da2 | |||
| b2d7744cc5 | |||
| 8124fc9add |
41
.VSCodeCounter/2025-03-17_16-24-17/details.md
Normal file
41
.VSCodeCounter/2025-03-17_16-24-17/details.md
Normal file
@@ -0,0 +1,41 @@
|
||||
# Details
|
||||
|
||||
Date : 2025-03-17 16:24:17
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 26 files, 6193 codes, 1008 comments, 1017 blanks, all 8218 lines
|
||||
|
||||
[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
|
||||
## Files
|
||||
| filename | language | code | comment | blank | total |
|
||||
| :--- | :--- | ---: | ---: | ---: | ---: |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md) | Markdown | 39 | 0 | 19 | 58 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx) | TypeScript JSX | 230 | 10 | 8 | 248 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx) | TypeScript JSX | 18 | 0 | 3 | 21 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx) | TypeScript JSX | 273 | 19 | 37 | 329 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx) | TypeScript JSX | 374 | 42 | 44 | 460 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx) | TypeScript JSX | 730 | 126 | 106 | 962 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx) | TypeScript JSX | 499 | 48 | 54 | 601 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx) | TypeScript JSX | 112 | 12 | 21 | 145 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx) | TypeScript JSX | 232 | 31 | 32 | 295 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx) | TypeScript JSX | 407 | 56 | 52 | 515 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx) | TypeScript JSX | 193 | 23 | 22 | 238 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx) | TypeScript JSX | 289 | 36 | 31 | 356 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx) | TypeScript JSX | 500 | 75 | 89 | 664 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx) | TypeScript JSX | 248 | 69 | 74 | 391 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx) | TypeScript JSX | 204 | 26 | 33 | 263 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx) | TypeScript JSX | 209 | 49 | 50 | 308 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx) | TypeScript JSX | 219 | 39 | 47 | 305 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx) | TypeScript JSX | 1,060 | 228 | 229 | 1,517 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx) | TypeScript JSX | 20 | 6 | 2 | 28 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts) | TypeScript | 4 | 0 | 1 | 5 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts) | TypeScript | 16 | 4 | 4 | 24 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts) | TypeScript | 124 | 4 | 14 | 142 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts) | TypeScript | 21 | 15 | 5 | 41 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts) | TypeScript | 43 | 24 | 7 | 74 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js) | JavaScript | 28 | 7 | 9 | 44 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts) | TypeScript | 101 | 59 | 24 | 184 |
|
||||
|
||||
[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
20
.VSCodeCounter/2025-03-17_16-24-17/diff-details.md
Normal file
20
.VSCodeCounter/2025-03-17_16-24-17/diff-details.md
Normal file
@@ -0,0 +1,20 @@
|
||||
# Diff Details
|
||||
|
||||
Date : 2025-03-17 16:24:17
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 5 files, -358 codes, -15 comments, -33 blanks, all -406 lines
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details
|
||||
|
||||
## Files
|
||||
| filename | language | code | comment | blank | total |
|
||||
| :--- | :--- | ---: | ---: | ---: | ---: |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SaveTemplateDialog.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SaveTemplateDialog.tsx) | TypeScript JSX | -83 | 0 | -4 | -87 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/TemplateManager.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/TemplateManager.tsx) | TypeScript JSX | -193 | -4 | -15 | -212 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx) | TypeScript JSX | -241 | -68 | -72 | -381 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useFilters.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useFilters.tsx) | TypeScript JSX | -89 | -12 | -16 | -117 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx) | TypeScript JSX | 248 | 69 | 74 | 391 |
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details
|
||||
23
.VSCodeCounter/2025-03-17_16-24-17/diff.md
Normal file
23
.VSCodeCounter/2025-03-17_16-24-17/diff.md
Normal file
@@ -0,0 +1,23 @@
|
||||
# Diff Summary
|
||||
|
||||
Date : 2025-03-17 16:24:17
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 5 files, -358 codes, -15 comments, -33 blanks, all -406 lines
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md)
|
||||
|
||||
## Languages
|
||||
| language | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| TypeScript JSX | 5 | -358 | -15 | -33 | -406 |
|
||||
|
||||
## Directories
|
||||
| path | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| . | 5 | -358 | -15 | -33 | -406 |
|
||||
| components | 3 | -517 | -72 | -91 | -680 |
|
||||
| hooks | 2 | 159 | 57 | 58 | 274 |
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md)
|
||||
31
.VSCodeCounter/2025-03-17_16-24-17/diff.txt
Normal file
31
.VSCodeCounter/2025-03-17_16-24-17/diff.txt
Normal file
@@ -0,0 +1,31 @@
|
||||
Date : 2025-03-17 16:24:17
|
||||
Directory : /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
Total : 5 files, -358 codes, -15 comments, -33 blanks, all -406 lines
|
||||
|
||||
Languages
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| language | files | code | comment | blank | total |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| TypeScript JSX | 5 | -358 | -15 | -33 | -406 |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Directories
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| path | files | code | comment | blank | total |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| . | 5 | -358 | -15 | -33 | -406 |
|
||||
| components | 3 | -517 | -72 | -91 | -680 |
|
||||
| hooks | 2 | 159 | 57 | 58 | 274 |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Files
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| filename | language | code | comment | blank | total |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SaveTemplateDialog.tsx | TypeScript JSX | -83 | 0 | -4 | -87 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/TemplateManager.tsx | TypeScript JSX | -193 | -4 | -15 | -212 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx | TypeScript JSX | -241 | -68 | -72 | -381 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useFilters.tsx | TypeScript JSX | -89 | -12 | -16 | -117 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx | TypeScript JSX | 248 | 69 | 74 | 391 |
|
||||
| Total | | -358 | -15 | -33 | -406 |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
1
.VSCodeCounter/2025-03-17_16-24-17/results.json
Normal file
1
.VSCodeCounter/2025-03-17_16-24-17/results.json
Normal file
File diff suppressed because one or more lines are too long
31
.VSCodeCounter/2025-03-17_16-24-17/results.md
Normal file
31
.VSCodeCounter/2025-03-17_16-24-17/results.md
Normal file
@@ -0,0 +1,31 @@
|
||||
# Summary
|
||||
|
||||
Date : 2025-03-17 16:24:17
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 26 files, 6193 codes, 1008 comments, 1017 blanks, all 8218 lines
|
||||
|
||||
Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
|
||||
## Languages
|
||||
| language | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| TypeScript JSX | 18 | 5,817 | 895 | 934 | 7,646 |
|
||||
| TypeScript | 6 | 309 | 106 | 55 | 470 |
|
||||
| Markdown | 1 | 39 | 0 | 19 | 58 |
|
||||
| JavaScript | 1 | 28 | 7 | 9 | 44 |
|
||||
|
||||
## Directories
|
||||
| path | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| . | 26 | 6,193 | 1,008 | 1,017 | 8,218 |
|
||||
| . (Files) | 3 | 63 | 6 | 22 | 91 |
|
||||
| components | 11 | 3,357 | 403 | 410 | 4,170 |
|
||||
| components (Files) | 6 | 2,124 | 245 | 252 | 2,621 |
|
||||
| components/cells | 5 | 1,233 | 158 | 158 | 1,549 |
|
||||
| hooks | 6 | 2,440 | 486 | 522 | 3,448 |
|
||||
| types | 1 | 16 | 4 | 4 | 24 |
|
||||
| utils | 5 | 317 | 109 | 59 | 485 |
|
||||
|
||||
Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
60
.VSCodeCounter/2025-03-17_16-24-17/results.txt
Normal file
60
.VSCodeCounter/2025-03-17_16-24-17/results.txt
Normal file
@@ -0,0 +1,60 @@
|
||||
Date : 2025-03-17 16:24:17
|
||||
Directory : /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
Total : 26 files, 6193 codes, 1008 comments, 1017 blanks, all 8218 lines
|
||||
|
||||
Languages
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| language | files | code | comment | blank | total |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| TypeScript JSX | 18 | 5,817 | 895 | 934 | 7,646 |
|
||||
| TypeScript | 6 | 309 | 106 | 55 | 470 |
|
||||
| Markdown | 1 | 39 | 0 | 19 | 58 |
|
||||
| JavaScript | 1 | 28 | 7 | 9 | 44 |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Directories
|
||||
+------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| path | files | code | comment | blank | total |
|
||||
+------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| . | 26 | 6,193 | 1,008 | 1,017 | 8,218 |
|
||||
| . (Files) | 3 | 63 | 6 | 22 | 91 |
|
||||
| components | 11 | 3,357 | 403 | 410 | 4,170 |
|
||||
| components (Files) | 6 | 2,124 | 245 | 252 | 2,621 |
|
||||
| components/cells | 5 | 1,233 | 158 | 158 | 1,549 |
|
||||
| hooks | 6 | 2,440 | 486 | 522 | 3,448 |
|
||||
| types | 1 | 16 | 4 | 4 | 24 |
|
||||
| utils | 5 | 317 | 109 | 59 | 485 |
|
||||
+------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Files
|
||||
+------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| filename | language | code | comment | blank | total |
|
||||
+------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md | Markdown | 39 | 0 | 19 | 58 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx | TypeScript JSX | 230 | 10 | 8 | 248 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx | TypeScript JSX | 18 | 0 | 3 | 21 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx | TypeScript JSX | 273 | 19 | 37 | 329 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx | TypeScript JSX | 374 | 42 | 44 | 460 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx | TypeScript JSX | 730 | 126 | 106 | 962 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx | TypeScript JSX | 499 | 48 | 54 | 601 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx | TypeScript JSX | 112 | 12 | 21 | 145 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx | TypeScript JSX | 232 | 31 | 32 | 295 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx | TypeScript JSX | 407 | 56 | 52 | 515 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx | TypeScript JSX | 193 | 23 | 22 | 238 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx | TypeScript JSX | 289 | 36 | 31 | 356 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx | TypeScript JSX | 500 | 75 | 89 | 664 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx | TypeScript JSX | 248 | 69 | 74 | 391 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx | TypeScript JSX | 204 | 26 | 33 | 263 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx | TypeScript JSX | 209 | 49 | 50 | 308 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx | TypeScript JSX | 219 | 39 | 47 | 305 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx | TypeScript JSX | 1,060 | 228 | 229 | 1,517 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx | TypeScript JSX | 20 | 6 | 2 | 28 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts | TypeScript | 4 | 0 | 1 | 5 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts | TypeScript | 16 | 4 | 4 | 24 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts | TypeScript | 124 | 4 | 14 | 142 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts | TypeScript | 21 | 15 | 5 | 41 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts | TypeScript | 43 | 24 | 7 | 74 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js | JavaScript | 28 | 7 | 9 | 44 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts | TypeScript | 101 | 59 | 24 | 184 |
|
||||
| Total | | 6,193 | 1,008 | 1,017 | 8,218 |
|
||||
+------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
42
.VSCodeCounter/2025-03-18_12-39-04/details.md
Normal file
42
.VSCodeCounter/2025-03-18_12-39-04/details.md
Normal file
@@ -0,0 +1,42 @@
|
||||
# Details
|
||||
|
||||
Date : 2025-03-18 12:39:04
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 27 files, 6925 codes, 1247 comments, 1248 blanks, all 9420 lines
|
||||
|
||||
[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
|
||||
## Files
|
||||
| filename | language | code | comment | blank | total |
|
||||
| :--- | :--- | ---: | ---: | ---: | ---: |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md) | Markdown | 39 | 0 | 19 | 58 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx) | TypeScript JSX | 230 | 10 | 8 | 248 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx) | TypeScript JSX | 18 | 0 | 3 | 21 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx) | TypeScript JSX | 273 | 19 | 37 | 329 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx) | TypeScript JSX | 113 | 17 | 10 | 140 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx) | TypeScript JSX | 377 | 49 | 54 | 480 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx) | TypeScript JSX | 969 | 182 | 158 | 1,309 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx) | TypeScript JSX | 509 | 50 | 57 | 616 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx) | TypeScript JSX | 112 | 12 | 21 | 145 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx) | TypeScript JSX | 233 | 34 | 33 | 300 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx) | TypeScript JSX | 420 | 66 | 59 | 545 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx) | TypeScript JSX | 193 | 23 | 22 | 238 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx) | TypeScript JSX | 227 | 36 | 32 | 295 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx) | TypeScript JSX | 500 | 75 | 89 | 664 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx) | TypeScript JSX | 264 | 75 | 81 | 420 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx) | TypeScript JSX | 204 | 26 | 33 | 263 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx) | TypeScript JSX | 337 | 88 | 92 | 517 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx) | TypeScript JSX | 360 | 78 | 85 | 523 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx) | TypeScript JSX | 1,190 | 288 | 289 | 1,767 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx) | TypeScript JSX | 20 | 6 | 2 | 28 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts) | TypeScript | 4 | 0 | 1 | 5 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts) | TypeScript | 16 | 4 | 4 | 24 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts) | TypeScript | 124 | 4 | 14 | 142 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts) | TypeScript | 21 | 15 | 5 | 41 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts) | TypeScript | 43 | 24 | 7 | 74 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js) | JavaScript | 28 | 7 | 9 | 44 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts) | TypeScript | 101 | 59 | 24 | 184 |
|
||||
|
||||
[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
26
.VSCodeCounter/2025-03-18_12-39-04/diff-details.md
Normal file
26
.VSCodeCounter/2025-03-18_12-39-04/diff-details.md
Normal file
@@ -0,0 +1,26 @@
|
||||
# Diff Details
|
||||
|
||||
Date : 2025-03-18 12:39:04
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 11 files, 732 codes, 239 comments, 231 blanks, all 1202 lines
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details
|
||||
|
||||
## Files
|
||||
| filename | language | code | comment | blank | total |
|
||||
| :--- | :--- | ---: | ---: | ---: | ---: |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx) | TypeScript JSX | 113 | 17 | 10 | 140 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx) | TypeScript JSX | 3 | 7 | 10 | 20 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx) | TypeScript JSX | 239 | 56 | 52 | 347 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx) | TypeScript JSX | 10 | 2 | 3 | 15 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx) | TypeScript JSX | 1 | 3 | 1 | 5 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx) | TypeScript JSX | 13 | 10 | 7 | 30 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx) | TypeScript JSX | -62 | 0 | 1 | -61 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx) | TypeScript JSX | 16 | 6 | 7 | 29 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx) | TypeScript JSX | 128 | 39 | 42 | 209 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx) | TypeScript JSX | 141 | 39 | 38 | 218 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx) | TypeScript JSX | 130 | 60 | 60 | 250 |
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details
|
||||
25
.VSCodeCounter/2025-03-18_12-39-04/diff.md
Normal file
25
.VSCodeCounter/2025-03-18_12-39-04/diff.md
Normal file
@@ -0,0 +1,25 @@
|
||||
# Diff Summary
|
||||
|
||||
Date : 2025-03-18 12:39:04
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 11 files, 732 codes, 239 comments, 231 blanks, all 1202 lines
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md)
|
||||
|
||||
## Languages
|
||||
| language | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| TypeScript JSX | 11 | 732 | 239 | 231 | 1,202 |
|
||||
|
||||
## Directories
|
||||
| path | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| . | 11 | 732 | 239 | 231 | 1,202 |
|
||||
| components | 7 | 317 | 95 | 84 | 496 |
|
||||
| components (Files) | 4 | 365 | 82 | 75 | 522 |
|
||||
| components/cells | 3 | -48 | 13 | 9 | -26 |
|
||||
| hooks | 4 | 415 | 144 | 147 | 706 |
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md)
|
||||
39
.VSCodeCounter/2025-03-18_12-39-04/diff.txt
Normal file
39
.VSCodeCounter/2025-03-18_12-39-04/diff.txt
Normal file
@@ -0,0 +1,39 @@
|
||||
Date : 2025-03-18 12:39:04
|
||||
Directory : /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
Total : 11 files, 732 codes, 239 comments, 231 blanks, all 1202 lines
|
||||
|
||||
Languages
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| language | files | code | comment | blank | total |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| TypeScript JSX | 11 | 732 | 239 | 231 | 1,202 |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Directories
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| path | files | code | comment | blank | total |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| . | 11 | 732 | 239 | 231 | 1,202 |
|
||||
| components | 7 | 317 | 95 | 84 | 496 |
|
||||
| components (Files) | 4 | 365 | 82 | 75 | 522 |
|
||||
| components/cells | 3 | -48 | 13 | 9 | -26 |
|
||||
| hooks | 4 | 415 | 144 | 147 | 706 |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Files
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| filename | language | code | comment | blank | total |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx | TypeScript JSX | 113 | 17 | 10 | 140 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx | TypeScript JSX | 3 | 7 | 10 | 20 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx | TypeScript JSX | 239 | 56 | 52 | 347 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx | TypeScript JSX | 10 | 2 | 3 | 15 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx | TypeScript JSX | 1 | 3 | 1 | 5 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx | TypeScript JSX | 13 | 10 | 7 | 30 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx | TypeScript JSX | -62 | 0 | 1 | -61 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx | TypeScript JSX | 16 | 6 | 7 | 29 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx | TypeScript JSX | 128 | 39 | 42 | 209 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx | TypeScript JSX | 141 | 39 | 38 | 218 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx | TypeScript JSX | 130 | 60 | 60 | 250 |
|
||||
| Total | | 732 | 239 | 231 | 1,202 |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
1
.VSCodeCounter/2025-03-18_12-39-04/results.json
Normal file
1
.VSCodeCounter/2025-03-18_12-39-04/results.json
Normal file
File diff suppressed because one or more lines are too long
31
.VSCodeCounter/2025-03-18_12-39-04/results.md
Normal file
31
.VSCodeCounter/2025-03-18_12-39-04/results.md
Normal file
@@ -0,0 +1,31 @@
|
||||
# Summary
|
||||
|
||||
Date : 2025-03-18 12:39:04
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 27 files, 6925 codes, 1247 comments, 1248 blanks, all 9420 lines
|
||||
|
||||
Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
|
||||
## Languages
|
||||
| language | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| TypeScript JSX | 19 | 6,549 | 1,134 | 1,165 | 8,848 |
|
||||
| TypeScript | 6 | 309 | 106 | 55 | 470 |
|
||||
| Markdown | 1 | 39 | 0 | 19 | 58 |
|
||||
| JavaScript | 1 | 28 | 7 | 9 | 44 |
|
||||
|
||||
## Directories
|
||||
| path | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| . | 27 | 6,925 | 1,247 | 1,248 | 9,420 |
|
||||
| . (Files) | 3 | 63 | 6 | 22 | 91 |
|
||||
| components | 12 | 3,674 | 498 | 494 | 4,666 |
|
||||
| components (Files) | 7 | 2,489 | 327 | 327 | 3,143 |
|
||||
| components/cells | 5 | 1,185 | 171 | 167 | 1,523 |
|
||||
| hooks | 6 | 2,855 | 630 | 669 | 4,154 |
|
||||
| types | 1 | 16 | 4 | 4 | 24 |
|
||||
| utils | 5 | 317 | 109 | 59 | 485 |
|
||||
|
||||
Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
61
.VSCodeCounter/2025-03-18_12-39-04/results.txt
Normal file
61
.VSCodeCounter/2025-03-18_12-39-04/results.txt
Normal file
@@ -0,0 +1,61 @@
|
||||
Date : 2025-03-18 12:39:04
|
||||
Directory : /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
Total : 27 files, 6925 codes, 1247 comments, 1248 blanks, all 9420 lines
|
||||
|
||||
Languages
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| language | files | code | comment | blank | total |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| TypeScript JSX | 19 | 6,549 | 1,134 | 1,165 | 8,848 |
|
||||
| TypeScript | 6 | 309 | 106 | 55 | 470 |
|
||||
| Markdown | 1 | 39 | 0 | 19 | 58 |
|
||||
| JavaScript | 1 | 28 | 7 | 9 | 44 |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Directories
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| path | files | code | comment | blank | total |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| . | 27 | 6,925 | 1,247 | 1,248 | 9,420 |
|
||||
| . (Files) | 3 | 63 | 6 | 22 | 91 |
|
||||
| components | 12 | 3,674 | 498 | 494 | 4,666 |
|
||||
| components (Files) | 7 | 2,489 | 327 | 327 | 3,143 |
|
||||
| components/cells | 5 | 1,185 | 171 | 167 | 1,523 |
|
||||
| hooks | 6 | 2,855 | 630 | 669 | 4,154 |
|
||||
| types | 1 | 16 | 4 | 4 | 24 |
|
||||
| utils | 5 | 317 | 109 | 59 | 485 |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Files
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| filename | language | code | comment | blank | total |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md | Markdown | 39 | 0 | 19 | 58 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx | TypeScript JSX | 230 | 10 | 8 | 248 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx | TypeScript JSX | 18 | 0 | 3 | 21 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx | TypeScript JSX | 273 | 19 | 37 | 329 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx | TypeScript JSX | 113 | 17 | 10 | 140 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx | TypeScript JSX | 377 | 49 | 54 | 480 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx | TypeScript JSX | 969 | 182 | 158 | 1,309 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx | TypeScript JSX | 509 | 50 | 57 | 616 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx | TypeScript JSX | 112 | 12 | 21 | 145 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx | TypeScript JSX | 233 | 34 | 33 | 300 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx | TypeScript JSX | 420 | 66 | 59 | 545 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx | TypeScript JSX | 193 | 23 | 22 | 238 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx | TypeScript JSX | 227 | 36 | 32 | 295 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx | TypeScript JSX | 500 | 75 | 89 | 664 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx | TypeScript JSX | 264 | 75 | 81 | 420 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx | TypeScript JSX | 204 | 26 | 33 | 263 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx | TypeScript JSX | 337 | 88 | 92 | 517 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx | TypeScript JSX | 360 | 78 | 85 | 523 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx | TypeScript JSX | 1,190 | 288 | 289 | 1,767 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx | TypeScript JSX | 20 | 6 | 2 | 28 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts | TypeScript | 4 | 0 | 1 | 5 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts | TypeScript | 16 | 4 | 4 | 24 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts | TypeScript | 124 | 4 | 14 | 142 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts | TypeScript | 21 | 15 | 5 | 41 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts | TypeScript | 43 | 24 | 7 | 74 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js | JavaScript | 28 | 7 | 9 | 44 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts | TypeScript | 101 | 59 | 24 | 184 |
|
||||
| Total | | 6,925 | 1,247 | 1,248 | 9,420 |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
42
.VSCodeCounter/2025-03-18_13-49-23/details.md
Normal file
42
.VSCodeCounter/2025-03-18_13-49-23/details.md
Normal file
@@ -0,0 +1,42 @@
|
||||
# Details
|
||||
|
||||
Date : 2025-03-18 13:49:23
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 27 files, 6961 codes, 1254 comments, 1252 blanks, all 9467 lines
|
||||
|
||||
[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
|
||||
## Files
|
||||
| filename | language | code | comment | blank | total |
|
||||
| :--- | :--- | ---: | ---: | ---: | ---: |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md) | Markdown | 39 | 0 | 19 | 58 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx) | TypeScript JSX | 230 | 10 | 8 | 248 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx) | TypeScript JSX | 18 | 0 | 3 | 21 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx) | TypeScript JSX | 273 | 19 | 37 | 329 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx) | TypeScript JSX | 113 | 17 | 10 | 140 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx) | TypeScript JSX | 395 | 51 | 55 | 501 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx) | TypeScript JSX | 969 | 182 | 158 | 1,309 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx) | TypeScript JSX | 527 | 55 | 60 | 642 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx) | TypeScript JSX | 112 | 12 | 21 | 145 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx) | TypeScript JSX | 233 | 34 | 33 | 300 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx) | TypeScript JSX | 420 | 66 | 59 | 545 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx) | TypeScript JSX | 193 | 23 | 22 | 238 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx) | TypeScript JSX | 227 | 36 | 32 | 295 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx) | TypeScript JSX | 500 | 75 | 89 | 664 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx) | TypeScript JSX | 264 | 75 | 81 | 420 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx) | TypeScript JSX | 204 | 26 | 33 | 263 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx) | TypeScript JSX | 337 | 88 | 92 | 517 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx) | TypeScript JSX | 360 | 78 | 85 | 523 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx) | TypeScript JSX | 1,190 | 288 | 289 | 1,767 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx) | TypeScript JSX | 20 | 6 | 2 | 28 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts) | TypeScript | 4 | 0 | 1 | 5 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts) | TypeScript | 16 | 4 | 4 | 24 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts) | TypeScript | 124 | 4 | 14 | 142 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts) | TypeScript | 21 | 15 | 5 | 41 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts) | TypeScript | 43 | 24 | 7 | 74 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js) | JavaScript | 28 | 7 | 9 | 44 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts) | TypeScript | 101 | 59 | 24 | 184 |
|
||||
|
||||
[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
17
.VSCodeCounter/2025-03-18_13-49-23/diff-details.md
Normal file
17
.VSCodeCounter/2025-03-18_13-49-23/diff-details.md
Normal file
@@ -0,0 +1,17 @@
|
||||
# Diff Details
|
||||
|
||||
Date : 2025-03-18 13:49:23
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 2 files, 36 codes, 7 comments, 4 blanks, all 47 lines
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details
|
||||
|
||||
## Files
|
||||
| filename | language | code | comment | blank | total |
|
||||
| :--- | :--- | ---: | ---: | ---: | ---: |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx) | TypeScript JSX | 18 | 2 | 1 | 21 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx) | TypeScript JSX | 18 | 5 | 3 | 26 |
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details
|
||||
22
.VSCodeCounter/2025-03-18_13-49-23/diff.md
Normal file
22
.VSCodeCounter/2025-03-18_13-49-23/diff.md
Normal file
@@ -0,0 +1,22 @@
|
||||
# Diff Summary
|
||||
|
||||
Date : 2025-03-18 13:49:23
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 2 files, 36 codes, 7 comments, 4 blanks, all 47 lines
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md)
|
||||
|
||||
## Languages
|
||||
| language | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| TypeScript JSX | 2 | 36 | 7 | 4 | 47 |
|
||||
|
||||
## Directories
|
||||
| path | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| . | 2 | 36 | 7 | 4 | 47 |
|
||||
| components | 2 | 36 | 7 | 4 | 47 |
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md)
|
||||
27
.VSCodeCounter/2025-03-18_13-49-23/diff.txt
Normal file
27
.VSCodeCounter/2025-03-18_13-49-23/diff.txt
Normal file
@@ -0,0 +1,27 @@
|
||||
Date : 2025-03-18 13:49:23
|
||||
Directory : /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
Total : 2 files, 36 codes, 7 comments, 4 blanks, all 47 lines
|
||||
|
||||
Languages
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| language | files | code | comment | blank | total |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| TypeScript JSX | 2 | 36 | 7 | 4 | 47 |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Directories
|
||||
+---------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| path | files | code | comment | blank | total |
|
||||
+---------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| . | 2 | 36 | 7 | 4 | 47 |
|
||||
| components | 2 | 36 | 7 | 4 | 47 |
|
||||
+---------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Files
|
||||
+---------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| filename | language | code | comment | blank | total |
|
||||
+---------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx | TypeScript JSX | 18 | 2 | 1 | 21 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx | TypeScript JSX | 18 | 5 | 3 | 26 |
|
||||
| Total | | 36 | 7 | 4 | 47 |
|
||||
+---------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
1
.VSCodeCounter/2025-03-18_13-49-23/results.json
Normal file
1
.VSCodeCounter/2025-03-18_13-49-23/results.json
Normal file
File diff suppressed because one or more lines are too long
31
.VSCodeCounter/2025-03-18_13-49-23/results.md
Normal file
31
.VSCodeCounter/2025-03-18_13-49-23/results.md
Normal file
@@ -0,0 +1,31 @@
|
||||
# Summary
|
||||
|
||||
Date : 2025-03-18 13:49:23
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 27 files, 6961 codes, 1254 comments, 1252 blanks, all 9467 lines
|
||||
|
||||
Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
|
||||
## Languages
|
||||
| language | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| TypeScript JSX | 19 | 6,585 | 1,141 | 1,169 | 8,895 |
|
||||
| TypeScript | 6 | 309 | 106 | 55 | 470 |
|
||||
| Markdown | 1 | 39 | 0 | 19 | 58 |
|
||||
| JavaScript | 1 | 28 | 7 | 9 | 44 |
|
||||
|
||||
## Directories
|
||||
| path | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| . | 27 | 6,961 | 1,254 | 1,252 | 9,467 |
|
||||
| . (Files) | 3 | 63 | 6 | 22 | 91 |
|
||||
| components | 12 | 3,710 | 505 | 498 | 4,713 |
|
||||
| components (Files) | 7 | 2,525 | 334 | 331 | 3,190 |
|
||||
| components/cells | 5 | 1,185 | 171 | 167 | 1,523 |
|
||||
| hooks | 6 | 2,855 | 630 | 669 | 4,154 |
|
||||
| types | 1 | 16 | 4 | 4 | 24 |
|
||||
| utils | 5 | 317 | 109 | 59 | 485 |
|
||||
|
||||
Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
61
.VSCodeCounter/2025-03-18_13-49-23/results.txt
Normal file
61
.VSCodeCounter/2025-03-18_13-49-23/results.txt
Normal file
@@ -0,0 +1,61 @@
|
||||
Date : 2025-03-18 13:49:23
|
||||
Directory : /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
Total : 27 files, 6961 codes, 1254 comments, 1252 blanks, all 9467 lines
|
||||
|
||||
Languages
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| language | files | code | comment | blank | total |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| TypeScript JSX | 19 | 6,585 | 1,141 | 1,169 | 8,895 |
|
||||
| TypeScript | 6 | 309 | 106 | 55 | 470 |
|
||||
| Markdown | 1 | 39 | 0 | 19 | 58 |
|
||||
| JavaScript | 1 | 28 | 7 | 9 | 44 |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Directories
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| path | files | code | comment | blank | total |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| . | 27 | 6,961 | 1,254 | 1,252 | 9,467 |
|
||||
| . (Files) | 3 | 63 | 6 | 22 | 91 |
|
||||
| components | 12 | 3,710 | 505 | 498 | 4,713 |
|
||||
| components (Files) | 7 | 2,525 | 334 | 331 | 3,190 |
|
||||
| components/cells | 5 | 1,185 | 171 | 167 | 1,523 |
|
||||
| hooks | 6 | 2,855 | 630 | 669 | 4,154 |
|
||||
| types | 1 | 16 | 4 | 4 | 24 |
|
||||
| utils | 5 | 317 | 109 | 59 | 485 |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Files
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| filename | language | code | comment | blank | total |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md | Markdown | 39 | 0 | 19 | 58 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx | TypeScript JSX | 230 | 10 | 8 | 248 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx | TypeScript JSX | 18 | 0 | 3 | 21 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx | TypeScript JSX | 273 | 19 | 37 | 329 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx | TypeScript JSX | 113 | 17 | 10 | 140 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx | TypeScript JSX | 395 | 51 | 55 | 501 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx | TypeScript JSX | 969 | 182 | 158 | 1,309 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx | TypeScript JSX | 527 | 55 | 60 | 642 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx | TypeScript JSX | 112 | 12 | 21 | 145 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx | TypeScript JSX | 233 | 34 | 33 | 300 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx | TypeScript JSX | 420 | 66 | 59 | 545 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx | TypeScript JSX | 193 | 23 | 22 | 238 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx | TypeScript JSX | 227 | 36 | 32 | 295 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx | TypeScript JSX | 500 | 75 | 89 | 664 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx | TypeScript JSX | 264 | 75 | 81 | 420 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx | TypeScript JSX | 204 | 26 | 33 | 263 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx | TypeScript JSX | 337 | 88 | 92 | 517 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx | TypeScript JSX | 360 | 78 | 85 | 523 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx | TypeScript JSX | 1,190 | 288 | 289 | 1,767 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx | TypeScript JSX | 20 | 6 | 2 | 28 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts | TypeScript | 4 | 0 | 1 | 5 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts | TypeScript | 16 | 4 | 4 | 24 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts | TypeScript | 124 | 4 | 14 | 142 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts | TypeScript | 21 | 15 | 5 | 41 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts | TypeScript | 43 | 24 | 7 | 74 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js | JavaScript | 28 | 7 | 9 | 44 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts | TypeScript | 101 | 59 | 24 | 184 |
|
||||
| Total | | 6,961 | 1,254 | 1,252 | 9,467 |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
172
.claude/CLAUDE.md
Normal file
172
.claude/CLAUDE.md
Normal file
@@ -0,0 +1,172 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Project Overview
|
||||
|
||||
This is a full-stack inventory management system with a React + TypeScript frontend and Node.js/Express backend using PostgreSQL. The system includes product management, analytics, forecasting, purchase orders, and a comprehensive dashboard for business metrics.
|
||||
|
||||
**Monorepo Structure:**
|
||||
- `inventory/` - Vite-based React frontend with TypeScript
|
||||
- `inventory-server/` - Express backend API server
|
||||
- Root `package.json` contains shared dependencies
|
||||
|
||||
## Development Commands
|
||||
|
||||
### Frontend (inventory/)
|
||||
```bash
|
||||
cd inventory
|
||||
npm run dev # Start dev server on port 5175
|
||||
npm run build # Build for production (outputs to build/ then copies to ../inventory-server/frontend/build)
|
||||
npm run lint # Run ESLint
|
||||
npm run preview # Preview production build
|
||||
```
|
||||
|
||||
### Backend (inventory-server/)
|
||||
```bash
|
||||
cd inventory-server
|
||||
npm run dev # Start with nodemon (auto-reload)
|
||||
npm start # Start server (production)
|
||||
npm run prod # Start with PM2 for production
|
||||
npm run prod:stop # Stop PM2 instance
|
||||
npm run prod:restart # Restart PM2 instance
|
||||
npm run prod:logs # View PM2 logs
|
||||
npm run setup # Create required directories (logs, uploads)
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
### Frontend Architecture
|
||||
|
||||
**Router Structure:** React Router with lazy loading for code splitting:
|
||||
- Main chunks: Core inventory, Dashboard, Product Import, Chat Archive
|
||||
- Authentication flow uses `RequireAuth` and `Protected` components with permission-based access
|
||||
- All routes except `/login` and `/small` require authentication
|
||||
|
||||
**Key Directories:**
|
||||
- `src/pages/` - Top-level page components (Overview, Products, Analytics, Dashboard, etc.)
|
||||
- `src/components/` - Organized by feature (dashboard/, products/, analytics/, etc.)
|
||||
- `src/components/ui/` - shadcn/ui components
|
||||
- `src/types/` - TypeScript type definitions
|
||||
- `src/contexts/` - React contexts (AuthContext, DashboardScrollContext)
|
||||
- `src/hooks/` - Custom React hooks (use-toast, useDebounce, use-mobile)
|
||||
- `src/utils/` - Utility functions (emojiUtils, productUtils, naturalLanguagePeriod)
|
||||
- `src/services/` - API service layer
|
||||
- `src/config/` - Configuration files
|
||||
|
||||
**State Management:**
|
||||
- React Context for auth and global state
|
||||
- @tanstack/react-query for server state management
|
||||
- zustand for client state management
|
||||
- Local storage for auth tokens, session storage for login state
|
||||
|
||||
**Key Dependencies:**
|
||||
- UI: Radix UI primitives, shadcn/ui, Tailwind CSS, Framer Motion
|
||||
- Data: @tanstack/react-table, react-data-grid, @tanstack/react-virtual
|
||||
- Forms: react-hook-form, zod
|
||||
- Charts: recharts, chart.js, react-chartjs-2
|
||||
- File handling: xlsx for Excel export, react-dropzone for uploads
|
||||
- Other: axios for HTTP, date-fns/luxon for dates
|
||||
|
||||
**Path Alias:** `@/` maps to `./src/`
|
||||
|
||||
### Backend Architecture
|
||||
|
||||
**Entry Point:** `inventory-server/src/server.js`
|
||||
|
||||
**Key Directories:**
|
||||
- `src/routes/` - Express route handlers (products, dashboard, analytics, import, etc.)
|
||||
- `src/middleware/` - Express middleware (CORS, auth, etc.)
|
||||
- `src/utils/` - Utility functions (database connection, API helpers)
|
||||
- `src/types/` - Type definitions (e.g., status-codes)
|
||||
|
||||
**Database:**
|
||||
- PostgreSQL with connection pooling (pg library)
|
||||
- Pool initialized in `utils/db.js` via `initPool()`
|
||||
- Pool attached to `app.locals.pool` for route access
|
||||
- Environment variables loaded from `/var/www/html/inventory/.env` (production path)
|
||||
|
||||
**API Routes:** All prefixed with `/api/`
|
||||
- `/api/products` - Product CRUD operations
|
||||
- `/api/dashboard` - Dashboard metrics and data
|
||||
- `/api/analytics` - Analytics and reporting
|
||||
- `/api/orders` - Order management
|
||||
- `/api/purchase-orders` - Purchase order management
|
||||
- `/api/csv` - CSV import/export (data management)
|
||||
- `/api/import` - Product import workflows
|
||||
- `/api/config` - Configuration management
|
||||
- `/api/metrics` - System metrics
|
||||
- `/api/ai-validation` - AI-powered validation
|
||||
- `/api/ai-prompts` - AI prompt management
|
||||
- `/api/templates` - Template management
|
||||
- `/api/reusable-images` - Image management
|
||||
- `/api/categoriesAggregate`, `/api/vendorsAggregate`, `/api/brandsAggregate` - Aggregate data endpoints
|
||||
|
||||
**Authentication:**
|
||||
- External auth service at `/auth-inv` endpoint
|
||||
- Token-based authentication (Bearer tokens)
|
||||
- Frontend stores tokens in localStorage
|
||||
- Protected routes verify tokens via auth service `/me` endpoint
|
||||
|
||||
**File Uploads:**
|
||||
- Multer middleware for file handling
|
||||
- Uploads directory: `inventory-server/uploads/`
|
||||
|
||||
### Development Proxy Setup
|
||||
|
||||
The Vite dev server (port 5175) proxies API requests to `https://inventory.kent.pw`:
|
||||
- `/api/*` → production API
|
||||
- `/auth-inv/*` → authentication service
|
||||
- `/chat-api/*` → chat service
|
||||
- `/uploads/*` → uploaded files
|
||||
- Various third-party services (Aircall, Klaviyo, Meta, Gorgias, Typeform, ACOT, Clarity)
|
||||
|
||||
### Build Process
|
||||
|
||||
When building the frontend:
|
||||
1. TypeScript compilation (`tsc -b`)
|
||||
2. Vite build (outputs to `inventory/build/`)
|
||||
3. Custom Vite plugin copies build to `inventory-server/frontend/build/`
|
||||
4. Manual chunks for vendor splitting (react-vendor, ui-vendor, query-vendor)
|
||||
|
||||
## Testing
|
||||
|
||||
Run tests for individual components or features:
|
||||
```bash
|
||||
# No test suite currently configured
|
||||
# Tests would typically use Jest or Vitest with React Testing Library
|
||||
```
|
||||
|
||||
## Common Development Workflows
|
||||
|
||||
### Adding a New Page
|
||||
1. Create page component in `inventory/src/pages/YourPage.tsx`
|
||||
2. Add lazy import in `inventory/src/App.tsx`
|
||||
3. Add route with `<Protected>` wrapper and permission check
|
||||
4. Add corresponding backend route in `inventory-server/src/routes/`
|
||||
5. Update permission system if needed
|
||||
|
||||
### Adding a New API Endpoint
|
||||
1. Create or update route file in `inventory-server/src/routes/`
|
||||
2. Use `executeQuery()` helper for database queries
|
||||
3. Register router in `inventory-server/src/server.js`
|
||||
4. Frontend can access at `/api/{route-name}`
|
||||
|
||||
### Working with Database
|
||||
- Use parameterized queries: `executeQuery(sql, [param1, param2])`
|
||||
- Pool is accessed via `db.getPool()` or `app.locals.pool`
|
||||
- Connection helper: `db.getConnection()` returns a client for transactions
|
||||
|
||||
### Permissions System
|
||||
- User permissions stored in `user.permissions` array (permission codes)
|
||||
- Check permissions in `<Protected page="permission_code">` component
|
||||
- Admin users (`is_admin: true`) have access to all pages
|
||||
|
||||
## Important Notes
|
||||
|
||||
- Environment variables must be configured in `/var/www/html/inventory/.env` for production
|
||||
- The frontend expects the backend at `/api` (proxied in dev, served together in production)
|
||||
- PM2 is used for production process management
|
||||
- Database uses PostgreSQL with SSL support (configurable via `DB_SSL` env var)
|
||||
- File uploads stored in `inventory-server/uploads/` directory
|
||||
- Build artifacts in `inventory/build/` are copied to `inventory-server/frontend/build/`
|
||||
22
.gitignore
vendored
22
.gitignore
vendored
@@ -26,6 +26,7 @@ dist-ssr
|
||||
dashboard/build/**
|
||||
dashboard-server/frontend/build/**
|
||||
**/build/**
|
||||
.fuse_hidden**
|
||||
._*
|
||||
|
||||
# Build directories
|
||||
@@ -49,6 +50,11 @@ dashboard-server/meta-server/._package-lock.json
|
||||
dashboard-server/meta-server/._services
|
||||
*.tsbuildinfo
|
||||
|
||||
uploads/*
|
||||
uploads/**/*
|
||||
**/uploads/*
|
||||
**/uploads/**/*
|
||||
|
||||
# CSV data files
|
||||
*.csv
|
||||
csv/*
|
||||
@@ -58,3 +64,19 @@ csv/**/*
|
||||
!csv/.gitkeep
|
||||
inventory/tsconfig.tsbuildinfo
|
||||
inventory-server/scripts/.fuse_hidden00000fa20000000a
|
||||
|
||||
.VSCodeCounter/
|
||||
.VSCodeCounter/*
|
||||
.VSCodeCounter/**/*
|
||||
|
||||
*/chat/db-convert/db/*
|
||||
*/chat/db-convert/mongo_converter_env/*
|
||||
|
||||
# Ignore compiled Vite config to avoid duplication
|
||||
vite.config.js
|
||||
inventory-server/inventory_backup.sql
|
||||
chat-files.tar.gz
|
||||
chat-migration*/
|
||||
**/chat-migration*/
|
||||
chat-migration*/**
|
||||
**/chat-migration*/**
|
||||
|
||||
4
CLAUDE.md
Normal file
4
CLAUDE.md
Normal file
@@ -0,0 +1,4 @@
|
||||
* Avoid using glob tool for search as it may not work properly on this codebase. Search using bash instead.
|
||||
* If you use the task tool to have an agent investigate something, make sure to let it know to avoid using glob
|
||||
* Prefer solving tasks in a single session. Only spawn subagents for genuinely independent workstreams.
|
||||
* The postgres/query tool is not working and not connected to the current version of the database. If you need to query the database for any reason you can use "ssh netcup" and use psql on the server with inventory_readonly 6D3GUkxuFgi2UghwgnUd
|
||||
276
METRICS_AUDIT.md
Normal file
276
METRICS_AUDIT.md
Normal file
@@ -0,0 +1,276 @@
|
||||
# Metrics Pipeline Audit Report
|
||||
|
||||
**Date:** 2026-02-08
|
||||
**Scope:** All 6 SQL scripts in `inventory-server/scripts/metrics-new/`, import pipeline, custom functions, and post-calculation data verification.
|
||||
|
||||
---
|
||||
|
||||
## Executive Summary
|
||||
|
||||
The metrics pipeline is architecturally sound and the core calculations are mostly correct. The 30-day sales, revenue, replenishment, and aggregate metrics (brand/vendor/category) all cross-check accurately between the snapshots, product_metrics, and direct orders queries. However, several issues were found ranging from **critical data bugs** to **design limitations** that affect accuracy of specific metrics.
|
||||
|
||||
**Issues found: 13** (3 Critical, 4 Medium, 6 Low/Informational)
|
||||
|
||||
---
|
||||
|
||||
## CRITICAL Issues
|
||||
|
||||
### C1. `net_revenue` in daily snapshots never subtracts returns ($35.6K affected)
|
||||
|
||||
**Location:** `update_daily_snapshots.sql`, line 181
|
||||
**Symptom:** `net_revenue` is stored as `gross_revenue - discounts` but should be `gross_revenue - discounts - returns_revenue`.
|
||||
|
||||
The SQL formula on line 181 appears correct:
|
||||
```sql
|
||||
COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00) AS net_revenue
|
||||
```
|
||||
|
||||
However, actual data shows `net_revenue = gross_revenue - discounts` for ALL 3,252 snapshots that have returns. Total returns not subtracted: **$35,630.03** across 2,946 products. This may be caused by the `returns_revenue` in the SalesData CTE not properly flowing through to the INSERT, or by a prior version of the code that stored these values differently. The profit column (line 184) has the same issue: `(gross - discounts) - cogs` instead of `(gross - discounts - returns) - cogs`.
|
||||
|
||||
**Impact:** Net revenue and profit are overstated by the amount of returns. This cascades to all metrics derived from snapshots: `revenue_30d`, `profit_30d`, `margin_30d`, `avg_ros_30d`, and all brand/vendor/category aggregate revenue.
|
||||
|
||||
**Recommended fix:** Debug why the returns subtraction isn't taking effect. The formula in the SQL looks correct, so this may be a data-type issue or an execution path issue. After fixing, rebuild snapshots.
|
||||
|
||||
**Status:** Owner will resolve. Code formula is correct; snapshots need rebuilding after prior fix deployment.
|
||||
|
||||
---
|
||||
|
||||
### C2. `eod_stock_quantity` uses CURRENT stock, not historical end-of-day stock
|
||||
|
||||
**Location:** `update_daily_snapshots.sql`, lines 123-132 (CurrentStock CTE)
|
||||
**Symptom:** Every snapshot for a given product shows the same stock quantity regardless of the snapshot date.
|
||||
|
||||
The `CurrentStock` CTE simply reads `stock_quantity` from the `products` table:
|
||||
```sql
|
||||
SELECT pid, stock_quantity, ... FROM public.products
|
||||
```
|
||||
|
||||
This means a snapshot from January 10 shows the SAME stock as today (February 8). Verified in data:
|
||||
- Product 662561: stock = 36 on every date (Feb 1-7)
|
||||
- Product 665397: stock = 25 on every date (Feb 1-7)
|
||||
- All products checked show identical stock across all snapshot dates
|
||||
|
||||
**Impact:** All stock-derived metrics are inaccurate for historical analysis:
|
||||
- `eod_stock_cost`, `eod_stock_retail`, `eod_stock_gross` (all wrong for past dates)
|
||||
- `stockout_flag` (based on current stock, not historical)
|
||||
- `stockout_days_30d` (undercounted since stockout_flag uses current stock)
|
||||
- `avg_stock_units_30d`, `avg_stock_cost_30d` (no variance, just current stock repeated)
|
||||
- `gmroi_30d`, `stockturn_30d` (based on avg_stock which is flat)
|
||||
- `sell_through_30d` (denominator uses current stock assumption)
|
||||
- `service_level_30d`, `fill_rate_30d`
|
||||
|
||||
**This is a known architectural limitation** noted in MEMORY.md. Fixing requires either:
|
||||
1. Storing stock snapshots separately at end-of-day (ideally via a cron job that records stock before any changes)
|
||||
2. Reconstructing historical stock from orders and receivings (complex but possible)
|
||||
|
||||
**Status: FIXED.** MySQL's `snap_product_value` table (daily EOD stock per product since 2012) is now imported into PostgreSQL `stock_snapshots` table via `scripts/import/stock-snapshots.js`. The `CurrentStock` CTE in `update_daily_snapshots.sql` now uses `LEFT JOIN stock_snapshots` for historical stock, falling back to `products.stock_quantity` when no historical data exists. Requires: run import, then rebuild daily snapshots.
|
||||
|
||||
---
|
||||
|
||||
### C3. `ON CONFLICT DO UPDATE WHERE` check skips 91%+ of product_metrics updates
|
||||
|
||||
**Location:** `update_product_metrics.sql`, lines 558-574
|
||||
**Symptom:** 623,205 of 681,912 products (91.4%) have `last_calculated` older than 1 day. 592,369 are over 30 days old. 914 products with active 30-day sales haven't been updated in over 7 days.
|
||||
|
||||
The upsert's `WHERE` clause only updates if specific fields changed:
|
||||
```sql
|
||||
WHERE product_metrics.current_stock IS DISTINCT FROM EXCLUDED.current_stock OR
|
||||
product_metrics.current_price IS DISTINCT FROM EXCLUDED.current_price OR ...
|
||||
```
|
||||
|
||||
Fields NOT checked include: `stockout_days_30d`, `margin_30d`, `gmroi_30d`, `demand_pattern`, `seasonality_index`, `sales_growth_*`, `service_level_30d`, and many others. If a product's stock, price, sales, and revenue haven't changed, the entire row is skipped even though growth metrics, variability, and other derived fields may need updating.
|
||||
|
||||
**Impact:** Most derived metrics (growth, demand patterns, seasonality) are stale for the majority of products. Products with steady sales but unchanged stock/price never get their growth metrics recalculated.
|
||||
|
||||
**Recommended fix:** Either:
|
||||
1. Remove the `WHERE` clause entirely (accept the performance cost of writing all rows every run)
|
||||
2. Add `last_calculated` age check: `OR product_metrics.last_calculated < NOW() - INTERVAL '7 days'`
|
||||
3. Add the missing fields to the change-detection check
|
||||
|
||||
**Status: FIXED.** Added 12 derived fields to the `IS DISTINCT FROM` check (`profit_30d`, `cogs_30d`, `margin_30d`, `stockout_days_30d`, `sell_through_30d`, `sales_growth_30d_vs_prev`, `revenue_growth_30d_vs_prev`, `demand_pattern`, `seasonal_pattern`, `seasonality_index`, `service_level_30d`, `fill_rate_30d`) plus a time-based safety net: `OR product_metrics.last_calculated < NOW() - INTERVAL '1 day'`. This guarantees every row is refreshed at least daily.
|
||||
|
||||
---
|
||||
|
||||
## MEDIUM Issues
|
||||
|
||||
### M1. Demand variability calculated only over activity days, not full 30-day window
|
||||
|
||||
**Location:** `update_product_metrics.sql`, DemandVariability CTE (lines 206-223)
|
||||
**Symptom:** Variance, std_dev, and CV are computed over only the days that appear in snapshots (activity days), not the full 30-day period including zero-sales days.
|
||||
|
||||
Example: Product 41141 (Mexican Poppy) sold 102 units in 30 days across only 3 snapshot days (1, 1, 100). The variance/CV is calculated over just those 3 data points instead of 30 (with 27 zero-sales days).
|
||||
|
||||
**Impact:**
|
||||
- CV is computed on sparse data (3-10 points instead of 30), making it statistically unreliable
|
||||
- Products with sporadic large orders appear less variable than they really are
|
||||
- `demand_pattern` classification is affected (stable/variable/sporadic/lumpy)
|
||||
|
||||
**Recommended fix:** Join against a generated 30-day date series and COALESCE missing days to 0 units sold before computing variance/stddev/CV.
|
||||
|
||||
**Status: FIXED.** Rewrote `DemandVariability` CTE to use `generate_series()` for the full 30-day date range, `CROSS JOIN` with distinct PIDs from snapshots, and `LEFT JOIN` actual snapshot data with `COALESCE(dps.units_sold, 0)` for missing days. Variance/stddev/CV now computed over all 30 data points.
|
||||
|
||||
---
|
||||
|
||||
### M2. `costeach` fallback to `price * 0.5` affects 32.5% of recent orders
|
||||
|
||||
**Location:** `orders.js`, line 600 and 634
|
||||
**Symptom:** When no cost record exists in `order_costs`, the import falls back to `price * 0.5`.
|
||||
|
||||
Data shows 9,839 of 30,266 recent orders (32.5%) use this fallback. Among these, 79 paid products have `costeach = 0` because `price = 0 * 0.5 = 0`, even though the product has a real cost_price.
|
||||
|
||||
The daily snapshot has a second line of defense (using `get_weighted_avg_cost()` and then `p.cost_price`), but the orders table's `costeach` column itself contains inaccurate data for ~1/3 of orders.
|
||||
|
||||
**Impact:** COGS calculations at the order level are approximate for 1/3 of orders. The snapshot's fallback chain mitigates this somewhat, but any analytics using `orders.costeach` directly will be affected.
|
||||
|
||||
**Status: FIXED.** Added `products.cost_price` as intermediate fallback: `COALESCE(oc.costeach, p.cost_price, oi.price * 0.5)`. The products table join was added to both the `order_totals` CTE and the outer SELECT in `orders.js`. Requires a full orders re-import to apply retroactively.
|
||||
|
||||
---
|
||||
|
||||
### M3. `lifetime_sales` uses MySQL `total_sold` (status >= 20) but orders import uses status >= 15
|
||||
|
||||
**Location:** `products.js` line 200 vs `orders.js` line 69
|
||||
**Symptom:** `total_sold` in the products table comes from MySQL with `order_status >= 20`, excluding status 15 (canceled) and 16 (combined). But the orders import fetches orders with `order_status >= 15`.
|
||||
|
||||
Verified in MySQL: For product 31286, `total_sold` (>=20) = 13,786 vs (>=15) = 13,905 (difference of 119 units).
|
||||
|
||||
**Impact:** `lifetime_sales` in product_metrics (sourced from `products.total_sold`) slightly understates compared to what the orders table contains. The `lifetime_revenue_quality` field correctly flags most as "estimated" since the orders table only covers ~5 years while `total_sold` is all-time. This is a minor inconsistency (< 1% difference).
|
||||
|
||||
**Status:** Accepted. < 1% difference, not worth the complexity of aligning thresholds.
|
||||
|
||||
---
|
||||
|
||||
### M4. `sell_through_30d` has 868 NULL values and 547 anomalous values for products with sales
|
||||
|
||||
**Location:** `update_product_metrics.sql`, lines 356-361
|
||||
**Formula:** `(sales_30d / (current_stock + sales_30d + returns_units_30d - received_qty_30d)) * 100`
|
||||
|
||||
- 868 products with sales but NULL sell_through (denominator = 0, which happens when `current_stock + sales - received = 0`, i.e. all stock came from receiving and was sold)
|
||||
- 259 products with sell_through > 100%
|
||||
- 288 products with negative sell_through
|
||||
|
||||
**Impact:** Sell-through rate is unreliable for products with significant receiving activity in the same period. The formula tries to approximate "beginning inventory" but the approximation breaks when current stock ≠ actual beginning stock (which is always, per issue C2).
|
||||
|
||||
**Status:** Will improve once C2 fix (historical stock) is deployed and snapshots are rebuilt, since `current_stock` in the formula will then reflect actual beginning inventory.
|
||||
|
||||
---
|
||||
|
||||
## LOW / INFORMATIONAL Issues
|
||||
|
||||
### L1. Snapshots only cover ~1,167 products/day out of 681K
|
||||
|
||||
Only products with order or receiving activity on a given day get snapshots. This is by design (the `ProductsWithActivity` CTE on line 133 of `update_daily_snapshots.sql`), but it means:
|
||||
- 560K+ products have zero snapshot history
|
||||
- Stockout tracking is impossible for products with no sales (they can't appear in snapshots)
|
||||
- The "avg_stock" metrics (avg_stock_units_30d, etc.) only average over activity days, not all 30 days
|
||||
|
||||
This is acceptable for storage efficiency but should be understood when interpreting metrics.
|
||||
|
||||
**Status:** Accepted (by design).
|
||||
|
||||
---
|
||||
|
||||
### L2. `detect_seasonal_pattern` function only compares current month to yearly average
|
||||
|
||||
The seasonality detection is simplistic: it compares current month's avg daily sales to yearly avg. This means:
|
||||
- It can only detect if the CURRENT month is above average, not identify historical seasonal patterns
|
||||
- Running in January vs July will give completely different results for the same product
|
||||
- The "peak_season" field always shows the current month/quarter when seasonal (not the actual peak)
|
||||
|
||||
This is noted as a P5 (low priority) feature and is adequate for a first pass but should not be relied upon for demand planning.
|
||||
|
||||
**Status: FIXED.** Rewrote `detect_seasonal_pattern` function to compare monthly average sales across the full last 12 months. Uses CV across months + peak-to-average ratio for classification: `strong` (CV > 0.5, peak > 150%), `moderate` (CV > 0.3, peak > 120%), `none`. Peak season now identifies the actual highest-sales month. Requires at least 3 months of data. Saved in `db/functions.sql`.
|
||||
|
||||
---
|
||||
|
||||
### L3. Free product with negative revenue in top sellers
|
||||
|
||||
Product 476848 ("Thank You, From ACOT!") shows 254 sales with -$1.00 revenue because one order applied a $1 discount to a $0 product. This is a data oddity, not a calculation bug. Could be addressed by excluding $0-price products from revenue metrics or by data cleanup.
|
||||
|
||||
**Status:** Accepted (data oddity, not a bug).
|
||||
|
||||
---
|
||||
|
||||
### L4. `landing_cost_price` is always NULL
|
||||
|
||||
`current_landing_cost_price` in product_metrics is mapped from `current_effective_cost` which is just `cost_price`. The `landing_cost_price` concept (cost + shipping + duties) is not implemented. The field exists but has no meaningful data.
|
||||
|
||||
**Status: FIXED.** Removed `landing_cost_price` from `db/schema.sql`, `current_landing_cost_price` from `db/metrics-schema-new.sql`, `update_product_metrics.sql`, and `backfill/populate_initial_product_metrics.sql`. Column should be dropped from the live database via `ALTER TABLE`.
|
||||
|
||||
---
|
||||
|
||||
### L5. Custom SQL functions not tracked in version control
|
||||
|
||||
All 6 custom functions (`calculate_sales_velocity`, `get_weighted_avg_cost`, `safe_divide`, `std_numeric`, `classify_demand_pattern`, `detect_seasonal_pattern`) and the `category_hierarchy` materialized view exist only in the database. They are not defined in any migration or schema file in the repository.
|
||||
|
||||
If the database needs to be recreated, these would be lost.
|
||||
|
||||
**Status: FIXED.** All 6 functions and the `category_hierarchy` materialized view definition saved to `inventory-server/db/functions.sql`. File is re-runnable via `psql -f functions.sql`.
|
||||
|
||||
---
|
||||
|
||||
### L6. `get_weighted_avg_cost` limited to last 10 receivings
|
||||
|
||||
The function `LIMIT 10` for performance, but this means products with many small receivings may not accurately reflect the true weighted average cost if the cost has changed significantly beyond the last 10 receiving records.
|
||||
|
||||
**Status: FIXED.** Removed `LIMIT 10` from `get_weighted_avg_cost`. Data shows max receivings per product is 142 (p95 = 11, avg = 3), so performance impact is negligible. Updated definition in `db/functions.sql`.
|
||||
|
||||
---
|
||||
|
||||
## Verification Summary
|
||||
|
||||
### What's Working Correctly
|
||||
|
||||
| Check | Result |
|
||||
|-------|--------|
|
||||
| 30d sales: product_metrics vs orders vs snapshots | **MATCH** (verified top 10 sellers) |
|
||||
| Replenishment formula: manual calc vs stored | **MATCH** (verified 10 products) |
|
||||
| Brand metrics vs sum of product_metrics | **MATCH** (0 difference across all brands) |
|
||||
| Order status mapping (numeric → text) | **CORRECT** (all statuses mapped, no numeric remain) |
|
||||
| Cost price: PostgreSQL vs MySQL source | **MATCH** (within rounding, verified 5 products) |
|
||||
| total_sold: PostgreSQL vs MySQL source | **MATCH** (verified 5 products) |
|
||||
| Category rollups (rolled-up > direct for parents) | **CORRECT** |
|
||||
| ABC classification distribution | **REASONABLE** (A: 8K, B: 12.5K, C: 113K) |
|
||||
| Lead time calculation (PO → receiving) | **CORRECT** (verified examples) |
|
||||
|
||||
### Data Overview
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Total products | 681,912 |
|
||||
| Products in product_metrics | 681,912 (100%) |
|
||||
| Products with 30d sales | 10,291 (1.5%) |
|
||||
| Products with negative profit & revenue | 139 (mostly cost > price) |
|
||||
| Products with negative stock | 0 |
|
||||
| Snapshot date range | 2020-06-18 to 2026-02-08 |
|
||||
| Avg products per snapshot day | 1,167 |
|
||||
| Order date range | 2020-06-18 to 2026-02-08 |
|
||||
| Total orders | 2,885,825 |
|
||||
| 'returned' status orders | 0 (returns via negative quantity only) |
|
||||
|
||||
---
|
||||
|
||||
## Fix Status Summary
|
||||
|
||||
| Issue | Severity | Status | Deployment Action Needed |
|
||||
|-------|----------|--------|--------------------------|
|
||||
| C1 | Critical | Owner resolving | Rebuild daily snapshots |
|
||||
| C2 | Critical | **FIXED** | Run import, rebuild daily snapshots |
|
||||
| C3 | Critical | **FIXED** | Deploy updated `update_product_metrics.sql` |
|
||||
| M1 | Medium | **FIXED** | Deploy updated `update_product_metrics.sql` |
|
||||
| M2 | Medium | **FIXED** | Full orders re-import (`--full`) |
|
||||
| M3 | Medium | Accepted | None |
|
||||
| M4 | Medium | Pending C2 | Will improve after C2 deployment |
|
||||
| L1 | Low | Accepted | None |
|
||||
| L2 | Low | **FIXED** | Deploy `db/functions.sql` to database |
|
||||
| L3 | Low | Accepted | None |
|
||||
| L4 | Low | **FIXED** | `ALTER TABLE` to drop columns |
|
||||
| L5 | Low | **FIXED** | None (file committed) |
|
||||
| L6 | Low | **FIXED** | Deploy `db/functions.sql` to database |
|
||||
|
||||
### Deployment Steps
|
||||
|
||||
1. Deploy `db/functions.sql` to PostgreSQL: `psql -d inventory_db -f db/functions.sql` (L2, L6)
|
||||
2. Run import (includes stock snapshots first load) (C2, M2)
|
||||
3. Drop stale columns: `ALTER TABLE products DROP COLUMN IF EXISTS landing_cost_price; ALTER TABLE product_metrics DROP COLUMN IF EXISTS current_landing_cost_price;` (L4)
|
||||
4. Rebuild daily snapshots (C1, C2)
|
||||
5. Re-run metrics calculation (C3, M1 take effect automatically)
|
||||
375
PRODUCT_IMPORT_ENHANCEMENTS.md
Normal file
375
PRODUCT_IMPORT_ENHANCEMENTS.md
Normal file
@@ -0,0 +1,375 @@
|
||||
# Product Import Module - Enhancement & Issues Outline
|
||||
|
||||
This document outlines the investigation and implementation requirements for each requested enhancement to the product import module.
|
||||
|
||||
---
|
||||
|
||||
## 1. UPC Import - Strip Quotes and Spaces ✅ IMPLEMENTED
|
||||
|
||||
**Issue:** When importing UPCs, strip `'`, `"` characters and any spaces, leaving only numbers.
|
||||
|
||||
**Implementation (Completed):**
|
||||
- Modified `normalizeUpcValue()` in [Import.tsx:661-667](inventory/src/pages/Import.tsx#L661-L667)
|
||||
- Strips single quotes, double quotes, smart quotes (`'"`), and whitespace before processing
|
||||
- Then handles scientific notation and extracts only digits
|
||||
|
||||
**Files Modified:**
|
||||
- `inventory/src/pages/Import.tsx` - `normalizeUpcValue()` function
|
||||
|
||||
---
|
||||
|
||||
## 2. AI Context Columns in Validation Payloads ✅ IMPLEMENTED
|
||||
|
||||
**Issue:** The match columns step has a setting to use a field only for AI context (`isAiSupplemental`). Update AI description validation to include any columns selected with this option in the payload. Also include in sanity check payload. Not needed for names.
|
||||
|
||||
**Current Implementation:**
|
||||
- AI Supplemental toggle: [MatchColumnsStep.tsx:102-118](inventory/src/components/product-import/steps/MatchColumnsStep/MatchColumnsStep.tsx#L102-L118)
|
||||
- AI supplemental data stored in `__aiSupplemental` field on each row
|
||||
- Description payload builder: [inlineAiPayload.ts:183-195](inventory/src/components/product-import/steps/ValidationStep/utils/inlineAiPayload.ts#L183-L195)
|
||||
|
||||
**Implementation:**
|
||||
1. **Update `buildDescriptionValidationPayload()` in `inlineAiPayload.ts`** to include AI supplemental data:
|
||||
```typescript
|
||||
export const buildDescriptionValidationPayload = (
|
||||
row: Data<string>,
|
||||
fieldOptions: FieldOptionsMap,
|
||||
productLinesCache: Map<string, SelectOption[]>,
|
||||
sublinesCache: Map<string, SelectOption[]>
|
||||
) => {
|
||||
const payload: Record<string, unknown> = {
|
||||
name: row.name,
|
||||
description: row.description,
|
||||
company_name: getFieldOptionLabel(row.company, fieldOptions, 'company'),
|
||||
company_id: row.company,
|
||||
categories: getFieldOptionLabel(row.category, fieldOptions, 'category'),
|
||||
};
|
||||
|
||||
// Add AI supplemental context if present
|
||||
if (row.__aiSupplemental && typeof row.__aiSupplemental === 'object') {
|
||||
payload.additional_context = row.__aiSupplemental;
|
||||
}
|
||||
|
||||
return payload;
|
||||
};
|
||||
```
|
||||
|
||||
2. **Update sanity check payload** - Locate sanity check submission logic and include `__aiSupplemental` data
|
||||
|
||||
3. **Verify `__aiSupplemental` is properly populated** from MatchColumnsStep when columns are marked as AI context only
|
||||
|
||||
**Files to Modify:**
|
||||
- `inventory/src/components/product-import/steps/ValidationStep/utils/inlineAiPayload.ts`
|
||||
- Backend sanity check endpoint (if separate from description validation)
|
||||
- Verify data flow in `MatchColumnsStep.tsx` → `ValidationStep`
|
||||
|
||||
---
|
||||
|
||||
## 3. Fresh Taxonomy Data Per Session ✅ IMPLEMENTED
|
||||
|
||||
**Issue:** Ensure taxonomy data is brought in fresh with each session - cache should be invalidated if we exit the import flow and start again.
|
||||
|
||||
**Current Implementation:**
|
||||
- Field options cached 5 minutes: [ValidationStep/index.tsx:128-133](inventory/src/components/product-import/steps/ValidationStep/index.tsx#L128-L133)
|
||||
- Product lines cache: `productLinesCache` in Zustand store
|
||||
- Sublines cache: `sublinesCache` in Zustand store
|
||||
- Caches set to 10-minute stale time
|
||||
|
||||
**Implementation:**
|
||||
1. **Add cache invalidation on import flow mount/unmount** in `UploadFlow.tsx`:
|
||||
```typescript
|
||||
useEffect(() => {
|
||||
// On mount - invalidate import-related query cache
|
||||
queryClient.invalidateQueries({ queryKey: ['import-field-options'] });
|
||||
|
||||
return () => {
|
||||
// On unmount - clear caches
|
||||
queryClient.removeQueries({ queryKey: ['import-field-options'] });
|
||||
queryClient.removeQueries({ queryKey: ['product-lines'] });
|
||||
queryClient.removeQueries({ queryKey: ['sublines'] });
|
||||
};
|
||||
}, []);
|
||||
```
|
||||
|
||||
2. **Clear Zustand store caches** when exiting import flow:
|
||||
- Add action to `validationStore.ts` to clear `productLinesCache` and `sublinesCache`
|
||||
- Call this action on unmount of `UploadFlow` or when navigating away
|
||||
|
||||
3. **Consider adding a `sessionId`** that changes on each import flow start, used as part of cache keys
|
||||
|
||||
**Files to Modify:**
|
||||
- `inventory/src/components/product-import/steps/UploadFlow.tsx` - Add cleanup effect
|
||||
- `inventory/src/components/product-import/steps/ValidationStep/store/validationStore.ts` - Add cache clear action
|
||||
- Potentially `inventory/src/components/product-import/steps/ValidationStep/index.tsx` - Query key updates
|
||||
|
||||
---
|
||||
|
||||
## 4. Save Template from Confirmation Page ✅ IMPLEMENTED
|
||||
|
||||
**Issue:** Add option to save rows of submitted data as a new template on the confirmation page after completing the import flow. Verify this works with new validation step changes.
|
||||
|
||||
**Current Implementation:**
|
||||
- **Import Results section already exists** inline in [Import.tsx:968-1150](inventory/src/pages/Import.tsx#L968-L1150)
|
||||
- Shows created products (lines 1021-1097) with image, name, UPC, item number
|
||||
- Shows errored products (lines 1100-1138) with error details
|
||||
- "Fix products with errors" button resumes validation flow for failed items
|
||||
- Template saving logic in ValidationStep: [useTemplateManagement.ts:204-266](inventory/src/components/product-import/steps/ValidationStep/hooks/useTemplateManagement.ts#L204-L266)
|
||||
- Saves via `POST /api/templates`
|
||||
- `importOutcome.submittedProducts` contains the full product data for each row
|
||||
|
||||
**Implementation:**
|
||||
1. **Add "Save as Template" button** to each created product row in the results section (around line 1087-1092 in Import.tsx):
|
||||
```typescript
|
||||
// Add button after the item number display
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => handleSaveAsTemplate(index)}
|
||||
>
|
||||
<BookmarkPlus className="h-4 w-4" />
|
||||
</Button>
|
||||
```
|
||||
|
||||
2. **Add state and dialog** for template saving in Import.tsx:
|
||||
```typescript
|
||||
const [templateSaveDialogOpen, setTemplateSaveDialogOpen] = useState(false);
|
||||
const [selectedProductForTemplate, setSelectedProductForTemplate] = useState<NormalizedProduct | null>(null);
|
||||
```
|
||||
|
||||
3. **Extract/reuse template save logic** from `useTemplateManagement.ts`:
|
||||
- The `saveNewTemplate()` function (lines 204-266) can be extracted into a shared utility
|
||||
- Or create a `SaveTemplateDialog` component that can be used in both places
|
||||
- Key fields needed: `company` (for template name), `product_type`, and all product field values
|
||||
|
||||
4. **Data mapping consideration:**
|
||||
- `importOutcome.submittedProducts` uses `NormalizedProduct` type
|
||||
- Templates expect raw field values - may need to map back from normalized format
|
||||
- Exclude metadata fields: `['id', '__index', '__meta', '__template', '__original', '__corrected', '__changes', '__aiSupplemental']`
|
||||
|
||||
**Files to Modify:**
|
||||
- `inventory/src/pages/Import.tsx` - Add save template button, state, and dialog
|
||||
- Consider creating `inventory/src/components/product-import/SaveTemplateDialog.tsx` for reusability
|
||||
- Potentially extract core save logic from `useTemplateManagement.ts` into shared utility
|
||||
|
||||
---
|
||||
|
||||
## 5. Sheet Preview on Select Sheet Step ✅ IMPLEMENTED
|
||||
|
||||
**Issue:** On the select sheet step, show a preview of the first 10 lines or so of each sheet underneath the options.
|
||||
|
||||
**Implementation (Completed):**
|
||||
- Added `workbook` prop to `SelectSheetStep` component
|
||||
- Added `sheetPreviews` memoized computation using `XLSXLib.utils.sheet_to_json()`
|
||||
- Shows first 10 rows, 8 columns max per sheet
|
||||
- Added `truncateCell()` helper to limit cell content to 30 characters with ellipsis
|
||||
- Each sheet option is now a clickable card with:
|
||||
- Radio button and sheet name
|
||||
- Row count indicator
|
||||
- Scrollable preview table with horizontal scroll
|
||||
- Selected state highlighted with primary border
|
||||
- Updated `UploadFlow.tsx` to pass workbook prop
|
||||
|
||||
**Files Modified:**
|
||||
- `inventory/src/components/product-import/steps/SelectSheetStep/SelectSheetStep.tsx`
|
||||
- `inventory/src/components/product-import/steps/UploadFlow.tsx`
|
||||
|
||||
---
|
||||
|
||||
## 6. Empty Row Removal ✅ IMPLEMENTED
|
||||
|
||||
**Issue:** When importing a sheet, automatically remove completely empty rows.
|
||||
|
||||
**Current Implementation:**
|
||||
- Empty columns are filtered: [MatchColumnsStep.tsx:616-634](inventory/src/components/product-import/steps/MatchColumnsStep/MatchColumnsStep.tsx#L616-L634)
|
||||
- A "Remove empty/duplicates" button exists that removes empty rows, single-value rows, AND duplicates
|
||||
- The automatic removal should ONLY remove completely empty rows, not duplicates or single-value rows
|
||||
|
||||
**Implementation (Completed):**
|
||||
- Added `isRowCompletelyEmpty()` helper function to [SelectHeaderStep.tsx](inventory/src/components/product-import/steps/SelectHeaderStep/SelectHeaderStep.tsx)
|
||||
- Added `useMemo` to filter empty rows on initial data load
|
||||
- Uses `Object.values(row)` to check all cell values (matches existing button logic)
|
||||
- Only removes rows where ALL values are undefined, null, or whitespace-only strings
|
||||
- Manual "Remove Empty/Duplicates" button still available for additional cleanup (duplicates, single-value rows)
|
||||
|
||||
**Files Modified:**
|
||||
- `inventory/src/components/product-import/steps/SelectHeaderStep/SelectHeaderStep.tsx`
|
||||
|
||||
---
|
||||
|
||||
## 7. Unit Conversion for Weight/Dimensions ✅ IMPLEMENTED
|
||||
|
||||
**Issue:** Add unit conversion feature for weight and dimensions columns - similar to calculator button on cost/msrp, add button that opens popover with options to convert grams → oz, lbs → oz for the whole column at once.
|
||||
|
||||
**Current Implementation:**
|
||||
- Calculator button on price columns: [ValidationTable.tsx:1491-1627](inventory/src/components/product-import/steps/ValidationStep/components/ValidationTable.tsx#L1491-L1627)
|
||||
- `PriceColumnHeader` component shows calculator icon on hover
|
||||
- Weight field defined in config with validation
|
||||
|
||||
**Implementation:**
|
||||
1. **Create `UnitConversionColumnHeader` component** (similar to `PriceColumnHeader`):
|
||||
```typescript
|
||||
const UnitConversionColumnHeader = ({ field, table }) => {
|
||||
const [showPopover, setShowPopover] = useState(false);
|
||||
|
||||
const conversions = {
|
||||
weight: [
|
||||
{ label: 'Grams → Ounces', factor: 0.035274 },
|
||||
{ label: 'Pounds → Ounces', factor: 16 },
|
||||
{ label: 'Kilograms → Ounces', factor: 35.274 },
|
||||
],
|
||||
dimensions: [
|
||||
{ label: 'Centimeters → Inches', factor: 0.393701 },
|
||||
{ label: 'Millimeters → Inches', factor: 0.0393701 },
|
||||
]
|
||||
};
|
||||
|
||||
const applyConversion = (factor: number) => {
|
||||
// Batch update all cells in column
|
||||
table.rows.forEach((row, index) => {
|
||||
const currentValue = parseFloat(row[field.key]);
|
||||
if (!isNaN(currentValue)) {
|
||||
updateCell(index, field.key, (currentValue * factor).toFixed(2));
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
return (
|
||||
<Popover open={showPopover} onOpenChange={setShowPopover}>
|
||||
<PopoverTrigger>
|
||||
<Scale className="h-4 w-4" /> {/* or similar icon */}
|
||||
</PopoverTrigger>
|
||||
<PopoverContent>
|
||||
{conversions[fieldType].map(conv => (
|
||||
<Button key={conv.label} onClick={() => applyConversion(conv.factor)}>
|
||||
{conv.label}
|
||||
</Button>
|
||||
))}
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
);
|
||||
};
|
||||
```
|
||||
|
||||
2. **Identify weight/dimension fields** in config:
|
||||
- `weight_oz`, `length_in`, `width_in`, `height_in` (check actual field keys)
|
||||
|
||||
3. **Add to column header render logic** in ValidationTable
|
||||
|
||||
**Files to Modify:**
|
||||
- `inventory/src/components/product-import/steps/ValidationStep/components/ValidationTable.tsx`
|
||||
- Potentially create new component file for `UnitConversionColumnHeader`
|
||||
- Update column header rendering to use new component for weight/dimension fields
|
||||
|
||||
---
|
||||
|
||||
## 8. Expanded MSRP Auto-Fill from Cost ✅ IMPLEMENTED
|
||||
|
||||
**Issue:** Expand auto-fill functionality for MSRP from cost - open small popover with options for 2x, 2.1x, 2.2x, 2.3x, 2.4x, 2.5x multipliers, plus checkbox to round up to nearest 9.
|
||||
|
||||
**Current Implementation:**
|
||||
- Calculator on MSRP column: [ValidationTable.tsx:1540-1584](inventory/src/components/product-import/steps/ValidationStep/components/ValidationTable.tsx#L1540-L1584)
|
||||
- Currently only does `Cost × 2` then subtracts 0.01 if whole number
|
||||
|
||||
**Implementation:**
|
||||
1. **Replace simple click with popover** in `PriceColumnHeader`:
|
||||
```typescript
|
||||
const [selectedMultiplier, setSelectedMultiplier] = useState(2.0);
|
||||
const [roundToNine, setRoundToNine] = useState(false);
|
||||
const multipliers = [2.0, 2.1, 2.2, 2.3, 2.4, 2.5];
|
||||
|
||||
const roundUpToNine = (value: number): number => {
|
||||
// 1.41 → 1.49, 2.78 → 2.79, 12.32 → 12.39
|
||||
const wholePart = Math.floor(value);
|
||||
const decimal = value - wholePart;
|
||||
if (decimal <= 0.09) return wholePart + 0.09;
|
||||
if (decimal <= 0.19) return wholePart + 0.19;
|
||||
// ... continue pattern, or:
|
||||
const lastDigit = Math.floor(decimal * 10);
|
||||
return wholePart + (lastDigit / 10) + 0.09;
|
||||
};
|
||||
|
||||
const calculateMsrp = (cost: number): number => {
|
||||
let result = cost * selectedMultiplier;
|
||||
if (roundToNine) {
|
||||
result = roundUpToNine(result);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
```
|
||||
|
||||
2. **Create popover UI**:
|
||||
```tsx
|
||||
<Popover>
|
||||
<PopoverTrigger><Calculator className="h-4 w-4" /></PopoverTrigger>
|
||||
<PopoverContent className="w-48">
|
||||
<div className="space-y-2">
|
||||
<Label>Multiplier</Label>
|
||||
<div className="grid grid-cols-3 gap-1">
|
||||
{multipliers.map(m => (
|
||||
<Button
|
||||
key={m}
|
||||
variant={selectedMultiplier === m ? 'default' : 'outline'}
|
||||
size="sm"
|
||||
onClick={() => setSelectedMultiplier(m)}
|
||||
>
|
||||
{m}x
|
||||
</Button>
|
||||
))}
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Checkbox checked={roundToNine} onCheckedChange={setRoundToNine} />
|
||||
<Label>Round to .X9</Label>
|
||||
</div>
|
||||
<Button onClick={applyCalculation} className="w-full">
|
||||
Apply
|
||||
</Button>
|
||||
</div>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
```
|
||||
|
||||
**Files to Modify:**
|
||||
- `inventory/src/components/product-import/steps/ValidationStep/components/ValidationTable.tsx` - `PriceColumnHeader` component
|
||||
|
||||
---
|
||||
|
||||
## 9. Debug Mode - Skip API Submission ✅ IMPLEMENTED
|
||||
|
||||
**Issue:** Add a third switch in the footer of image upload step (visible only to users with `admin:debug` permission) that will not submit data to any API, only complete the process and show results page as if it had worked.
|
||||
|
||||
**Implementation (Completed):**
|
||||
- Added `skipApiSubmission` state to `ImageUploadStep.tsx`
|
||||
- Added amber-colored "Skip API (Debug)" switch (visible only with `admin:debug` permission)
|
||||
- When skip is active, "Use Test API" and "Use Test Database" switches are hidden
|
||||
- Added `skipApiSubmission?: boolean` to `SubmitOptions` type in `types.ts`
|
||||
- In `Import.tsx`, when `skipApiSubmission` is true:
|
||||
- Skips the actual API call entirely
|
||||
- Generates mock success response with mock PIDs
|
||||
- Shows `[DEBUG]` prefix in toast and result message
|
||||
- Displays results page as if submission succeeded
|
||||
|
||||
**Files Modified:**
|
||||
- `inventory/src/components/product-import/types.ts` - Added `skipApiSubmission` to `SubmitOptions`
|
||||
- `inventory/src/components/product-import/steps/ImageUploadStep/ImageUploadStep.tsx` - Added switch UI
|
||||
- `inventory/src/pages/Import.tsx` - Added skip logic in `handleData()`
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
| # | Enhancement | Complexity | Status |
|
||||
|---|-------------|------------|--------|
|
||||
| 1 | Strip UPC quotes/spaces | Low | ✅ Implemented |
|
||||
| 2 | AI context in validation | Medium | ✅ Implemented |
|
||||
| 3 | Fresh taxonomy per session | Medium | ✅ Implemented |
|
||||
| 4 | Save template from confirmation | Medium-High | ✅ Implemented |
|
||||
| 5 | Sheet preview | Low-Medium | ✅ Implemented |
|
||||
| 6 | Remove empty rows | Low | ✅ Implemented |
|
||||
| 7 | Unit conversion | Medium | ✅ Implemented |
|
||||
| 8 | MSRP multiplier options | Medium | ✅ Implemented |
|
||||
| 9 | Debug skip API | Low-Medium | ✅ Implemented |
|
||||
|
||||
**Implemented:** 9 of 9 items - All enhancements complete!
|
||||
|
||||
---
|
||||
|
||||
*Document generated: 2026-01-25*
|
||||
204
docs/PERMISSIONS.md
Normal file
204
docs/PERMISSIONS.md
Normal file
@@ -0,0 +1,204 @@
|
||||
# Permission System Documentation
|
||||
|
||||
This document outlines the permission system implemented in the Inventory Manager application.
|
||||
|
||||
## Permission Structure
|
||||
|
||||
Permissions follow this naming convention:
|
||||
|
||||
- Page access: `access:{page_name}`
|
||||
- Settings sections: `settings:{section_name}`
|
||||
- Admin features: `admin:{feature}`
|
||||
|
||||
Examples:
|
||||
- `access:products` - Can access the Products page
|
||||
- `settings:user_management` - Can access User Management settings
|
||||
- `admin:debug` - Can see debug information
|
||||
|
||||
## Permission Components
|
||||
|
||||
### PermissionGuard
|
||||
|
||||
The core component that conditionally renders content based on permissions.
|
||||
|
||||
```tsx
|
||||
<PermissionGuard
|
||||
permission="settings:user_management"
|
||||
fallback={<p>No permission</p>}
|
||||
>
|
||||
<button>Manage Users</button>
|
||||
</PermissionGuard>
|
||||
```
|
||||
|
||||
Options:
|
||||
- `permission`: Single permission code
|
||||
- `anyPermissions`: Array of permissions (ANY match grants access)
|
||||
- `allPermissions`: Array of permissions (ALL required)
|
||||
- `adminOnly`: For admin-only sections
|
||||
- `page`: Page name (checks `access:{page}` permission)
|
||||
- `fallback`: Content to show if permission check fails
|
||||
|
||||
### PermissionProtectedRoute
|
||||
|
||||
Protects entire pages based on page access permissions.
|
||||
|
||||
```tsx
|
||||
<Route path="/products" element={
|
||||
<PermissionProtectedRoute page="products">
|
||||
<Products />
|
||||
</PermissionProtectedRoute>
|
||||
} />
|
||||
```
|
||||
|
||||
### ProtectedSection
|
||||
|
||||
Protects sections within a page based on action permissions.
|
||||
|
||||
```tsx
|
||||
<ProtectedSection page="products" action="create">
|
||||
<button>Add Product</button>
|
||||
</ProtectedSection>
|
||||
```
|
||||
|
||||
### PermissionButton
|
||||
|
||||
Button that automatically handles permissions.
|
||||
|
||||
```tsx
|
||||
<PermissionButton
|
||||
page="products"
|
||||
action="create"
|
||||
onClick={handleCreateProduct}
|
||||
>
|
||||
Add Product
|
||||
</PermissionButton>
|
||||
```
|
||||
|
||||
### SettingsSection
|
||||
|
||||
Specific component for settings with built-in permission checks.
|
||||
|
||||
```tsx
|
||||
<SettingsSection
|
||||
title="System Settings"
|
||||
description="Configure global settings"
|
||||
permission="settings:global"
|
||||
>
|
||||
{/* Settings content */}
|
||||
</SettingsSection>
|
||||
```
|
||||
|
||||
## Permission Hooks
|
||||
|
||||
### usePermissions
|
||||
|
||||
Core hook for checking any permission.
|
||||
|
||||
```tsx
|
||||
const { hasPermission, hasPageAccess, isAdmin } = usePermissions();
|
||||
if (hasPermission('settings:user_management')) {
|
||||
// Can access user management
|
||||
}
|
||||
```
|
||||
|
||||
### usePagePermission
|
||||
|
||||
Specialized hook for page-level permissions.
|
||||
|
||||
```tsx
|
||||
const { canView, canCreate, canEdit, canDelete } = usePagePermission('products');
|
||||
if (canView()) {
|
||||
// Can view products
|
||||
}
|
||||
```
|
||||
|
||||
## Database Schema
|
||||
|
||||
Permissions are stored in the database:
|
||||
- `permissions` table: Stores all available permissions
|
||||
- `user_permissions` junction table: Maps permissions to users
|
||||
|
||||
Admin users automatically have all permissions.
|
||||
|
||||
## Implemented Permission Codes
|
||||
|
||||
### Page Access Permissions
|
||||
| Code | Description |
|
||||
|------|-------------|
|
||||
| `access:dashboard` | Access to Dashboard page |
|
||||
| `access:overview` | Access to Overview page |
|
||||
| `access:products` | Access to Products page |
|
||||
| `access:categories` | Access to Categories page |
|
||||
| `access:brands` | Access to Brands page |
|
||||
| `access:vendors` | Access to Vendors page |
|
||||
| `access:purchase_orders` | Access to Purchase Orders page |
|
||||
| `access:analytics` | Access to Analytics page |
|
||||
| `access:forecasting` | Access to Forecasting page |
|
||||
| `access:import` | Access to Import page |
|
||||
| `access:settings` | Access to Settings page |
|
||||
| `access:chat` | Access to Chat Archive page |
|
||||
|
||||
### Settings Permissions
|
||||
| Code | Description |
|
||||
|------|-------------|
|
||||
| `settings:global` | Access to Global Settings section |
|
||||
| `settings:products` | Access to Product Settings section |
|
||||
| `settings:vendors` | Access to Vendor Settings section |
|
||||
| `settings:data_management` | Access to Data Management settings |
|
||||
| `settings:calculation_settings` | Access to Calculation Settings |
|
||||
| `settings:library_management` | Access to Image Library Management |
|
||||
| `settings:performance_metrics` | Access to Performance Metrics |
|
||||
| `settings:prompt_management` | Access to AI Prompt Management |
|
||||
| `settings:stock_management` | Access to Stock Management |
|
||||
| `settings:templates` | Access to Template Management |
|
||||
| `settings:user_management` | Access to User Management |
|
||||
|
||||
### Admin Permissions
|
||||
| Code | Description |
|
||||
|------|-------------|
|
||||
| `admin:debug` | Can see debug information and features |
|
||||
|
||||
## Implementation Examples
|
||||
|
||||
### Page Protection
|
||||
|
||||
In `App.tsx`:
|
||||
```tsx
|
||||
<Route path="/products" element={
|
||||
<PermissionProtectedRoute page="products">
|
||||
<Products />
|
||||
</PermissionProtectedRoute>
|
||||
} />
|
||||
```
|
||||
|
||||
### Component Level Protection
|
||||
|
||||
```tsx
|
||||
const { hasPermission } = usePermissions();
|
||||
|
||||
function handleAction() {
|
||||
if (!hasPermission('settings:user_management')) {
|
||||
toast.error("You don't have permission");
|
||||
return;
|
||||
}
|
||||
// Action logic
|
||||
}
|
||||
```
|
||||
|
||||
### UI Element Protection
|
||||
|
||||
```tsx
|
||||
<PermissionGuard permission="settings:user_management">
|
||||
<button onClick={handleManageUsers}>
|
||||
Manage Users
|
||||
</button>
|
||||
</PermissionGuard>
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- **Page Access**: These permissions control which pages a user can navigate to
|
||||
- **Settings Access**: These permissions control access to different sections within the Settings page
|
||||
- **Admin Features**: Special permissions for administrative functions
|
||||
- **CRUD Operations**: The application currently focuses on viewing and managing data rather than creating/editing/deleting individual records
|
||||
- **User Management**: User CRUD operations are handled through the settings interface rather than dedicated user management pages
|
||||
396
docs/ValidationStep-Refactoring-Plan.md
Normal file
396
docs/ValidationStep-Refactoring-Plan.md
Normal file
@@ -0,0 +1,396 @@
|
||||
# ValidationStep Component Refactoring Plan
|
||||
|
||||
## Overview
|
||||
|
||||
This document outlines a comprehensive plan to refactor the current ValidationStep component (4000+ lines) into a more maintainable, modular structure. The new implementation will be developed alongside the existing component without modifying the original code. Once completed, the previous step in the workflow will offer the option to continue to either the original ValidationStep or the new implementation.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Current Component Analysis](#current-component-analysis)
|
||||
2. [New Architecture Design](#new-architecture-design)
|
||||
3. [Component Structure](#component-structure)
|
||||
4. [State Management](#state-management)
|
||||
5. [Key Features Implementation](#key-features-implementation)
|
||||
6. [Integration Plan](#integration-plan)
|
||||
7. [Testing Strategy](#testing-strategy)
|
||||
8. [Project Timeline](#project-timeline)
|
||||
9. [Design Principles](#design-principles)
|
||||
10. [Appendix: Function Reference](#appendix-function-reference)
|
||||
|
||||
## Current Component Analysis
|
||||
|
||||
The current ValidationStep component has several issues:
|
||||
|
||||
- **Size**: At over 4000 lines, it's difficult to maintain and understand
|
||||
- **Multiple responsibilities**: Handles validation, UI rendering, template management, and more
|
||||
- **Special cases**: Contains numerous special case handlers and exceptions
|
||||
- **Complex state management**: State is distributed across multiple useState calls
|
||||
- **Tightly coupled concerns**: UI, validation logic, and business rules are intertwined
|
||||
|
||||
### Key Features to Preserve
|
||||
|
||||
1. **Data Validation**
|
||||
- Field-level validation (required, regex, unique)
|
||||
- Row-level validation (supplier, company fields)
|
||||
- UPC validation with API integration
|
||||
- AI-assisted validation
|
||||
|
||||
2. **Template Management**
|
||||
- Saving, loading, and applying templates
|
||||
- Template-based validation
|
||||
|
||||
3. **UI Components**
|
||||
- Editable table with specialized cell renderers
|
||||
- Error display and management
|
||||
- Filtering and sorting capabilities
|
||||
- Status indicators and progress tracking
|
||||
|
||||
4. **Special Field Handling**
|
||||
- Input fields with price formatting
|
||||
- Multi-input fields with separator configuration
|
||||
- Select fields with dropdown options
|
||||
- Checkbox fields with boolean value mapping
|
||||
- UPC fields with specialized validation
|
||||
|
||||
5. **User Interaction Flows**
|
||||
- Tab and keyboard navigation
|
||||
- Bulk operations (select all, apply template)
|
||||
- Row validation on value change
|
||||
- Error reporting and display
|
||||
|
||||
## New Architecture Design
|
||||
|
||||
The new architecture will follow these principles:
|
||||
|
||||
1. **Separation of Concerns**
|
||||
- UI rendering separate from business logic
|
||||
- Validation logic isolated from state management
|
||||
- Clear interfaces between components
|
||||
|
||||
2. **Composable Components**
|
||||
- Small, focused components with single responsibilities
|
||||
- Reusable pattern for different field types
|
||||
|
||||
3. **Centralized State Management**
|
||||
- Custom hooks for state management
|
||||
- Clear data flow patterns
|
||||
- Reduced prop drilling
|
||||
|
||||
4. **Consistent Error Handling**
|
||||
- Standardized error structure
|
||||
- Predictable error propagation
|
||||
- User-friendly error display
|
||||
|
||||
5. **Performance Optimization**
|
||||
- Virtualized table rendering
|
||||
- Memoization of expensive computations
|
||||
- Deferred validation for better user experience
|
||||
|
||||
## Component Structure
|
||||
|
||||
The new ValidationStepNew folder has the following structure:
|
||||
|
||||
```
|
||||
ValidationStepNew/
|
||||
├── index.tsx # Main entry point that composes all pieces
|
||||
├── components/ # UI Components
|
||||
│ ├── ValidationContainer.tsx # Main wrapper component
|
||||
│ ├── ValidationTable.tsx # Table implementation
|
||||
│ ├── ValidationCell.tsx # Cell component
|
||||
│ ├── ValidationSidebar.tsx # Sidebar with controls
|
||||
│ ├── ValidationToolbar.tsx # Top toolbar (removed as unnecessary)
|
||||
│ ├── TemplateManager.tsx # Template management
|
||||
│ ├── FilterPanel.tsx # Filtering interface (integrated into Container)
|
||||
│ └── cells/ # Specialized cell renderers
|
||||
│ ├── InputCell.tsx
|
||||
│ ├── SelectCell.tsx
|
||||
│ ├── MultiInputCell.tsx
|
||||
│ └── CheckboxCell.tsx
|
||||
├── hooks/ # Custom hooks
|
||||
│ ├── useValidationState.tsx # Main state management
|
||||
│ ├── useTemplates.tsx # Template-related logic (integrated into ValidationState)
|
||||
│ ├── useFilters.tsx # Filtering logic (integrated into ValidationState)
|
||||
│ └── useUpcValidation.tsx # UPC-specific validation
|
||||
└── utils/ # Utility functions
|
||||
├── validationUtils.ts # Validation helper functions
|
||||
├── formatters.ts # Value formatting utilities
|
||||
└── constants.ts # Constant values and configuration
|
||||
```
|
||||
|
||||
### Component Responsibilities
|
||||
|
||||
#### ValidationContainer
|
||||
- Main container component
|
||||
- Coordinates between subcomponents
|
||||
- Manages global state
|
||||
- Handles navigation events (next, back)
|
||||
- Contains filter controls
|
||||
|
||||
#### ValidationTable
|
||||
- Displays the data in tabular form
|
||||
- Manages selection state
|
||||
- Handles keyboard navigation
|
||||
- Integrates with TanStack Table
|
||||
- Displays properly styled rows and cells
|
||||
|
||||
#### ValidationCell
|
||||
- Factory component that renders appropriate cell type
|
||||
- Manages cell-level state
|
||||
- Handles validation errors display
|
||||
- Manages edit mode
|
||||
- Shows consistent error indicators
|
||||
|
||||
#### TemplateManager
|
||||
- Handles template selection UI
|
||||
- Provides template save/load functionality
|
||||
- Manages template application to rows
|
||||
|
||||
#### Cell Components
|
||||
- **InputCell**: Handles text input with multiline and price support
|
||||
- **MultiInputCell**: Handles multiple values with separator configuration
|
||||
- **SelectCell**: Command/popover component for single selection
|
||||
- **CheckboxCell**: Boolean value selection with mapping support
|
||||
|
||||
## State Management
|
||||
|
||||
### Core State Interface
|
||||
|
||||
```typescript
|
||||
interface ValidationState<T extends string> {
|
||||
// Core data
|
||||
data: RowData<T>[];
|
||||
filteredData: RowData<T>[];
|
||||
|
||||
// Validation state
|
||||
isValidating: boolean;
|
||||
validationErrors: Map<number, Record<string, Error[]>>;
|
||||
rowValidationStatus: Map<number, 'pending' | 'validating' | 'validated' | 'error'>;
|
||||
|
||||
// Selection state
|
||||
rowSelection: RowSelectionState;
|
||||
|
||||
// Template state
|
||||
templates: Template[];
|
||||
selectedTemplateId: string | null;
|
||||
|
||||
// Filter state
|
||||
filters: FilterState;
|
||||
|
||||
// Methods
|
||||
updateRow: (rowIndex: number, key: T, value: any) => void;
|
||||
validateRow: (rowIndex: number) => Promise<void>;
|
||||
validateUpc: (rowIndex: number, upcValue: string) => Promise<void>;
|
||||
applyTemplate: (templateId: string, rowIndexes: number[]) => void;
|
||||
saveTemplate: (name: string, type: string) => void;
|
||||
setFilters: (newFilters: Partial<FilterState>) => void;
|
||||
// Additional methods...
|
||||
}
|
||||
```
|
||||
|
||||
### useValidationState Hook
|
||||
|
||||
The main state management hook handles:
|
||||
|
||||
- Data manipulation (update, sort, filter)
|
||||
- Selection management
|
||||
- Validation coordination
|
||||
- Integration with validation utilities
|
||||
- Template management
|
||||
- Filtering and sorting
|
||||
|
||||
## Key Features Implementation
|
||||
|
||||
### 1. Field Type Handling
|
||||
|
||||
Implemented a strategy pattern for different field types:
|
||||
|
||||
```typescript
|
||||
// In ValidationCell
|
||||
const renderCellContent = () => {
|
||||
const fieldType = field.fieldType.type
|
||||
|
||||
switch (fieldType) {
|
||||
case 'input':
|
||||
return <InputCell<T> field={field} value={value} onChange={onChange} ... />
|
||||
case 'multi-input':
|
||||
return <MultiInputCell<T> field={field} value={value} onChange={onChange} ... />
|
||||
case 'select':
|
||||
return <SelectCell<T> field={field} value={value} onChange={onChange} ... />
|
||||
// etc.
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Validation Logic
|
||||
|
||||
Validation is broken down into clear steps:
|
||||
|
||||
1. **Field Validation**: Apply field-level validations (required, regex, etc.)
|
||||
2. **Row Validation**: Apply row-level validations and rowHook
|
||||
3. **Table Validation**: Apply table-level validations (unique) and tableHook
|
||||
|
||||
Validation now happens automatically without explicit buttons, with immediate feedback on field blur.
|
||||
|
||||
### 3. UI Components
|
||||
|
||||
UI components follow these principles:
|
||||
|
||||
1. **Consistent Styling**: All components use shadcn UI for consistent look and feel
|
||||
2. **Visual Feedback**: Errors are clearly indicated with icons and border styling
|
||||
3. **Intuitive Editing**: Fields show outlines even when not in focus, and edit on click
|
||||
4. **Proper Command Pattern**: Select and multi-select fields use command/popover pattern for better UX
|
||||
5. **Focus Management**: Fields close when clicking away and perform validation on blur
|
||||
|
||||
## Design Principles
|
||||
|
||||
Based on user preferences and best practices, the following design principles guide this refactoring:
|
||||
|
||||
1. **Automatic Validation**
|
||||
- Validation should happen automatically without explicit buttons
|
||||
- All validation should run on initial data load
|
||||
- Fields should validate on blur (when user clicks away)
|
||||
|
||||
2. **Modern UI Patterns**
|
||||
- Command/popover components for all selects and multi-selects
|
||||
- Consistent field outlines and borders even when not in focus
|
||||
- Badge patterns for multi-select items
|
||||
- Clear visual indicators for errors
|
||||
|
||||
3. **Reduced Complexity**
|
||||
- Remove unnecessary UI elements like "validate all" buttons
|
||||
- Eliminate redundant state and toast notifications
|
||||
- Simplify component hierarchy where possible
|
||||
- Find root causes rather than adding special cases
|
||||
|
||||
4. **Consistent Component Behavior**
|
||||
- Fields should close when clicking away
|
||||
- All inputs should follow the same editing pattern
|
||||
- Error handling should be consistent across all field types
|
||||
- Multi-select fields should allow selecting multiple items with clear visual feedback
|
||||
|
||||
## Integration Plan
|
||||
|
||||
### 1. Creating the New Component Structure
|
||||
|
||||
Folder structure has been created without modifying the existing code:
|
||||
|
||||
```bash
|
||||
mkdir -p inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/{components,hooks,utils}
|
||||
mkdir -p inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells
|
||||
```
|
||||
|
||||
### 2. Implementing Basic Components
|
||||
|
||||
Core components have been implemented:
|
||||
|
||||
1. Created index.tsx as the main entry point
|
||||
2. Implemented ValidationContainer with basic state management
|
||||
3. Created ValidationTable for data display
|
||||
4. Implemented basic cell rendering with specialized cell types
|
||||
|
||||
### 3. Implementing State Management
|
||||
|
||||
State management has been implemented:
|
||||
|
||||
1. Created useValidationState hook
|
||||
2. Implemented data transformation utilities
|
||||
3. Added validation logic
|
||||
|
||||
### 4. Integrating with Previous Step
|
||||
|
||||
The previous step component allows choosing between validation implementations, enabling gradual testing and adoption.
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
1. **Unit Tests**
|
||||
- Test individual utility functions
|
||||
- Test hooks in isolation
|
||||
- Test individual UI components
|
||||
|
||||
2. **Integration Tests**
|
||||
- Test component interactions
|
||||
- Test state management flow
|
||||
- Test validation logic integration
|
||||
|
||||
3. **Comparison Tests**
|
||||
- Compare output of new component with original
|
||||
- Verify that all functionality works the same
|
||||
|
||||
4. **Performance Tests**
|
||||
- Measure render times
|
||||
- Evaluate memory usage
|
||||
- Compare against original component
|
||||
|
||||
## Project Timeline
|
||||
|
||||
1. **Phase 1: Initial Structure (Completed)**
|
||||
- Set up folder structure
|
||||
- Implement basic components
|
||||
- Create core state management
|
||||
|
||||
2. **Phase 2: Core Functionality (In Progress)**
|
||||
- Implement validation logic (completed)
|
||||
- Create cell renderers (completed)
|
||||
- Add template management (in progress)
|
||||
|
||||
3. **Phase 3: Special Features (Upcoming)**
|
||||
- Implement UPC validation
|
||||
- Add AI validation
|
||||
- Handle special cases
|
||||
|
||||
4. **Phase 4: UI Refinement (Ongoing)**
|
||||
- Improve error display (completed)
|
||||
- Enhance user interactions (completed)
|
||||
- Optimize performance (in progress)
|
||||
|
||||
5. **Phase 5: Testing and Integration (Upcoming)**
|
||||
- Write tests
|
||||
- Fix bugs
|
||||
- Integrate with previous step
|
||||
|
||||
## Appendix: Function Reference
|
||||
|
||||
This section documents the core functions from the original ValidationStep that need to be preserved in the new implementation.
|
||||
|
||||
### Validation Functions
|
||||
|
||||
1. **validateRegex** - Validates values against regex patterns
|
||||
2. **getValidationError** - Determines field-level validation errors
|
||||
3. **validateAndCommit** - Validates and commits new values
|
||||
4. **validateData** - Validates all data rows
|
||||
5. **validateUpcAndGenerateItemNumbers** - Validates UPC codes and generates item numbers
|
||||
|
||||
### Formatting Functions
|
||||
|
||||
1. **formatPrice** - Formats price values
|
||||
2. **getDisplayValue** - Gets formatted display value based on field type
|
||||
3. **isMultiInputType** - Checks if field is multi-input type
|
||||
4. **getMultiInputSeparator** - Gets separator for multi-input fields
|
||||
5. **isPriceField** - Checks if field should be formatted as price
|
||||
|
||||
### Template Functions
|
||||
|
||||
1. **loadTemplates** - Loads templates from storage
|
||||
2. **saveTemplate** - Saves a new template
|
||||
3. **applyTemplate** - Applies a template to selected rows
|
||||
4. **getTemplateDisplayText** - Gets display text for a template
|
||||
|
||||
### AI Validation Functions
|
||||
|
||||
1. **handleAiValidation** - Triggers AI validation
|
||||
2. **showCurrentPrompt** - Shows current AI prompt
|
||||
3. **getFieldDisplayValue** - Gets display value for a field
|
||||
4. **highlightDifferences** - Highlights differences between original and corrected values
|
||||
5. **getFieldDisplayValueWithHighlight** - Gets display value with highlighted differences
|
||||
6. **revertAiChange** - Reverts an AI-suggested change
|
||||
7. **isChangeReverted** - Checks if an AI change has been reverted
|
||||
|
||||
### Event Handlers
|
||||
|
||||
1. **handleUpcValueUpdate** - Handles UPC value updates
|
||||
2. **handleBlur** - Handles input blur events
|
||||
3. **handleWheel** - Handles wheel events for navigation
|
||||
4. **copyValueDown** - Copies a value to cells below
|
||||
5. **handleSkuGeneration** - Generates SKUs
|
||||
|
||||
By following this refactoring plan, we continue to transform the monolithic ValidationStep component into a modular, maintainable set of components while preserving all existing functionality and aligning with user preferences for design and behavior.
|
||||
137
docs/ValidationStepNew-Implementation-Status.md
Normal file
137
docs/ValidationStepNew-Implementation-Status.md
Normal file
@@ -0,0 +1,137 @@
|
||||
# ValidationStepNew Implementation Status
|
||||
|
||||
## Overview
|
||||
|
||||
This document outlines the current status of the ValidationStepNew implementation, a refactored version of the original ValidationStep component. The goal is to create a more maintainable, modular component that preserves all functionality of the original while eliminating technical debt and implementing modern UI patterns.
|
||||
|
||||
## Design Principles
|
||||
|
||||
Based on the user's preferences, we're following these core design principles:
|
||||
|
||||
1. **Automatic Validation**
|
||||
- ✅ Validation runs automatically on data load
|
||||
- ✅ No explicit "validate all" button needed
|
||||
- ✅ Fields validate on blur when user clicks away
|
||||
- ✅ Immediate visual feedback for validation errors
|
||||
|
||||
2. **Modern UI Patterns**
|
||||
- ✅ Command/popover components for selects and multi-selects
|
||||
- ✅ Consistent field outlines and borders even when not in focus
|
||||
- ✅ Badge pattern for multi-select field items
|
||||
- ✅ Visual indicators for errors with appropriate styling
|
||||
|
||||
3. **Reduced Complexity**
|
||||
- ✅ Removed unnecessary UI elements like "validate all" button
|
||||
- ✅ Eliminated redundant toast notifications
|
||||
- ✅ Simplified component hierarchy
|
||||
- ✅ Fixed root causes rather than adding special cases
|
||||
|
||||
4. **Consistent Behavior**
|
||||
- ✅ Fields close when clicking away
|
||||
- ✅ All inputs follow the same editing pattern
|
||||
- ✅ Error handling is consistent across field types
|
||||
- ✅ Multi-select fields allow selecting multiple items
|
||||
|
||||
## Completed Components
|
||||
|
||||
### Core Structure
|
||||
- ✅ Main component structure
|
||||
- ✅ Directory organization
|
||||
- ✅ TypeScript interfaces
|
||||
- ✅ Props definition and passing
|
||||
|
||||
### State Management
|
||||
- ✅ `useValidationState` hook for centralized state
|
||||
- ✅ Data validation logic
|
||||
- ✅ Integration with rowHook and tableHook
|
||||
- ✅ Error tracking and management
|
||||
- ✅ Row selection
|
||||
- ✅ Automatic validation on data load
|
||||
|
||||
### UI Components
|
||||
- ✅ ValidationContainer with appropriate layout
|
||||
- ✅ ValidationTable with shadcn UI components
|
||||
- ✅ ValidationCell factory component
|
||||
- ✅ Row select/deselect functionality
|
||||
- ✅ Error display and indicators
|
||||
- ✅ Selection action bar
|
||||
|
||||
### Cell Components
|
||||
- ✅ InputCell with price and multiline support
|
||||
- ✅ MultiInputCell with separator configuration
|
||||
- ✅ SelectCell using command/popover pattern
|
||||
- ✅ CheckboxCell with boolean mapping
|
||||
- ✅ Consistent styling across all field types
|
||||
- ✅ Proper edit/view state management
|
||||
- ✅ Outlined borders in both edit and view modes
|
||||
|
||||
### Utility Functions
|
||||
- ✅ Value formatting for display
|
||||
- ✅ Field type detection
|
||||
- ✅ Error creation and management
|
||||
- ✅ Price formatting
|
||||
|
||||
### UI Improvements
|
||||
- ✅ Consistent borders and field outlines
|
||||
- ✅ Fields that properly close when clicking away
|
||||
- ✅ Multi-select with badge UI pattern
|
||||
- ✅ Command pattern for searchable select menus
|
||||
- ✅ Better visual error indication
|
||||
|
||||
## Pending Tasks
|
||||
|
||||
### Enhanced Validation
|
||||
- ⏳ AI validation system
|
||||
- ⏳ Custom validation hooks
|
||||
- ⏳ Enhanced UPC validation with API integration
|
||||
- ⏳ Validation visualizations
|
||||
|
||||
### Advanced UI Features
|
||||
- ⏳ Table virtualization for performance
|
||||
- ⏳ Drag-and-drop reordering
|
||||
- ⏳ Bulk operations (copy down, fill all, etc.)
|
||||
- ⏳ Keyboard navigation improvements
|
||||
- ⏳ Template dialogs and management UI
|
||||
|
||||
### Special Features
|
||||
- ⏳ Image preview integration
|
||||
- ⏳ SKU generation system
|
||||
- ⏳ Item number generation
|
||||
- ⏳ Dependent dropdown values
|
||||
|
||||
### Testing
|
||||
- ⏳ Unit tests for utility functions
|
||||
- ⏳ Component tests
|
||||
- ⏳ Integration tests
|
||||
- ⏳ Performance benchmarks
|
||||
|
||||
## Known Issues
|
||||
|
||||
1. TypeScript error for `validationDisabled` property in ValidationCell.tsx
|
||||
2. Some type casting is needed due to complex generic types
|
||||
3. Need to address edge cases for multi-select fields validation
|
||||
4. Proper error handling for API calls needs implementation
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. Fix TypeScript errors in ValidationCell and related components
|
||||
2. Complete template management functionality
|
||||
3. Implement UPC validation with API integration
|
||||
4. Make multi-select field validation more robust
|
||||
5. Add comprehensive tests
|
||||
|
||||
## Performance Improvements
|
||||
|
||||
We've already implemented several performance optimizations:
|
||||
|
||||
1. ✅ More efficient state updates by removing unnecessary re-renders
|
||||
2. ✅ Better error handling to prevent cascading validations
|
||||
3. ✅ Improved component isolation to prevent unnecessary re-renders
|
||||
4. ✅ Automatic validation that doesn't block the UI
|
||||
|
||||
Additional planned improvements:
|
||||
|
||||
1. Virtualized table rendering for large datasets
|
||||
2. Memoization of expensive calculations
|
||||
3. Optimized state updates to minimize re-renders
|
||||
4. Batched API calls for validation
|
||||
2846
docs/ai-validation-redesign.md
Normal file
2846
docs/ai-validation-redesign.md
Normal file
File diff suppressed because it is too large
Load Diff
185
docs/calculate-issues.md
Normal file
185
docs/calculate-issues.md
Normal file
@@ -0,0 +1,185 @@
|
||||
1. **Missing Updates for Reorder Point and Safety Stock** [RESOLVED - product-metrics.js]
|
||||
- **Problem:** In the **product_metrics** table (used by the inventory health view), the fields **reorder_point** and **safety_stock** are never updated in the product metrics calculations. Although a helper function (`calculateReorderQuantities`) exists and computes these values, the update query in the `calculateProductMetrics` function does not assign any values to these columns.
|
||||
- **Effect:** The inventory health view relies on these fields (using COALESCE to default them to 0), which means that stock might never be classified as "Reorder" or "Healthy" based on the proper reorder point or safety stock calculations.
|
||||
- **Example:** Even if a product's base metrics would require a reorder (for example, if its days of inventory are low), the view always shows a value of 0 for reorder_point and safety_stock.
|
||||
- **Fix:** Update the product metrics query (or add a subsequent update) so that **pm.reorder_point** and **pm.safety_stock** are calculated (for instance, by integrating the logic from `calculateReorderQuantities`) and stored in the table.
|
||||
|
||||
2. **Overwritten Module Exports When Combining Scripts** [RESOLVED - calculate-metrics.js]
|
||||
- **Problem:** The code provided shows two distinct exports. The main metrics calculation module exports `calculateMetrics` (along with cancel and getProgress helpers), but later in the same concatenated file the module exports are overwritten.
|
||||
- **Effect:** If these two code sections end up in a single module file, the export for the main calculation will be lost. This would break any code that calls the overall metrics calculation.
|
||||
- **Example:** An external caller expecting to run `calculateMetrics` would instead receive the `calculateProductMetrics` function.
|
||||
- **Fix:** Make sure each script resides in its own module file. Verify that the module boundaries and exports are not accidentally merged or overwritten when deployed.
|
||||
|
||||
3. **Potential Formula Issue in EOQ Calculation (Reorder Qty)** [RESOLVED - product-metrics.js]
|
||||
- **Problem:** The helper function `calculateReorderQuantities` uses an EOQ formula with a holding cost expressed as a percentage (0.25) rather than a per‐unit cost.
|
||||
- **Effect:** If the intent was to use the traditional EOQ formula (which expects a holding cost per unit rather than a percentage), this could lead to an incorrect reorder quantity.
|
||||
- **Example:** For a given annual demand and fixed order cost, the computed reorder quantity might be higher or lower than expected.
|
||||
- **Fix:** Double-check the EOQ formula. If the intention is to compute based on a percentage, then document that clearly; otherwise, adjust the formula to use the proper holding cost value.
|
||||
|
||||
4. **Potential Overlap or Redundancy in GMROI Calculation** [RESOLVED - time-aggregates.js]
|
||||
- **Problem:** In the time aggregates function, GMROI is calculated in two steps. The initial INSERT query computes GMROI as
|
||||
|
||||
`CASE WHEN s.inventory_value > 0 THEN (s.total_revenue - s.total_cost) / s.inventory_value ELSE 0 END`
|
||||
|
||||
and then a subsequent UPDATE query recalculates it as an annualized value using gross profit and active days.
|
||||
|
||||
|
||||
- **Effect:** Overwriting a computed value may be intentional to refine the metric, but if not coordinated it can cause confusion or unexpected output in the `product_time_aggregates` table.
|
||||
- **Example:** A product's GMROI might first appear as a simple ratio but then be updated to a scaled value based on the number of active days, which could lead to inconsistent reporting if not documented.
|
||||
- **Fix:** Consolidated the GMROI calculation into a single step in the initial INSERT query, properly handling annualization and NULL values.
|
||||
|
||||
5. **Handling of Products Without Orders or Purchase Data** [RESOLVED - time-aggregates.js]
|
||||
- **Problem:** In the INSERT query of the time aggregates function, the UNION covers two cases: one for products with order data (from `monthly_sales`) and one for products that have entries in `monthly_stock` but no matching order data.
|
||||
- **Effect:** If a product has neither orders nor purchase orders, it won't get an entry in `product_time_aggregates`. Depending on business rules, this might be acceptable or might mean missing data.
|
||||
- **Example:** A product that's new or rarely ordered might not appear in the time aggregates view, potentially affecting downstream calculations.
|
||||
- **Fix:** Added an `all_products` CTE and modified the JOIN structure to ensure every product gets an entry with appropriate default values, even if it has no orders or purchase orders.
|
||||
|
||||
6. **Redundant Recalculation of Vendor Metrics**
|
||||
- **Problem:** Similar concepts from prior scripts where cumulative metrics (like **total_revenue** and **total_cost**) are calculated in multiple query steps without necessary validation or optimization. In the vendor metrics script, calculations for total revenue and margin are performed within a `WITH` clause, which is then used in other parts of the process, making it more complex than needed.
|
||||
- **Effect:** There's unnecessary duplication in querying the same data multiple times across subqueries. It could result in decreased performance and may even lead to excess computation if the subqueries are not optimized or correctly indexed.
|
||||
- **Example:** Vendor sales and vendor purchase orders (PO) metrics are calculated in separate `WITH` clauses, leading to repeated calculations.
|
||||
- **Fix:** Synthesize the required metrics into fewer queries or reuse the results within the `WITH` clause itself. Avoid redundant calculations of **revenue** and **cost** unless truly necessary.
|
||||
|
||||
7. **Handling Products Without Orders or Purchase Orders**
|
||||
- **Problem:** In your `calculateVendorMetrics` script, the initial insert for vendor sales doesn't fully address the products that might not have matching orders or purchase orders. If a vendor has products without any sales within the last 12 months, the results may not be fully accurate unless handled explicitly.
|
||||
- **Effect:** If no orders exist for a product associated with a particular vendor, that product will not contribute to the vendor's metrics, potentially omitting important data when calculating **total_orders** or **total_revenue**.
|
||||
- **Example:** The scripted statistics fill gaps, but products with no recent purchase or sales orders might not be counted accurately.
|
||||
- **Fix:** Include logic to handle scenarios where these products still need to be part of the vendor calculation. Use a `LEFT JOIN` wherever possible to account for cases without sales or purchase orders.
|
||||
|
||||
8. **Redundant `ON DUPLICATE KEY UPDATE`**
|
||||
- **Problem:** Multiple queries in the `calculateVendorMetrics` script use `ON DUPLICATE KEY UPDATE` clauses to handle repeated metrics updates. This is useful for ensuring the most up-to-date calculations but can cause inconsistencies if multiple calculations happen for the same product or vendor simultaneously.
|
||||
- **Effect:** This approach can lead to an inaccurate update of brand-specific data when insertion and update overlap. Each time you add a new batch, an existing entry could be overwritten if not handled correctly.
|
||||
- **Example:** Vendor country, category, or sales-related metrics could unintentionally update during processing.
|
||||
- **Fix:** Match on current status more robustly in case of existing rows to avoid unnecessary updates. Ensure that the key used for `ON DUPLICATE KEY` aligns with any foreign key relationships that might indicate an already processed entry.
|
||||
|
||||
9. **SQL Query Performance with Multiple Nested `WITH` Clauses**
|
||||
- **Problem:** Heavily nested queries (especially **WITH** clauses) may lead to slow performance depending on the size of the dataset.
|
||||
- **Effect:** Computational burden could be high when the database is large, e.g., querying **purchase orders**, **vendor sales**, and **product info** simultaneously. Even with proper indexes, the deployment might struggle in production environments.
|
||||
- **Example:** Multiple `WITH` clauses in the vendor and brand metrics calculation scripts might work fine in small datasets but degrade performance in production.
|
||||
- **Fix:** Combine some subqueries and reduce the layer of computations needed for calculating final metrics. Test performance on a production-sized dataset to see how nested queries are handled.
|
||||
|
||||
10. **Missing Updates for Reorder Metrics (Vendor/Brand)**
|
||||
- **Previously Identified Issue:** Inconsistent updates for **reorder_point** and **safety_stock** across earlier scripts.
|
||||
- **Current Impact on This Script:** The vendor and brand metrics do not have explicit updates for reorder point or safety stock, which are essential for inventory evaluation.
|
||||
- **Effect:** The correct thresholds and reorder logic for vendor product inventory aren't fully accounted for in these scripts.
|
||||
- **Fix:** Integrate relevant logic to update **reorder_point** or **safety_stock** within the vendor and brand metrics calculations. Ensure that it's consistently computed and stored.
|
||||
|
||||
11. **Data Integrity and Consistency**
|
||||
|
||||
**w**hen tracking sales growth or performance
|
||||
|
||||
|
||||
- **Problem:** Brand metrics include a sales growth clause where negative results can sometimes be skewed severely if period data varies considerably.
|
||||
- **Effect:** If period boundaries are incorrect or records are missing, this can create drastic growth rate calculations.
|
||||
- **Example:** If the "previous" period has no sales but "current" has a substantial increase, the growth rate will show as **100%**.
|
||||
- **Fix:** Implement checks that ensure both periods are valid and that the system calculates growth accurately, avoiding growth rates based solely on potential outliers. Replace consistent gaps with a no-growth rate or a meaningful zero.
|
||||
|
||||
12. **Exclusion of Vendors With No Sales**
|
||||
|
||||
The vendor metrics query is driven by the `vendor_sales` CTE, which aggregates data only for vendors that have orders in the past 12 months.
|
||||
|
||||
|
||||
- **Impact:** Vendors that have purchase activity (or simply exist in vendor_details) but no recent sales won't show up in vendor_metrics. This could cause the frontend to miss metrics for vendors that might still be important.
|
||||
- **Fix:** Consider adding a UNION or changing the driving set so that all vendors (for example, from vendor_details) are included—even if they have zero sales.
|
||||
13. **Identical Formulas for On-Time Delivery and Order Fill Rates**
|
||||
|
||||
Both metrics are calculated as `(received_orders / total_orders) * 100`.
|
||||
|
||||
|
||||
- **Impact:** If the business expects these to be distinct (for example, one might factor in on-time receipt versus mere receipt), then showing identical values on the frontend could be misleading.
|
||||
- **Fix:** Verify and adjust the formulas if on-time delivery and order fill rates should be computed differently.
|
||||
14. **Handling Nulls and Defaults in Aggregations**
|
||||
|
||||
The query uses COALESCE in most places, but be sure that every aggregated value (like average lead time) correctly defaults when no data is present.
|
||||
|
||||
|
||||
- **Impact:** Incorrect defaults might cause odd or missing numbers on the production interface.
|
||||
- **Fix:** Double-check that all numeric aggregates reliably default to 0 where needed.
|
||||
|
||||
15. **Inconsistent Stock Filtering Conditions**
|
||||
|
||||
In the main brand metrics query the CTE filters products with the condition
|
||||
|
||||
`p.stock_quantity <= 5000 AND p.stock_quantity >= 0`
|
||||
|
||||
whereas in the brand time-based metrics query the condition is only `p.stock_quantity <= 5000`.
|
||||
|
||||
|
||||
- **Impact:** This discrepancy may lead to inconsistent numbers (for example, if any products have negative stock, which might be due to data issues) between overall brand metrics and time-based metrics on the frontend.
|
||||
- **Fix:** Standardize the filtering criteria so that both queries treat out-of-range stock values in the same way.
|
||||
16. **Growth Rate Calculation Periods**
|
||||
|
||||
The growth rate is computed by comparing revenue from the last 3 months ("current") against a period from 15–12 months ago ("previous").
|
||||
|
||||
|
||||
- **Impact:** This narrow window may not reflect typical year-over-year performance and could lead to volatile or unexpected growth percentages on the frontend.
|
||||
- **Fix:** Revisit the business logic for growth—if a longer or different comparison period is preferred, adjust the date intervals accordingly.
|
||||
17. **Potential NULLs in Aggregated Time-Based Metrics**
|
||||
|
||||
In the brand time-based metrics query, aggregate expressions such as `SUM(o.quantity * o.price)` aren't wrapped with COALESCE.
|
||||
|
||||
|
||||
- **Impact:** If there are no orders for a given brand/month, these sums might return NULL rather than 0, which could propagate into the frontend display.
|
||||
- **Fix:** Wrap such aggregates in COALESCE (e.g. `COALESCE(SUM(o.quantity * o.price), 0)`) to ensure a default numeric value.
|
||||
|
||||
18. **Grouping by Category Status in Base Metrics Insert**
|
||||
- **Problem:** The INSERT for base category metrics groups by both `c.cat_id` and `c.status` even though the table's primary key is just `category_id`.
|
||||
- **Effect:** If a category's status changes over time, the grouping may produce unexpected updates (or even multiple groups before the duplicate key update kicks in), possibly causing the wrong status or aggregated figures to be stored.
|
||||
- **Example:** A category that toggles between "active" and "inactive" might have its metrics calculated differently on different runs.
|
||||
- **Fix:** Ensure that the grouping keys match the primary key (or that the status update logic is exactly as intended) so that a single row per category is maintained.
|
||||
19. **Potential Null Handling in Margin Calculations**
|
||||
- **Problem:** In the query for category time metrics, the calculation of average margin uses expressions such as `SUM(o.quantity * (o.price - GREATEST(p.cost_price, 0)))` without using `COALESCE` on `p.cost_price`.
|
||||
- **Effect:** If any product's `cost_price` is `NULL`, then `GREATEST(p.cost_price, 0)` returns `NULL` and the resulting sum (and thus the margin) could become `NULL` rather than defaulting to 0. This might lead to missing or misleading margin figures on the frontend.
|
||||
- **Example:** A product with a missing cost price would make the entire margin expression evaluate to `NULL` even when sales exist.
|
||||
- **Fix:** Replace `GREATEST(p.cost_price, 0)` with `GREATEST(COALESCE(p.cost_price, 0), 0)` (or simply use `COALESCE(p.cost_price, 0)`) to ensure that missing values are handled.
|
||||
20. **Data Coverage in Growth Rate Calculation**
|
||||
- **Problem:** The growth rate update depends on multiple CTEs (current period, previous period, and trend analysis) that require a minimum amount of data (for instance, `HAVING COUNT(*) >= 6` in the trend_stats CTE).
|
||||
- **Effect:** Categories with insufficient historical data will fall into the "ELSE" branch (or may even be skipped if no revenue is present), which might result in a growth rate of 0.0 or an unexpected value.
|
||||
- **Example:** A newly created category that has only two months of data won't have trend analysis, so its growth rate will be calculated solely by the simple difference, which might not reflect true performance.
|
||||
- **Fix:** Confirm that this fallback behavior is acceptable for production; if not, adjust the logic so that every category receives a consistent growth rate even with sparse data.
|
||||
21. **Omission of Forecasts for Zero–Sales Categories**
|
||||
- **Observation:** The category–sales metrics query uses a `HAVING AVG(cs.daily_quantity) > 0` clause.
|
||||
- **Effect:** Categories without any average daily sales will not receive a forecast record in `category_sales_metrics`. If the frontend expects a row (even with zeros) for every category, this will lead to missing data.
|
||||
- **Fix:** Verify that it's acceptable for categories with no sales to have no forecast entry. If not, adjust the query so that a default forecast (with zeros) is inserted.
|
||||
|
||||
22. **Randomness in Category-Level Forecast Revenue Calculation**
|
||||
- **Problem:** In the category-level forecasts query, the forecast revenue is multiplied by a factor of `(0.95 + (RAND() * 0.1))`.
|
||||
- **Effect:** This introduces randomness into the forecast figures so that repeated runs could yield slightly different values. If deterministic forecasts are expected on the production frontend, this could lead to inconsistent displays.
|
||||
- **Example:** The same category might show a 5% higher forecast on one run and 3% on another because of the random multiplier.
|
||||
- **Fix:** Confirm that this randomness is intentional for your forecasting model; if forecasts are meant to be reproducible, remove or replace the `RAND()` factor with a fixed multiplier.
|
||||
23. **Multi-Statement Cleanup of Temporary Tables**
|
||||
- **Problem:** The cleanup query drops multiple temporary tables in one call (separated by semicolons).
|
||||
- **Effect:** If your Node.js MySQL driver isn't configured to allow multi-statement execution, this query may fail, leaving temporary tables behind. Leftover temporary tables might eventually cause conflicts or resource issues.
|
||||
- **Example:** Running the cleanup query could produce an error like "multi-statement queries not enabled," preventing proper cleanup.
|
||||
- **Fix:** Either configure your database connection to allow multi-statements or issue separate queries for each temporary table drop to ensure that the cleanup runs successfully.
|
||||
24. **Handling Products with No Sales Data**
|
||||
- **Problem:** In the product-level forecast calculation, the CTE `daily_stats` includes a `HAVING AVG(ds.daily_quantity) > 0` clause.
|
||||
- **Effect:** Products that have no sales (or a zero average daily quantity) will be excluded from the forecasts. This means the frontend won't show forecasts for non–selling products, which might be acceptable but could also be a completeness issue.
|
||||
- **Example:** A product that has never sold will not appear in the `sales_forecasts` table.
|
||||
- **Fix:** Confirm that it is intended for forecasts to be generated only for products with some sales activity. If forecasts are required for all products, adjust the query to insert default forecast records for products with zero sales.
|
||||
25. **Complexity of the Forecast Formula Involving the Seasonality Factor**
|
||||
- **Issue:**
|
||||
|
||||
The sales forecast calculations incorporate an adjustment factor using `COALESCE(sf.seasonality_factor, 0)` to modify forecast units and revenue. This means that if the seasonality data is missing (or not populated), the factor defaults to 0.
|
||||
|
||||
|
||||
- **Potential Problem:**
|
||||
|
||||
A default value of 0 will drastically alter the forecast calculations—often leading to a forecast of 0 or an overly dampened forecast—when in reality the intended behavior might be to use a neutral multiplier (typically 1.0). This could result in forecasts that are not reflective of the actual seasonal impact, thereby skewing the figures that reach the frontend.
|
||||
|
||||
|
||||
- **Fix:**
|
||||
|
||||
Review your data source for seasonality (the `sales_seasonality` table) and ensure it's consistently populated. Alternatively, if missing seasonality data is possible, consider using a more neutral default (such as 1.0) in your COALESCE. This change would prevent the forecast formulas from over-simplifying (or even nullifying) the forecast output due to missing seasonality factors.
|
||||
|
||||
|
||||
26. **Group By with Seasonality Factor Variability**
|
||||
- **Observation:** In the forecast insertion query, the GROUP BY clause includes `sf.seasonality_factor` along with other fields.
|
||||
- **Effect:** If the seasonality factor differs (or is `NULL` versus a value) for different forecast dates, this might result in multiple rows for the same product and forecast date. However, the `ON DUPLICATE KEY UPDATE` clause will merge them—but only if the primary key (pid, forecast_date) is truly unique.
|
||||
- **Fix:** Verify that the grouping produces exactly one row per product per forecast date. If there's potential for multiple rows due to seasonality variability, consider applying a COALESCE or an aggregation on the seasonality factor so that it does not affect grouping.
|
||||
|
||||
27. **Memory Management for Temporary Tables** [RESOLVED - calculate-metrics.js]
|
||||
- **Problem:** In metrics calculations, temporary tables aren't always properly cleaned up if the process fails between creation and the DROP statement.
|
||||
- **Effect:** If a process fails after creating temporary tables but before dropping them, these tables remain in memory until the connection is closed. In a production environment with multiple calculation runs, this could lead to memory leaks or table name conflicts.
|
||||
- **Example:** The `temp_revenue_ranks` table creation in ABC classification could remain if the process fails before reaching the DROP statement.
|
||||
- **Fix:** Implement proper cleanup in a finally block or use transaction management that ensures temporary tables are always cleaned up, even in failure scenarios.
|
||||
72
docs/fix-multi-select.md
Normal file
72
docs/fix-multi-select.md
Normal file
@@ -0,0 +1,72 @@
|
||||
# Solution: Keeping Dropdowns Open During Multiple Selections
|
||||
|
||||
## The Problem
|
||||
|
||||
When implementing a multi-select dropdown in React, a common issue occurs:
|
||||
|
||||
1. You select an item in the dropdown
|
||||
2. The `onChange` handler is called, updating the data
|
||||
3. This triggers a re-render of the parent component (in this case, the entire table)
|
||||
4. During the re-render, the dropdown is unmounted and remounted
|
||||
5. This causes the dropdown to close before you can make multiple selections
|
||||
|
||||
## The Solution: Deferred State Updates
|
||||
|
||||
The key insight is to **separate local state management from parent state updates**:
|
||||
|
||||
```typescript
|
||||
// Step 1: Add local state to track selections
|
||||
const [internalValue, setInternalValue] = useState<string[]>(value)
|
||||
|
||||
// Step 2: Handle popover open state changes
|
||||
const handleOpenChange = useCallback((newOpen: boolean) => {
|
||||
if (open && !newOpen) {
|
||||
// Only update parent state when dropdown closes
|
||||
if (JSON.stringify(internalValue) !== JSON.stringify(value)) {
|
||||
onChange(internalValue);
|
||||
}
|
||||
}
|
||||
|
||||
setOpen(newOpen);
|
||||
|
||||
if (newOpen) {
|
||||
// Sync internal state with external state when opening
|
||||
setInternalValue(value);
|
||||
}
|
||||
}, [open, internalValue, value, onChange]);
|
||||
|
||||
// Step 3: Toggle selection only updates internal state
|
||||
const toggleSelection = useCallback((selectedValue: string) => {
|
||||
setInternalValue(prev => {
|
||||
if (prev.includes(selectedValue)) {
|
||||
return prev.filter(v => v !== selectedValue);
|
||||
} else {
|
||||
return [...prev, selectedValue];
|
||||
}
|
||||
});
|
||||
}, []);
|
||||
```
|
||||
|
||||
## Why This Works
|
||||
|
||||
1. **No parent re-renders during selection**: Since we're only updating local state, the parent component doesn't re-render during selection.
|
||||
2. **Consistent UI**: The dropdown shows accurate selected states using the internal value.
|
||||
3. **Data integrity**: The final selections are properly synchronized back to the parent when done.
|
||||
4. **Resilient to external changes**: Initial state is synchronized when opening the dropdown.
|
||||
|
||||
## Implementation Steps
|
||||
|
||||
1. Create a local state variable to track selections inside the component
|
||||
2. Only make selections against this local state while the dropdown is open
|
||||
3. Defer updating the parent until the dropdown is explicitly closed
|
||||
4. When opening, synchronize the internal state with the external value
|
||||
|
||||
## Benefits
|
||||
|
||||
This pattern:
|
||||
- Avoids re-render cycles that would unmount the dropdown
|
||||
- Maintains UI consistency during multi-selection
|
||||
- Simplifies the component's interaction with parent components
|
||||
- Works with existing component lifecycles rather than fighting against them
|
||||
|
||||
This solution is much simpler than trying to prevent event propagation or manipulating DOM events, and addresses the root cause of the issue: premature re-rendering.
|
||||
342
docs/import-from-prod-data-mapping.md
Normal file
342
docs/import-from-prod-data-mapping.md
Normal file
@@ -0,0 +1,342 @@
|
||||
# MySQL to PostgreSQL Import Process Documentation
|
||||
|
||||
This document outlines the data import process from the production MySQL database to the local PostgreSQL database, focusing on column mappings, data transformations, and the overall import architecture.
|
||||
|
||||
## Table of Contents
|
||||
1. [Overview](#overview)
|
||||
2. [Import Architecture](#import-architecture)
|
||||
3. [Column Mappings](#column-mappings)
|
||||
- [Categories](#categories)
|
||||
- [Products](#products)
|
||||
- [Product Categories (Relationship)](#product-categories-relationship)
|
||||
- [Orders](#orders)
|
||||
- [Purchase Orders](#purchase-orders)
|
||||
- [Metadata Tables](#metadata-tables)
|
||||
4. [Special Calculations](#special-calculations)
|
||||
5. [Implementation Notes](#implementation-notes)
|
||||
|
||||
## Overview
|
||||
|
||||
The import process extracts data from a MySQL 5.7 production database and imports it into a PostgreSQL database. It can operate in two modes:
|
||||
|
||||
- **Full Import**: Imports all data regardless of last sync time
|
||||
- **Incremental Import**: Only imports data that has changed since the last import
|
||||
|
||||
The process handles four main data types:
|
||||
- Categories (product categorization hierarchy)
|
||||
- Products (inventory items)
|
||||
- Orders (sales records)
|
||||
- Purchase Orders (vendor orders)
|
||||
|
||||
## Import Architecture
|
||||
|
||||
The import process follows these steps:
|
||||
|
||||
1. **Establish Connection**: Creates a SSH tunnel to the production server and establishes database connections
|
||||
2. **Setup Import History**: Creates a record of the current import operation
|
||||
3. **Import Categories**: Processes product categories in hierarchical order
|
||||
4. **Import Products**: Processes products with their attributes and category relationships
|
||||
5. **Import Orders**: Processes customer orders with line items, taxes, and discounts
|
||||
6. **Import Purchase Orders**: Processes vendor purchase orders with line items
|
||||
7. **Record Results**: Updates the import history with results
|
||||
8. **Close Connections**: Cleans up connections and resources
|
||||
|
||||
Each import step uses temporary tables for processing and wraps operations in transactions to ensure data consistency.
|
||||
|
||||
## Column Mappings
|
||||
|
||||
### Categories
|
||||
| PostgreSQL Column | MySQL Source | Transformation |
|
||||
|-------------------|---------------------------------|----------------------------------------------|
|
||||
| cat_id | product_categories.cat_id | Direct mapping |
|
||||
| name | product_categories.name | Direct mapping |
|
||||
| type | product_categories.type | Direct mapping |
|
||||
| parent_id | product_categories.master_cat_id| NULL for top-level categories (types 10, 20) |
|
||||
| description | product_categories.combined_name| Direct mapping |
|
||||
| status | N/A | Hard-coded 'active' |
|
||||
| created_at | N/A | Current timestamp |
|
||||
| updated_at | N/A | Current timestamp |
|
||||
|
||||
**Notes:**
|
||||
- Categories are processed in hierarchical order by type: [10, 20, 11, 21, 12, 13]
|
||||
- Type 10/20 are top-level categories with no parent
|
||||
- Types 11/21/12/13 are child categories that reference parent categories
|
||||
|
||||
### Products
|
||||
| PostgreSQL Column | MySQL Source | Transformation |
|
||||
|----------------------|----------------------------------|---------------------------------------------------------------|
|
||||
| pid | products.pid | Direct mapping |
|
||||
| title | products.description | Direct mapping |
|
||||
| description | products.notes | Direct mapping |
|
||||
| sku | products.itemnumber | Fallback to 'NO-SKU' if empty |
|
||||
| stock_quantity | shop_inventory.available_local | Capped at 5000, minimum 0 |
|
||||
| preorder_count | current_inventory.onpreorder | Default 0 |
|
||||
| notions_inv_count | product_notions_b2b.inventory | Default 0 |
|
||||
| price | product_current_prices.price_each| Default 0, filtered on active=1 |
|
||||
| regular_price | products.sellingprice | Default 0 |
|
||||
| cost_price | product_inventory | Weighted average: SUM(costeach * count) / SUM(count) when count > 0, or latest costeach |
|
||||
| vendor | suppliers.companyname | Via supplier_item_data.supplier_id |
|
||||
| vendor_reference | supplier_item_data | supplier_itemnumber or notions_itemnumber based on vendor |
|
||||
| notions_reference | supplier_item_data.notions_itemnumber | Direct mapping |
|
||||
| brand | product_categories.name | Linked via products.company |
|
||||
| line | product_categories.name | Linked via products.line |
|
||||
| subline | product_categories.name | Linked via products.subline |
|
||||
| artist | product_categories.name | Linked via products.artist |
|
||||
| categories | product_category_index | Comma-separated list of category IDs |
|
||||
| created_at | products.date_created | Validated date, NULL if invalid |
|
||||
| first_received | products.datein | Validated date, NULL if invalid |
|
||||
| landing_cost_price | NULL | Not set |
|
||||
| barcode | products.upc | Direct mapping |
|
||||
| harmonized_tariff_code| products.harmonized_tariff_code | Direct mapping |
|
||||
| updated_at | products.stamp | Validated date, NULL if invalid |
|
||||
| visible | shop_inventory | Calculated from show + buyable > 0 |
|
||||
| managing_stock | N/A | Hard-coded true |
|
||||
| replenishable | Multiple fields | Complex calculation based on reorder, dates, etc. |
|
||||
| permalink | N/A | Constructed URL with product ID |
|
||||
| moq | supplier_item_data | notions_qty_per_unit or supplier_qty_per_unit, minimum 1 |
|
||||
| uom | N/A | Hard-coded 1 |
|
||||
| rating | products.rating | Direct mapping |
|
||||
| reviews | products.rating_votes | Direct mapping |
|
||||
| weight | products.weight | Direct mapping |
|
||||
| length | products.length | Direct mapping |
|
||||
| width | products.width | Direct mapping |
|
||||
| height | products.height | Direct mapping |
|
||||
| country_of_origin | products.country_of_origin | Direct mapping |
|
||||
| location | products.location | Direct mapping |
|
||||
| total_sold | order_items | SUM(qty_ordered) for all order_items where prod_pid = pid |
|
||||
| baskets | mybasket | COUNT of records where mb.item = pid and qty > 0 |
|
||||
| notifies | product_notify | COUNT of records where pn.pid = pid |
|
||||
| date_last_sold | product_last_sold.date_sold | Validated date, NULL if invalid |
|
||||
| image | N/A | Constructed from pid and image URL pattern |
|
||||
| image_175 | N/A | Constructed from pid and image URL pattern |
|
||||
| image_full | N/A | Constructed from pid and image URL pattern |
|
||||
| options | NULL | Not set |
|
||||
| tags | NULL | Not set |
|
||||
|
||||
**Notes:**
|
||||
- Replenishable calculation:
|
||||
```javascript
|
||||
CASE
|
||||
WHEN p.reorder < 0 THEN 0
|
||||
WHEN (
|
||||
(COALESCE(pls.date_sold, '0000-00-00') = '0000-00-00' OR pls.date_sold <= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR))
|
||||
AND (p.datein = '0000-00-00 00:00:00' OR p.datein <= DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 5 YEAR))
|
||||
AND (p.date_refill = '0000-00-00 00:00:00' OR p.date_refill <= DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 5 YEAR))
|
||||
) THEN 0
|
||||
ELSE 1
|
||||
END
|
||||
```
|
||||
|
||||
In business terms, a product is considered NOT replenishable only if:
|
||||
- It was manually flagged as not replenishable (negative reorder value)
|
||||
- OR it shows no activity across ALL metrics (no sales AND no receipts AND no refills in the past 5 years)
|
||||
- Image URLs are constructed using this pattern:
|
||||
```javascript
|
||||
const paddedPid = pid.toString().padStart(6, '0');
|
||||
const prefix = paddedPid.slice(0, 3);
|
||||
const basePath = `${imageUrlBase}${prefix}/${pid}`;
|
||||
return {
|
||||
image: `${basePath}-t-${iid}.jpg`,
|
||||
image_175: `${basePath}-175x175-${iid}.jpg`,
|
||||
image_full: `${basePath}-o-${iid}.jpg`
|
||||
};
|
||||
```
|
||||
|
||||
### Product Categories (Relationship)
|
||||
|
||||
| PostgreSQL Column | MySQL Source | Transformation |
|
||||
|-------------------|-----------------------------------|---------------------------------------------------------------|
|
||||
| pid | products.pid | Direct mapping |
|
||||
| cat_id | product_category_index.cat_id | Direct mapping, filtered by category types |
|
||||
|
||||
**Notes:**
|
||||
- Only categories of types 10, 20, 11, 21, 12, 13 are imported
|
||||
- Categories 16 and 17 are explicitly excluded
|
||||
|
||||
### Orders
|
||||
|
||||
| PostgreSQL Column | MySQL Source | Transformation |
|
||||
|-------------------|-----------------------------------|---------------------------------------------------------------|
|
||||
| order_number | order_items.order_id | Direct mapping |
|
||||
| pid | order_items.prod_pid | Direct mapping |
|
||||
| sku | order_items.prod_itemnumber | Fallback to 'NO-SKU' if empty |
|
||||
| date | _order.date_placed_onlydate | Via join to _order table |
|
||||
| price | order_items.prod_price | Direct mapping |
|
||||
| quantity | order_items.qty_ordered | Direct mapping |
|
||||
| discount | Multiple sources | Complex calculation (see notes) |
|
||||
| tax | order_tax_info_products.item_taxes_to_collect | Via latest order_tax_info record |
|
||||
| tax_included | N/A | Hard-coded false |
|
||||
| shipping | N/A | Hard-coded 0 |
|
||||
| customer | _order.order_cid | Direct mapping |
|
||||
| customer_name | users | CONCAT(users.firstname, ' ', users.lastname) |
|
||||
| status | _order.order_status | Direct mapping |
|
||||
| canceled | _order.date_cancelled | Boolean: true if date_cancelled is not '0000-00-00 00:00:00' |
|
||||
| costeach | order_costs | From latest record or fallback to price * 0.5 |
|
||||
|
||||
**Notes:**
|
||||
- Only orders with order_status >= 15 and with a valid date_placed are processed
|
||||
- For incremental imports, only orders modified since last sync are processed
|
||||
- Discount calculation combines three sources:
|
||||
1. Base discount: order_items.prod_price_reg - order_items.prod_price
|
||||
2. Promo discount: SUM of order_discount_items.amount
|
||||
3. Proportional order discount: Calculation based on order subtotal proportion
|
||||
```javascript
|
||||
(oi.base_discount +
|
||||
COALESCE(ot.promo_discount, 0) +
|
||||
CASE
|
||||
WHEN om.summary_discount > 0 AND om.summary_subtotal > 0 THEN
|
||||
ROUND((om.summary_discount * (oi.price * oi.quantity)) / NULLIF(om.summary_subtotal, 0), 2)
|
||||
ELSE 0
|
||||
END)::DECIMAL(10,2)
|
||||
```
|
||||
- Taxes are taken from the latest tax record for an order
|
||||
- Cost data is taken from the latest non-pending cost record
|
||||
|
||||
### Purchase Orders
|
||||
|
||||
| PostgreSQL Column | MySQL Source | Transformation |
|
||||
|-------------------|-----------------------------------|---------------------------------------------------------------|
|
||||
| po_id | po.po_id | Default 0 if NULL |
|
||||
| pid | po_products.pid | Direct mapping |
|
||||
| sku | products.itemnumber | Fallback to 'NO-SKU' if empty |
|
||||
| name | products.description | Fallback to 'Unknown Product' |
|
||||
| cost_price | po_products.cost_each | Direct mapping |
|
||||
| po_cost_price | po_products.cost_each | Duplicate of cost_price |
|
||||
| vendor | suppliers.companyname | Fallback to 'Unknown Vendor' if empty |
|
||||
| date | po.date_ordered | Fallback to po.date_created if NULL |
|
||||
| expected_date | po.date_estin | Direct mapping |
|
||||
| status | po.status | Default 1 if NULL |
|
||||
| notes | po.short_note | Fallback to po.notes if NULL |
|
||||
| ordered | po_products.qty_each | Direct mapping |
|
||||
| received | N/A | Hard-coded 0 |
|
||||
| receiving_status | N/A | Hard-coded 1 |
|
||||
|
||||
**Notes:**
|
||||
- Only POs created within last 1 year (incremental) or 5 years (full) are processed
|
||||
- For incremental imports, only POs modified since last sync are processed
|
||||
|
||||
### Metadata Tables
|
||||
|
||||
#### import_history
|
||||
|
||||
| PostgreSQL Column | Source | Notes |
|
||||
|-------------------|-----------------------------------|---------------------------------------------------------------|
|
||||
| id | Auto-increment | Primary key |
|
||||
| table_name | Code | 'all_tables' for overall import |
|
||||
| start_time | NOW() | Import start time |
|
||||
| end_time | NOW() | Import completion time |
|
||||
| duration_seconds | Calculation | Elapsed seconds |
|
||||
| is_incremental | INCREMENTAL_UPDATE | Flag from config |
|
||||
| records_added | Calculation | Sum from all imports |
|
||||
| records_updated | Calculation | Sum from all imports |
|
||||
| status | Code | 'running', 'completed', 'failed', or 'cancelled' |
|
||||
| error_message | Exception | Error message if failed |
|
||||
| additional_info | JSON | Configuration and results |
|
||||
|
||||
#### sync_status
|
||||
|
||||
| PostgreSQL Column | Source | Notes |
|
||||
|----------------------|--------------------------------|---------------------------------------------------------------|
|
||||
| table_name | Code | Name of imported table |
|
||||
| last_sync_timestamp | NOW() | Timestamp of successful sync |
|
||||
| last_sync_id | NULL | Not used currently |
|
||||
|
||||
## Special Calculations
|
||||
|
||||
### Date Validation
|
||||
|
||||
MySQL dates are validated before insertion into PostgreSQL:
|
||||
|
||||
```javascript
|
||||
function validateDate(mysqlDate) {
|
||||
if (!mysqlDate || mysqlDate === '0000-00-00' || mysqlDate === '0000-00-00 00:00:00') {
|
||||
return null;
|
||||
}
|
||||
// Check if the date is valid
|
||||
const date = new Date(mysqlDate);
|
||||
return isNaN(date.getTime()) ? null : mysqlDate;
|
||||
}
|
||||
```
|
||||
|
||||
### Retry Mechanism
|
||||
|
||||
Operations that might fail temporarily are retried with exponential backoff:
|
||||
|
||||
```javascript
|
||||
async function withRetry(operation, errorMessage) {
|
||||
let lastError;
|
||||
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
console.error(`${errorMessage} (Attempt ${attempt}/${MAX_RETRIES}):`, error);
|
||||
if (attempt < MAX_RETRIES) {
|
||||
const backoffTime = RETRY_DELAY * Math.pow(2, attempt - 1);
|
||||
await new Promise(resolve => setTimeout(resolve, backoffTime));
|
||||
}
|
||||
}
|
||||
}
|
||||
throw lastError;
|
||||
}
|
||||
```
|
||||
|
||||
### Progress Tracking
|
||||
|
||||
Progress is tracked with estimated time remaining:
|
||||
|
||||
```javascript
|
||||
function estimateRemaining(startTime, current, total) {
|
||||
if (current === 0) return "Calculating...";
|
||||
const elapsedSeconds = (Date.now() - startTime) / 1000;
|
||||
const itemsPerSecond = current / elapsedSeconds;
|
||||
const remainingItems = total - current;
|
||||
const remainingSeconds = remainingItems / itemsPerSecond;
|
||||
return formatElapsedTime(remainingSeconds);
|
||||
}
|
||||
```
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
### Transaction Management
|
||||
|
||||
All imports use transactions to ensure data consistency:
|
||||
|
||||
- **Categories**: Uses savepoints for each category type
|
||||
- **Products**: Uses a single transaction for the entire import
|
||||
- **Orders**: Uses a single transaction with temporary tables
|
||||
- **Purchase Orders**: Uses a single transaction with temporary tables
|
||||
|
||||
### Memory Usage Optimization
|
||||
|
||||
To minimize memory usage when processing large datasets:
|
||||
|
||||
1. Data is processed in batches (100-5000 records per batch)
|
||||
2. Temporary tables are used for intermediate data
|
||||
3. Some queries use cursors to avoid loading all results at once
|
||||
|
||||
### MySQL vs PostgreSQL Compatibility
|
||||
|
||||
The scripts handle differences between MySQL and PostgreSQL:
|
||||
|
||||
1. MySQL-specific syntax like `USE INDEX` is removed for PostgreSQL
|
||||
2. `GROUP_CONCAT` in MySQL becomes string operations in PostgreSQL
|
||||
3. Transaction syntax differences are abstracted in the connection wrapper
|
||||
4. PostgreSQL's `ON CONFLICT` replaces MySQL's `ON DUPLICATE KEY UPDATE`
|
||||
|
||||
### SSH Tunnel
|
||||
|
||||
Database connections go through an SSH tunnel for security:
|
||||
|
||||
```javascript
|
||||
ssh.forwardOut(
|
||||
"127.0.0.1",
|
||||
0,
|
||||
sshConfig.prodDbConfig.host,
|
||||
sshConfig.prodDbConfig.port,
|
||||
async (err, stream) => {
|
||||
if (err) reject(err);
|
||||
resolve({ ssh, stream });
|
||||
}
|
||||
);
|
||||
```
|
||||
1380
docs/inventory-calculation-reference.md
Normal file
1380
docs/inventory-calculation-reference.md
Normal file
File diff suppressed because it is too large
Load Diff
73544
docs/klaviyoopenapi.json
Normal file
73544
docs/klaviyoopenapi.json
Normal file
File diff suppressed because it is too large
Load Diff
1065
docs/metrics-calculation-system.md
Normal file
1065
docs/metrics-calculation-system.md
Normal file
File diff suppressed because it is too large
Load Diff
1106
docs/prod_registry.class.php
Normal file
1106
docs/prod_registry.class.php
Normal file
File diff suppressed because it is too large
Load Diff
271
docs/routes-cleanup.md
Normal file
271
docs/routes-cleanup.md
Normal file
@@ -0,0 +1,271 @@
|
||||
**Analysis of Potential Issues**
|
||||
|
||||
1. **Obsolete Functionality:**
|
||||
* **`config.js` Legacy Endpoints:** The endpoints `GET /config/`, `PUT /config/stock-thresholds/:id`, `PUT /config/lead-time-thresholds/:id`, `PUT /config/sales-velocity/:id`, `PUT /config/abc-classification/:id`, `PUT /config/safety-stock/:id`, and `PUT /config/turnover/:id` appear **highly likely to be obsolete**. They reference older, single-row config tables (`stock_thresholds`, etc.) while newer endpoints (`/config/global`, `/config/products`, `/config/vendors`) manage settings in more structured tables (`settings_global`, `settings_product`, `settings_vendor`). Unless specifically required for backward compatibility, these legacy endpoints should be removed to avoid confusion and potential data conflicts.
|
||||
* **`analytics.js` Forecast Endpoint (`GET /analytics/forecast`):** This endpoint uses **MySQL syntax** (`DATEDIFF`, `DATE_FORMAT`, `JSON_OBJECT`, `?` placeholders) but seems intended to run within the analytics module which otherwise uses PostgreSQL (`req.app.locals.pool`, `date_trunc`, `::text`, `$1` placeholders). This endpoint is likely **obsolete or misplaced** and will not function correctly against the PostgreSQL database.
|
||||
* **`csv.js` Redundant Actions:**
|
||||
* `POST /csv/update` seems redundant with `POST /csv/full-update`. The latter uses the `runScript` helper and dedicated state (`activeFullUpdate`), appearing more robust. `/csv/update` might be older or incomplete.
|
||||
* `POST /csv/reset` seems redundant with `POST /csv/full-reset`. Similar reasoning applies; `/csv/full-reset` appears preferred.
|
||||
* **`products.js` Import Endpoint (`POST /products/import`):** This is **dangerous duplication**. The `/csv` module handles imports (`/csv/import`, `/csv/import-from-prod`) with locking (`activeImport`) to prevent concurrent operations. This endpoint lacks such locking and could corrupt data if run simultaneously with other CSV/reset operations. It should likely be removed.
|
||||
* **`products.js` Metrics Endpoint (`GET /products/:id/metrics`):** This is redundant. The `/metrics/:pid` endpoint provides the same, possibly more comprehensive, data directly from the `product_metrics` table. Clients should use `/metrics/:pid` instead.
|
||||
|
||||
2. **Overlap or Inappropriate Duplication of Effort:**
|
||||
* **AI Prompt Getters:** `GET /ai-prompts/type/general` and `GET /ai-prompts/type/system` could potentially be handled by adding a query parameter filter to `GET /ai-prompts/` (e.g., `GET /ai-prompts?prompt_type=general`). However, dedicated endpoints for single, specific items can sometimes be simpler. This is more of a design choice than a major issue.
|
||||
* **Vendor Performance/Metrics:** There are multiple ways to get vendor performance data:
|
||||
* `GET /analytics/vendors` (uses `vendor_metrics`)
|
||||
* `GET /dashboard/vendor/performance` (uses `purchase_orders`)
|
||||
* `GET /purchase-orders/vendor-metrics` (uses `purchase_orders`)
|
||||
* `GET /vendors-aggregate/` (uses `vendor_metrics`, augmented with `purchase_orders`)
|
||||
This suggests significant overlap. The `/vendors-aggregate` endpoint seems the most comprehensive, combining pre-aggregated data with some real-time info. The others, especially `/dashboard/vendor/performance` and `/purchase-orders/vendor-metrics` which calculate directly from `purchase_orders`, might be redundant or less performant.
|
||||
* **Product Listing:**
|
||||
* `GET /products/` lists products joining `products`, `product_metrics`, and `categories`.
|
||||
* `GET /metrics/` lists products primarily from `product_metrics`.
|
||||
They offer similar filtering/sorting. If `product_metrics` contains all necessary display fields, `GET /products/` might be partly redundant for simple listing views, although it does provide aggregated category names. Evaluate if both full list endpoints are necessary.
|
||||
* **Image Uploads/Management:** Image handling is split:
|
||||
* `products-import.js`: Uploads temporary images for product import to `/uploads/products/`, schedules deletion.
|
||||
* `reusable-images.js`: Uploads persistent images to `/uploads/reusable/`, stores metadata in DB.
|
||||
* `products-import.js` has `/check-file` and `/list-uploads` that can see *both* directories, while `reusable-images.js` has a `/check-file` that only sees its own. This separation could be confusing. Clarify the purpose and lifecycle of images in each directory.
|
||||
* **Background Task Management (`csv.js`):** The use of `activeImport` for multiple unrelated tasks (import, reset, metrics calc) prevents concurrency, which might be too restrictive. The cancellation logic (`/cancel`) only targets `full-update`/`full-reset`, not tasks locked by `activeImport`. This needs unification.
|
||||
* **Analytics/Dashboard Base Table Queries:** Several endpoints in `analytics.js` (`/pricing`, `/categories`) and `dashboard.js` (`/best-sellers`, `/sales/metrics`, `/trending/products`, `/key-metrics`, `/inventory-health`, `/sales-overview`) query base tables (`orders`, `products`, `purchase_orders`) directly, while many others leverage pre-aggregated `_metrics` tables. This inconsistency can lead to performance differences and suggests potential for optimization by using aggregates where possible.
|
||||
|
||||
3. **Obvious Mistakes / Data Issues:**
|
||||
* **AI Prompt Fetching:** `GET /ai-prompts/company/:companyId`, `/type/general`, `/type/system` return `result.rows[0]`. This assumes uniqueness. If the underlying DB constraints (`unique_company_prompt`, etc.) fail or aren't present, this could silently hide data if multiple rows match. The use of unique constraint handling in POST/PUT suggests this is likely intended and safe *if* DB constraints are solid.
|
||||
* **Mixed Databases & SSH Tunnels:** The heavy reliance in `ai_validation.js` and `products-import.js` on connecting to a production MySQL DB via SSH tunnel while also using a local PostgreSQL DB adds significant architectural complexity.
|
||||
* **Inefficiency:** In `ai_validation.js` (`generateDebugResponse`), an SSH tunnel and MySQL connection (`promptTunnel`, `promptConnection`) are established but seem unused when fetching prompts (which correctly come from the PG pool `res.app.locals.pool`). This is wasted effort.
|
||||
* **Improvement:** The `getDbConnection` function in `products-import.js` implements caching/pooling for the SSH/MySQL connection – this is much better and should ideally be used consistently wherever the production DB is accessed (e.g., in `ai_validation.js`).
|
||||
* **`products.js` Brand Filtering:** `GET /products/brands` filters brands based on having associated purchase orders with a cost >= 500. This seems arbitrary for a general list of brands and might return incomplete results depending on the use case.
|
||||
* **Type Handling:** Ensure `parseValue` handles all required types and edge cases correctly, especially for filtering complex queries in `*-aggregate` and `metrics` routes. Explicit type casting in SQL (`::numeric`, `::text`, etc.) is generally good practice in PostgreSQL.
|
||||
* **Dummy Data:** Several `dashboard.js` endpoints return hardcoded dummy data on errors or when no data is found. While this prevents UI crashes, it can mask real issues. Ensure logging is robust when fallbacks are used.
|
||||
|
||||
**Summary of Endpoints**
|
||||
|
||||
Here's a summary of the available endpoints, grouped by their likely file/module:
|
||||
|
||||
**1. AI Prompts (`ai_prompts.js`)**
|
||||
* `GET /`: Get all AI prompts.
|
||||
* `GET /:id`: Get a specific AI prompt by its ID.
|
||||
* `GET /company/:companyId`: Get the AI prompt for a specific company (expects one). **(Deprecated)**
|
||||
* `GET /type/general`: Get the general AI prompt (expects one). **(Deprecated)**
|
||||
* `GET /type/system`: Get the system AI prompt (expects one). **(Deprecated)**
|
||||
* `GET /by-type`: Get AI prompt by type (general, system, company_specific) with optional company parameter. **(New Consolidated Endpoint)**
|
||||
* `POST /`: Create a new AI prompt.
|
||||
* `PUT /:id`: Update an existing AI prompt.
|
||||
* `DELETE /:id`: Delete an AI prompt.
|
||||
|
||||
**2. AI Validation (`ai_validation.js`)**
|
||||
* `POST /debug`: Generate and view the structure of prompts and taxonomy data (for debugging, doesn't call OpenAI). Connects to Prod MySQL (taxonomy) and Local PG (prompts, performance).
|
||||
* `POST /validate`: Validate product data using OpenAI. Connects to Prod MySQL (taxonomy) and Local PG (prompts, performance).
|
||||
* `GET /test-taxonomy`: Test endpoint to query sample taxonomy data from Prod MySQL.
|
||||
|
||||
**3. Analytics (`analytics.js`)**
|
||||
* `GET /stats`: Get overall business statistics from metrics tables.
|
||||
* `GET /profit`: Get profit analysis data (by category, over time, top products) from metrics tables.
|
||||
* `GET /vendors`: Get vendor performance analysis from `vendor_metrics`.
|
||||
* `GET /stock`: Get stock analysis data (turnover, levels, critical items) from metrics tables.
|
||||
* `GET /pricing`: Get pricing analysis (price points, elasticity, recommendations) - **uses `orders` table**.
|
||||
* `GET /categories`: Get category performance analysis (revenue, profit, growth, distribution, trends) - **uses `orders` and `products` tables**.
|
||||
* `GET /forecast`: (**Likely Obsolete/Broken**) Attempts to get forecast data using MySQL syntax.
|
||||
|
||||
**4. Brands Aggregate (`brands-aggregate.js`)**
|
||||
* `GET /filter-options`: Get distinct brand names and statuses for UI filters (from `brand_metrics`).
|
||||
* `GET /stats`: Get overall statistics related to brands (from `brand_metrics`).
|
||||
* `GET /`: List brands with aggregated metrics, supporting filtering, sorting, pagination (from `brand_metrics`).
|
||||
|
||||
**5. Categories Aggregate (`categories-aggregate.js`)**
|
||||
* `GET /filter-options`: Get distinct category types, statuses, and counts for UI filters (from `category_metrics` & `categories`).
|
||||
* `GET /stats`: Get overall statistics related to categories (from `category_metrics` & `categories`).
|
||||
* `GET /`: List categories with aggregated metrics, supporting filtering, sorting (incl. hierarchy), pagination (from `category_metrics` & `categories`).
|
||||
|
||||
**6. Configuration (`config.js`)**
|
||||
* **(New)** `GET /global`: Get all global settings.
|
||||
* **(New)** `PUT /global`: Update global settings.
|
||||
* **(New)** `GET /products`: List product-specific settings with pagination/search.
|
||||
* **(New)** `PUT /products/:pid`: Update/Create product-specific settings.
|
||||
* **(New)** `POST /products/:pid/reset`: Reset product settings to defaults.
|
||||
* **(New)** `GET /vendors`: List vendor-specific settings with pagination/search.
|
||||
* **(New)** `PUT /vendors/:vendor`: Update/Create vendor-specific settings.
|
||||
* **(New)** `POST /vendors/:vendor/reset`: Reset vendor settings to defaults.
|
||||
* **(Legacy/Obsolete)** `GET /`: Get all config from old single-row tables.
|
||||
* **(Legacy/Obsolete)** `PUT /stock-thresholds/:id`: Update old stock thresholds.
|
||||
* **(Legacy/Obsolete)** `PUT /lead-time-thresholds/:id`: Update old lead time thresholds.
|
||||
* **(Legacy/Obsolete)** `PUT /sales-velocity/:id`: Update old sales velocity config.
|
||||
* **(Legacy/Obsolete)** `PUT /abc-classification/:id`: Update old ABC config.
|
||||
* **(Legacy/Obsolete)** `PUT /safety-stock/:id`: Update old safety stock config.
|
||||
* **(Legacy/Obsolete)** `PUT /turnover/:id`: Update old turnover config.
|
||||
|
||||
**7. CSV Operations & Background Tasks (`csv.js`)**
|
||||
* `GET /:type/progress`: SSE endpoint for full update/reset progress.
|
||||
* `GET /test`: Simple test endpoint.
|
||||
* `GET /status`: Check status of the generic background task lock (`activeImport`).
|
||||
* `GET /calculate-metrics/status`: Check status of metrics calculation.
|
||||
* `GET /history/import`: Get recent import history.
|
||||
* `GET /history/calculate`: Get recent metrics calculation history.
|
||||
* `GET /status/modules`: Get last calculation time per module.
|
||||
* `GET /status/tables`: Get last sync time per table.
|
||||
* `GET /status/table-counts`: Get record counts for key tables.
|
||||
* `POST /update`: (**Potentially Obsolete**) Trigger `update-csv.js` script.
|
||||
* `POST /import`: Trigger `import-csv.js` script.
|
||||
* `POST /cancel`: Cancel `/full-update` or `/full-reset` task.
|
||||
* `POST /reset`: (**Potentially Obsolete**) Trigger `reset-db.js` script.
|
||||
* `POST /reset-metrics`: Trigger `reset-metrics.js` script.
|
||||
* `POST /calculate-metrics`: Trigger `calculate-metrics.js` script.
|
||||
* `POST /import-from-prod`: Trigger `import-from-prod.js` script.
|
||||
* `POST /full-update`: Trigger `full-update.js` script (preferred update).
|
||||
* `POST /full-reset`: Trigger `full-reset.js` script (preferred reset).
|
||||
|
||||
**8. Dashboard (`dashboard.js`)**
|
||||
* `GET /stock/metrics`: Get dashboard stock summary metrics & brand breakdown.
|
||||
* `GET /purchase/metrics`: Get dashboard purchase order summary metrics & vendor breakdown.
|
||||
* `GET /replenishment/metrics`: Get dashboard replenishment summary & top variants.
|
||||
* `GET /forecast/metrics`: Get dashboard forecast summary, daily, and category breakdown.
|
||||
* `GET /overstock/metrics`: Get dashboard overstock summary & category breakdown.
|
||||
* `GET /overstock/products`: Get list of top overstocked products.
|
||||
* `GET /best-sellers`: Get dashboard best-selling products, brands, categories - **uses `orders`, `products`**.
|
||||
* `GET /sales/metrics`: Get dashboard sales summary for a period - **uses `orders`**.
|
||||
* `GET /low-stock/products`: Get list of top low stock/critical products.
|
||||
* `GET /trending/products`: Get list of trending products - **uses `orders`, `products`**.
|
||||
* `GET /vendor/performance`: Get dashboard vendor performance details - **uses `purchase_orders`**.
|
||||
* `GET /key-metrics`: Get dashboard summary KPIs - **uses multiple base tables**.
|
||||
* `GET /inventory-health`: Get dashboard inventory health overview - **uses `products`, `product_metrics`**.
|
||||
* `GET /replenish/products`: Get list of products needing replenishment (overlaps `/low-stock/products`).
|
||||
* `GET /sales-overview`: Get daily sales totals for chart - **uses `orders`**.
|
||||
|
||||
**9. Product Import Utilities (`products-import.js`)**
|
||||
* `POST /upload-image`: Upload temporary product image, schedule deletion.
|
||||
* `DELETE /delete-image`: Delete temporary product image.
|
||||
* `GET /field-options`: Get dropdown options for product fields from Prod MySQL (cached).
|
||||
* `GET /product-lines/:companyId`: Get product lines for a company from Prod MySQL (cached).
|
||||
* `GET /sublines/:lineId`: Get sublines for a line from Prod MySQL (cached).
|
||||
* `GET /check-file/:filename`: Check existence/permissions of uploaded file (temp or reusable).
|
||||
* `GET /list-uploads`: List files in upload directories.
|
||||
* `GET /search-products`: Search products in Prod MySQL DB.
|
||||
* `GET /check-upc-and-generate-sku`: Check UPC existence and generate SKU suggestion based on Prod MySQL data.
|
||||
* `GET /product-categories/:pid`: Get assigned categories for a product from Prod MySQL.
|
||||
|
||||
**10. Product Metrics (`product-metrics.js`)**
|
||||
* `GET /filter-options`: Get distinct filter values (vendor, brand, abcClass) from `product_metrics`.
|
||||
* `GET /`: List detailed product metrics with filtering, sorting, pagination (primary data access).
|
||||
* `GET /:pid`: Get full metrics record for a single product.
|
||||
|
||||
**11. Orders (`orders.js`)**
|
||||
* `GET /`: List orders with summary info, filtering, sorting, pagination, and stats.
|
||||
* `GET /:orderNumber`: Get details for a single order, including items.
|
||||
|
||||
**12. Products (`products.js`)**
|
||||
* `GET /brands`: Get distinct brands (filtered by PO value).
|
||||
* `GET /`: List products with core data + metrics, filtering, sorting, pagination.
|
||||
* `GET /trending`: Get trending products based on `product_metrics`.
|
||||
* `GET /:id`: Get details for a single product (core data + metrics).
|
||||
* `POST /import`: (**Likely Obsolete/Dangerous**) Import products from CSV.
|
||||
* `PUT /:id`: Update core product data.
|
||||
* `GET /:id/metrics`: (**Redundant**) Get metrics for a single product.
|
||||
* `GET /:id/time-series`: Get sales/PO history for a single product.
|
||||
|
||||
**13. Purchase Orders (`purchase-orders.js`)**
|
||||
* `GET /`: List purchase orders with summary info, filtering, sorting, pagination, and summary stats.
|
||||
* `GET /vendor-metrics`: Calculate vendor performance metrics from `purchase_orders`.
|
||||
* `GET /cost-analysis`: Calculate cost analysis by category from `purchase_orders`.
|
||||
* `GET /receiving-status`: Get summary counts based on PO receiving status.
|
||||
* `GET /order-vs-received`: List product ordered vs. received quantities.
|
||||
|
||||
**14. Reusable Images (`reusable-images.js`)**
|
||||
* `GET /`: List all reusable images.
|
||||
* `GET /by-company/:companyId`: List global and company-specific images.
|
||||
* `GET /global`: List only global images.
|
||||
* `GET /:id`: Get a single reusable image record.
|
||||
* `POST /upload`: Upload a new reusable image and create DB record.
|
||||
* `PUT /:id`: Update reusable image metadata (name, global, company).
|
||||
* `DELETE /:id`: Delete reusable image record and file.
|
||||
* `GET /check-file/:filename`: Check existence/permissions of a reusable image file.
|
||||
|
||||
**15. Templates (`templates.js`)**
|
||||
* `GET /`: List all product data templates.
|
||||
* `GET /:company/:productType`: Get a specific template.
|
||||
* `POST /`: Create a new template.
|
||||
* `PUT /:id`: Update an existing template.
|
||||
* `DELETE /:id`: Delete a template.
|
||||
|
||||
**16. Vendors Aggregate (`vendors-aggregate.js`)**
|
||||
* `GET /filter-options`: Get distinct vendor names and statuses for UI filters (from `vendor_metrics`).
|
||||
* `GET /stats`: Get overall statistics related to vendors (from `vendor_metrics` & `purchase_orders`).
|
||||
* `GET /`: List vendors with aggregated metrics, supporting filtering, sorting, pagination (from `vendor_metrics` & `purchase_orders`).
|
||||
|
||||
**Recommendations:**
|
||||
|
||||
1. **Address Obsolete Endpoints:** Prioritize removing or confirming the necessity of the endpoints marked as obsolete/redundant (legacy config, `/analytics/forecast`, `/csv/update`, `/csv/reset`, `/products/import`, `/products/:id/metrics`).
|
||||
2. **Consolidate Overlapping Functionality:** Review the multiple vendor performance and product listing endpoints. Decide on the primary method (e.g., using aggregate tables via `/vendors-aggregate` and `/metrics`) and refactor or remove the others. Clarify the image upload strategies.
|
||||
3. **Standardize Data Access:** Decide whether `dashboard` and `analytics` endpoints should primarily use aggregate tables (like `/metrics`, `/brands-aggregate`, etc.) or if direct access to base tables is sometimes necessary. Aim for consistency and document the reasoning. Optimize queries hitting base tables if they must remain.
|
||||
4. **Improve Background Task Management:** Refactor `csv.js` to use a unified locking mechanism (maybe separate locks per task type?) and a consistent cancellation strategy for all spawned/managed processes. Clarify the purpose of `update` vs `full-update` and `reset` vs `full-reset`.
|
||||
5. **Optimize DB Connections:** Ensure the `getDbConnection` pooling/caching helper from `products-import.js` is used *consistently* across all modules interacting with the production MySQL database (especially `ai_validation.js`). Remove unnecessary tunnel creations.
|
||||
6. **Review Data Integrity:** Double-check the assumptions made (e.g., uniqueness of AI prompts) and ensure database constraints enforce them. Review the `GET /products/brands` filtering logic.
|
||||
|
||||
## Changes Made
|
||||
|
||||
1. **Removed Obsolete Legacy Endpoints in `config.js`**:
|
||||
- Removed `GET /config/` endpoint
|
||||
- Removed `PUT /config/stock-thresholds/:id` endpoint
|
||||
- Removed `PUT /config/lead-time-thresholds/:id` endpoint
|
||||
- Removed `PUT /config/sales-velocity/:id` endpoint
|
||||
- Removed `PUT /config/abc-classification/:id` endpoint
|
||||
- Removed `PUT /config/safety-stock/:id` endpoint
|
||||
- Removed `PUT /config/turnover/:id` endpoint
|
||||
|
||||
These endpoints were obsolete as they referenced older, single-row config tables that have been replaced by newer endpoints using the structured tables `settings_global`, `settings_product`, and `settings_vendor`.
|
||||
|
||||
2. **Removed MySQL Syntax `/forecast` Endpoint in `analytics.js`**:
|
||||
- Removed `GET /analytics/forecast` endpoint that was using MySQL-specific syntax incompatible with the PostgreSQL database used elsewhere in the application.
|
||||
|
||||
3. **Renamed and Removed Redundant Endpoints**:
|
||||
- Renamed `csv.js` to `data-management.js` while maintaining the same `/csv/*` endpoint paths for consistency
|
||||
- Removed deprecated `/csv/update` endpoint (now fully replaced by `/csv/full-update`)
|
||||
- Removed deprecated `/csv/reset` endpoint (now fully replaced by `/csv/full-reset`)
|
||||
- Removed deprecated `/products/import` endpoint (now handled by `/csv/import`)
|
||||
- Removed deprecated `/products/:id/metrics` endpoint (now handled by `/metrics/:pid`)
|
||||
|
||||
4. **Fixed Data Integrity Issues**:
|
||||
- Improved `GET /products/brands` endpoint by removing the arbitrary filtering logic that was only showing brands with purchase orders that had a total cost of at least $500
|
||||
- The updated endpoint now returns all distinct brands from visible products, providing more complete data
|
||||
|
||||
5. **Optimized Database Connections**:
|
||||
- Created a new `dbConnection.js` utility file that encapsulates the optimized database connection management logic
|
||||
- Improved the `ai-validation.js` file to use this shared connection management, eliminating unnecessary repeated tunnel creation
|
||||
- Added proper connection pooling with timeout-based connection reuse, reducing the overhead of repeatedly creating SSH tunnels
|
||||
- Added query result caching for frequently accessed data to improve performance
|
||||
|
||||
These changes improve maintainability by removing duplicate code, enhance consistency by standardizing on the newer endpoint patterns, and optimize performance by reducing redundant database connections.
|
||||
|
||||
## Additional Improvements
|
||||
|
||||
1. **Further Database Connection Optimizations**:
|
||||
- Extended the use of the optimized database connection utility to additional endpoints in `ai-validation.js`
|
||||
- Updated the `/validate` endpoint and `/test-taxonomy` endpoint to use `getDbConnection`
|
||||
- Ensured consistent connection management across all routes that access the production database
|
||||
|
||||
2. **AI Prompts Data Integrity Verification**:
|
||||
- Confirmed proper uniqueness constraints are in place in the database schema for AI prompts
|
||||
- The schema includes:
|
||||
- `unique_company_prompt` constraint ensuring only one prompt per company
|
||||
- `idx_unique_general_prompt` index ensuring only one general prompt in the system
|
||||
- `idx_unique_system_prompt` index ensuring only one system prompt in the system
|
||||
- Endpoint handlers properly handle uniqueness constraint violations with appropriate 409 Conflict responses
|
||||
- Validation ensures company-specific prompts have company IDs, while general/system prompts do not
|
||||
|
||||
3. **AI Prompts Endpoint Consolidation**:
|
||||
- Added a new consolidated `/by-type` endpoint that handles all types of prompts (general, system, company_specific)
|
||||
- Marked the existing separate endpoints as deprecated with console warnings
|
||||
- Maintained backward compatibility while providing a cleaner API moving forward
|
||||
|
||||
## Completed Items
|
||||
|
||||
✅ Removed obsolete legacy endpoints in `config.js`
|
||||
✅ Removed MySQL syntax `/forecast` endpoint in `analytics.js`
|
||||
✅ Fixed `GET /products/brands` endpoint filtering logic
|
||||
✅ Created reusable database connection utility (`dbConnection.js`)
|
||||
✅ Optimized database connections in `ai-validation.js`
|
||||
✅ Verified data integrity in AI prompts handling
|
||||
✅ Consolidated AI prompts endpoints with a unified `/by-type` endpoint
|
||||
|
||||
## Remaining Items
|
||||
|
||||
- Consider adding additional error handling and logging for database connections
|
||||
- Perform load testing on the optimized database connections to ensure they handle high traffic properly
|
||||
23
docs/setup-chat.md
Normal file
23
docs/setup-chat.md
Normal file
@@ -0,0 +1,23 @@
|
||||
This portion of the application is going to be a read only chat archive. It will pull data from a rocketchat export converted to postgresql. This is a separate database than the rest of the inventory application uses, but it will still use users and permissions from the inventory database. Both databases are on the same postgres instance.
|
||||
|
||||
For now, let's add a select to the top of the page that allows me to "view as" any of the users in the rocketchat database. We'll connect this to the authorization in the main application later.
|
||||
|
||||
The db connection info is stored in the .env file in the inventory-server root. It contains these variables
|
||||
DB_HOST=localhost
|
||||
DB_USER=rocketchat_user
|
||||
DB_PASSWORD=password
|
||||
DB_NAME=rocketchat_converted
|
||||
DB_PORT=5432
|
||||
|
||||
Not all of the information in this database is relevant as it's a direct export from another app with more features. You can use the query tool to examine the structure and data available.
|
||||
|
||||
Server-side files should use similar conventions and the same technologies as the inventory-server (inventor-server root) and auth-server (inventory-server/auth). I will provide my current pm2 ecosystem file upon request for you to add the configuration for the new "chat-server". I use Caddy on the server and can provide my caddyfile to assist with configuring the api routes. All configuration and routes for the chat-server should go in the inventory-server/chat folder or subfolders you create.
|
||||
|
||||
The folder you see as inventory-server is actually a direct mount of the /var/www/html/inventory folder on the server. You can read and write files from there like usual, but any terminal commands for the server I will have to run myself.
|
||||
|
||||
The "Chat" page should be added to the main application sidebar and a similar page to the others should be created in inventory/src/pages. All other frontend pages should go in inventory/src/components/chat.
|
||||
|
||||
The application uses shadcn components and those should be used for all ui elements where possible (located in inventory/src/components/ui). The UI should match existing pages and components.
|
||||
|
||||
|
||||
|
||||
112
docs/split-up-pos.md
Normal file
112
docs/split-up-pos.md
Normal file
@@ -0,0 +1,112 @@
|
||||
Okay, I understand completely now. The core issue is that the previous approaches tried too hard to reconcile every receipt back to a specific PO line within the `purchase_orders` table structure, which doesn't reflect the reality where receipts can be independent events. Your downstream scripts, especially `daily_snapshots` and `product_metrics`, rely on having a complete picture of *all* receivings.
|
||||
|
||||
Let's pivot to a model that respects both distinct data streams: **Orders (Intent)** and **Receivings (Actuals)**.
|
||||
|
||||
**Proposed Solution: Separate `purchase_orders` and `receivings` Tables**
|
||||
|
||||
This is the cleanest way to model the reality you've described.
|
||||
|
||||
1. **`purchase_orders` Table:**
|
||||
* **Purpose:** Tracks the status and details of purchase *orders* placed. Represents the *intent* to receive goods.
|
||||
* **Key Columns:** `po_id`, `pid`, `ordered` (quantity ordered), `po_cost_price`, `date` (order/created date), `expected_date`, `status` (PO lifecycle: 'ordered', 'canceled', 'done'), `vendor`, `notes`, etc.
|
||||
* **Crucially:** This table *does not* need a `received` column or a `receiving_history` column derived from complex allocations. It focuses solely on the PO itself.
|
||||
|
||||
2. **`receivings` Table (New or Refined):**
|
||||
* **Purpose:** Tracks every single line item received, regardless of whether it was linked to a PO during the receiving process. Represents the *actual* goods that arrived.
|
||||
* **Key Columns:**
|
||||
* `receiving_id` (Identifier for the overall receiving document/batch)
|
||||
* `pid` (Product ID received)
|
||||
* `received_qty` (Quantity received for this specific line)
|
||||
* `cost_each` (Actual cost paid for this item on this receiving)
|
||||
* `received_date` (Actual date the item was received)
|
||||
* `received_by` (Employee ID/Name)
|
||||
* `source_po_id` (The `po_id` entered on the receiving screen, *nullable*. Stores the original link attempt, even if it was wrong or missing)
|
||||
* `source_receiving_status` (The status from the source `receivings` table: 'partial_received', 'full_received', 'paid', 'canceled')
|
||||
|
||||
**How the Import Script Changes:**
|
||||
|
||||
1. **Fetch POs:** Fetch data from `po` and `po_products`.
|
||||
2. **Populate `purchase_orders`:**
|
||||
* Insert/Update rows into `purchase_orders` based directly on the fetched PO data.
|
||||
* Set `po_id`, `pid`, `ordered`, `po_cost_price`, `date` (`COALESCE(date_ordered, date_created)`), `expected_date`.
|
||||
* Set `status` by mapping the source `po.status` code directly ('ordered', 'canceled', 'done', etc.).
|
||||
* **No complex allocation needed here.**
|
||||
3. **Fetch Receivings:** Fetch data from `receivings` and `receivings_products`.
|
||||
4. **Populate `receivings`:**
|
||||
* For *every* line item fetched from `receivings_products`:
|
||||
* Perform necessary data validation (dates, numbers).
|
||||
* Insert a new row into `receivings` with all the relevant details (`receiving_id`, `pid`, `received_qty`, `cost_each`, `received_date`, `received_by`, `source_po_id`, `source_receiving_status`).
|
||||
* Use `ON CONFLICT (receiving_id, pid)` (or similar unique key based on your source data) `DO UPDATE SET ...` for incremental updates if necessary, or simply delete/re-insert based on `receiving_id` for simplicity if performance allows.
|
||||
|
||||
**Impact on Downstream Scripts (and how to adapt):**
|
||||
|
||||
* **Initial Query (Active POs):**
|
||||
* `SELECT ... FROM purchase_orders po WHERE po.status NOT IN ('canceled', 'done', 'paid_equivalent_status?') AND po.date >= ...`
|
||||
* `active_pos`: `COUNT(DISTINCT po.po_id)` based on the filtered POs.
|
||||
* `overdue_pos`: Add `AND po.expected_date < CURRENT_DATE`.
|
||||
* `total_units`: `SUM(po.ordered)`. Represents total units *ordered* on active POs.
|
||||
* `total_cost`: `SUM(po.ordered * po.po_cost_price)`. Cost of units *ordered*.
|
||||
* `total_retail`: `SUM(po.ordered * pm.current_price)`. Retail value of units *ordered*.
|
||||
* **Result:** This query now cleanly reports on the status of *orders* placed, which seems closer to its original intent. The filter `po.receiving_status NOT IN ('partial_received', 'full_received', 'paid')` is replaced by `po.status NOT IN ('canceled', 'done', 'paid_equivalent?')`. The 90% received check is removed as `received` is not reliably tracked *on the PO* anymore.
|
||||
|
||||
* **`daily_product_snapshots`:**
|
||||
* **`SalesData` CTE:** No change needed.
|
||||
* **`ReceivingData` CTE:** **Must be changed.** Query the **`receivings`** table instead of `purchase_orders`.
|
||||
```sql
|
||||
ReceivingData AS (
|
||||
SELECT
|
||||
rl.pid,
|
||||
COUNT(DISTINCT rl.receiving_id) as receiving_doc_count,
|
||||
SUM(rl.received_qty) AS units_received,
|
||||
SUM(rl.received_qty * rl.cost_each) AS cost_received
|
||||
FROM public.receivings rl
|
||||
WHERE rl.received_date::date = _date
|
||||
-- Optional: Filter out canceled receivings if needed
|
||||
-- AND rl.source_receiving_status <> 'canceled'
|
||||
GROUP BY rl.pid
|
||||
),
|
||||
```
|
||||
* **Result:** This now accurately reflects *all* units received on a given day from the definitive source.
|
||||
|
||||
* **`update_product_metrics`:**
|
||||
* **`CurrentInfo` CTE:** No change needed (pulls from `products`).
|
||||
* **`OnOrderInfo` CTE:** Needs re-evaluation. How do you want to define "On Order"?
|
||||
* **Option A (Strict PO View):** `SUM(po.ordered)` from `purchase_orders po WHERE po.status NOT IN ('canceled', 'done', 'paid_equivalent?')`. This is quantity on *open orders*, ignoring fulfillment state. Simple, but might overestimate if items arrived unlinked.
|
||||
* **Option B (Approximate Fulfillment):** `SUM(po.ordered)` from open POs MINUS `SUM(rl.received_qty)` from `receivings rl` where `rl.source_po_id = po.po_id` (summing only directly linked receivings). Better, but still misses fulfillment via unlinked receivings.
|
||||
* **Option C (Heuristic):** `SUM(po.ordered)` from open POs MINUS `SUM(rl.received_qty)` from `receivings rl` where `rl.pid = po.pid` and `rl.received_date >= po.date`. This *tries* to account for unlinked receivings but is imprecise.
|
||||
* **Recommendation:** Start with **Option A** for simplicity, clearly labeling it "Quantity on Open POs". You might need a separate process or metric for a more nuanced view of expected vs. actual pipeline.
|
||||
```sql
|
||||
-- Example for Option A
|
||||
OnOrderInfo AS (
|
||||
SELECT
|
||||
pid,
|
||||
SUM(ordered) AS on_order_qty, -- Total qty on open POs
|
||||
SUM(ordered * po_cost_price) AS on_order_cost -- Cost of qty on open POs
|
||||
FROM public.purchase_orders
|
||||
WHERE status NOT IN ('canceled', 'done', 'paid_equivalent?') -- Define your open statuses
|
||||
GROUP BY pid
|
||||
),
|
||||
```
|
||||
* **`HistoricalDates` CTE:**
|
||||
* `date_first_sold`, `max_order_date`: No change (queries `orders`).
|
||||
* `date_first_received_calc`, `date_last_received_calc`: **Must be changed.** Query `MIN(rl.received_date)` and `MAX(rl.received_date)` from the **`receivings`** table grouped by `pid`.
|
||||
* **`SnapshotAggregates` CTE:**
|
||||
* `received_qty_30d`, `received_cost_30d`: These are calculated from `daily_product_snapshots`, which are now correctly sourced from `receivings`, so this part is fine.
|
||||
* **Forecasting Calculations:** Will use the chosen definition of `on_order_qty`. Be aware of the implications of Option A (potentially inflated if unlinked receivings fulfill orders).
|
||||
* **Result:** Metrics are calculated based on distinct order data and complete receiving data. The definition of "on order" needs careful consideration.
|
||||
|
||||
**Summary of this Approach:**
|
||||
|
||||
* **Pros:**
|
||||
* Accurately models distinct order and receiving events.
|
||||
* Provides a definitive source (`receivings`) for all received inventory.
|
||||
* Simplifies the `purchase_orders` table and its import logic.
|
||||
* Avoids complex/potentially inaccurate allocation logic for unlinked receivings within the main tables.
|
||||
* Avoids synthetic records.
|
||||
* Fixes downstream reporting (`daily_snapshots` receiving data).
|
||||
* **Cons:**
|
||||
* Requires creating/managing the `receivings` table.
|
||||
* Requires modifying downstream queries (`ReceivingData`, `OnOrderInfo`, `HistoricalDates`).
|
||||
* Calculating a precise "net quantity still expected to arrive" (true on-order minus all relevant fulfillment) becomes more complex and may require specific business rules or heuristics outside the basic table structure if Option A for `OnOrderInfo` isn't sufficient.
|
||||
|
||||
This two-table approach (`purchase_orders` + `receivings`) seems the most robust and accurate way to handle your requirement for complete receiving records independent of potentially flawed PO linking. It directly addresses the shortcomings of the previous attempts.
|
||||
239
docs/validate-table-changes-implementation-issue4.md
Normal file
239
docs/validate-table-changes-implementation-issue4.md
Normal file
@@ -0,0 +1,239 @@
|
||||
# Validation Display Issue Implementation
|
||||
|
||||
## Issue Being Addressed
|
||||
|
||||
**Validation Display Issue**: Validation isn't happening beyond checking if a cell is required or not. All validation rules defined in import.tsx need to be respected.
|
||||
* Required fields correctly show a red border when empty (✅ ALREADY WORKING)
|
||||
* Non-empty fields with validation errors (regex, unique, etc.) should show a red border AND an alert circle icon with tooltip explaining the error (❌ NOT WORKING)
|
||||
|
||||
## Implementation Attempts
|
||||
|
||||
!!!!**NOTE** All previous attempts have been reverted and are no longer part of the code, please take this into account when trying a new solution. !!!!
|
||||
|
||||
### Attempt 1: Fix Validation Display Logic
|
||||
|
||||
**Approach**: Modified `processErrors` function to separate required errors from validation errors and show alert icons only for non-empty fields with validation errors.
|
||||
|
||||
**Changes Made**:
|
||||
```typescript
|
||||
function processErrors(value: any, errors: ErrorObject[]) {
|
||||
// ...existing code...
|
||||
|
||||
// Separate required errors from other validation errors
|
||||
const requiredErrors = errors.filter(error =>
|
||||
error.message?.toLowerCase().includes('required')
|
||||
);
|
||||
const validationErrors = errors.filter(error =>
|
||||
!error.message?.toLowerCase().includes('required')
|
||||
);
|
||||
|
||||
const isRequiredButEmpty = valueIsEmpty && requiredErrors.length > 0;
|
||||
const hasValidationErrors = validationErrors.length > 0;
|
||||
const shouldShowErrorIcon = hasValidationErrors && !valueIsEmpty;
|
||||
|
||||
// ...more code...
|
||||
}
|
||||
```
|
||||
|
||||
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
|
||||
|
||||
### Attempt 2: Comprehensive Fix for Validation Display
|
||||
|
||||
**Approach**: Completely rewrote `processErrors` function with consistent empty value detection, clear error separation, and improved error message extraction.
|
||||
|
||||
**Changes Made**:
|
||||
```typescript
|
||||
function processErrors(value: any, errors: ErrorObject[]) {
|
||||
if (!errors || errors.length === 0) {
|
||||
return { filteredErrors: [], hasError: false, isRequiredButEmpty: false,
|
||||
shouldShowErrorIcon: false, errorMessages: '' };
|
||||
}
|
||||
|
||||
const valueIsEmpty = isEmpty(value);
|
||||
const requiredErrors = errors.filter(error =>
|
||||
error.message?.toLowerCase().includes('required')
|
||||
);
|
||||
const validationErrors = errors.filter(error =>
|
||||
!error.message?.toLowerCase().includes('required')
|
||||
);
|
||||
|
||||
let filteredErrors = valueIsEmpty ? requiredErrors : validationErrors;
|
||||
|
||||
const isRequiredButEmpty = valueIsEmpty && requiredErrors.length > 0;
|
||||
const hasValidationErrors = validationErrors.length > 0;
|
||||
const hasError = isRequiredButEmpty || hasValidationErrors;
|
||||
const shouldShowErrorIcon = hasValidationErrors && !valueIsEmpty;
|
||||
|
||||
let errorMessages = '';
|
||||
if (shouldShowErrorIcon) {
|
||||
errorMessages = validationErrors.map(getErrorMessage).join('\n');
|
||||
}
|
||||
|
||||
return { filteredErrors, hasError, isRequiredButEmpty, shouldShowErrorIcon, errorMessages };
|
||||
}
|
||||
```
|
||||
|
||||
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
|
||||
|
||||
### Attempt 3: Simplified Error Processing Logic
|
||||
|
||||
**Approach**: Refactored `processErrors` to use shared `isEmpty` function, simplified error icon logic, and made error message extraction more direct.
|
||||
|
||||
**Changes Made**:
|
||||
```typescript
|
||||
function processErrors(value: any, errors: ErrorObject[]) {
|
||||
if (!errors || errors.length === 0) {
|
||||
return { filteredErrors: [], hasError: false, isRequiredButEmpty: false,
|
||||
shouldShowErrorIcon: false, errorMessages: '' };
|
||||
}
|
||||
|
||||
const valueIsEmpty = isEmpty(value);
|
||||
const requiredErrors = errors.filter(error =>
|
||||
error.message?.toLowerCase().includes('required')
|
||||
);
|
||||
const validationErrors = errors.filter(error =>
|
||||
!error.message?.toLowerCase().includes('required')
|
||||
);
|
||||
|
||||
let filteredErrors = valueIsEmpty ? requiredErrors : validationErrors;
|
||||
|
||||
const isRequiredButEmpty = valueIsEmpty && requiredErrors.length > 0;
|
||||
const hasValidationErrors = !valueIsEmpty && validationErrors.length > 0;
|
||||
const hasError = isRequiredButEmpty || hasValidationErrors;
|
||||
const shouldShowErrorIcon = hasValidationErrors;
|
||||
|
||||
let errorMessages = '';
|
||||
if (shouldShowErrorIcon) {
|
||||
errorMessages = validationErrors.map(getErrorMessage).join('\n');
|
||||
}
|
||||
|
||||
return { filteredErrors, hasError, isRequiredButEmpty, shouldShowErrorIcon, errorMessages };
|
||||
}
|
||||
```
|
||||
|
||||
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
|
||||
|
||||
### Attempt 4: Consistent Error Processing Across Components
|
||||
|
||||
**Approach**: Updated both `processErrors` function and `ValidationCell` component to ensure consistent error handling between components.
|
||||
|
||||
**Changes Made**:
|
||||
```typescript
|
||||
// In processErrors function
|
||||
function processErrors(value: any, errors: ErrorObject[]) {
|
||||
// Similar to Attempt 3 with consistent error handling
|
||||
}
|
||||
|
||||
// In ValidationCell component
|
||||
const ValidationCell = ({ field, value, onChange, errors, /* other props */ }) => {
|
||||
// ...existing code...
|
||||
|
||||
// Use the processErrors function to handle validation errors
|
||||
const { hasError, isRequiredButEmpty, shouldShowErrorIcon, errorMessages } =
|
||||
React.useMemo(() => processErrors(value, errors), [value, errors]);
|
||||
|
||||
// ...rest of the component...
|
||||
}
|
||||
```
|
||||
|
||||
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
|
||||
|
||||
### Attempt 5: Unified Error Processing with ItemNumberCell
|
||||
|
||||
**Approach**: Replaced custom error processing in `ValidationCell` with the same `processErrors` utility used by `ItemNumberCell`.
|
||||
|
||||
**Changes Made**:
|
||||
```typescript
|
||||
const ValidationCell = ({ field, value, onChange, errors, /* other props */ }) => {
|
||||
// State and context setup...
|
||||
|
||||
// For item_number fields, use the specialized component
|
||||
if (fieldKey === 'item_number') {
|
||||
return <ItemNumberCell {...props} />;
|
||||
}
|
||||
|
||||
// Use the same processErrors utility function that ItemNumberCell uses
|
||||
const { hasError, isRequiredButEmpty, shouldShowErrorIcon, errorMessages } =
|
||||
React.useMemo(() => processErrors(value, errors), [value, errors]);
|
||||
|
||||
// Rest of component...
|
||||
}
|
||||
```
|
||||
|
||||
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
|
||||
|
||||
### Attempt 6: Standardize Error Processing Across Cell Types
|
||||
|
||||
**Approach**: Standardized error handling across all cell types using the shared `processErrors` utility function.
|
||||
|
||||
**Changes Made**: Similar to Attempt 5, with focus on standardizing the approach for determining when to show validation error icons.
|
||||
|
||||
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
|
||||
|
||||
### Attempt 7: Replace Custom Error Processing with Shared Utility
|
||||
|
||||
**Approach**: Ensured consistent error handling between `ItemNumberCell` and regular `ValidationCell` components.
|
||||
|
||||
**Changes Made**: Similar to Attempts 5 and 6, with focus on using the shared utility function consistently.
|
||||
|
||||
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
|
||||
|
||||
### Attempt 8: Improved Error Normalization and Deep Comparison
|
||||
|
||||
**Approach**: Modified `MemoizedCell` in `ValidationTable.tsx` to use deep comparison for error objects and improved error normalization.
|
||||
|
||||
**Changes Made**:
|
||||
```typescript
|
||||
// Create a memoized cell component
|
||||
const MemoizedCell = React.memo(({ field, value, onChange, errors, /* other props */ }) => {
|
||||
return <ValidationCell {...props} />;
|
||||
}, (prev, next) => {
|
||||
// Basic prop comparison
|
||||
if (prev.value !== next.value) return false;
|
||||
if (prev.isValidating !== next.isValidating) return false;
|
||||
if (prev.itemNumber !== next.itemNumber) return false;
|
||||
|
||||
// Deep compare errors - critical for validation display
|
||||
if (!prev.errors && next.errors) return false;
|
||||
if (prev.errors && !next.errors) return false;
|
||||
if (prev.errors && next.errors) {
|
||||
if (prev.errors.length !== next.errors.length) return false;
|
||||
|
||||
// Compare each error object
|
||||
for (let i = 0; i < prev.errors.length; i++) {
|
||||
if (prev.errors[i].message !== next.errors[i].message) return false;
|
||||
if (prev.errors[i].level !== next.errors[i].level) return false;
|
||||
if (prev.errors[i].source !== next.errors[i].source) return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Compare options...
|
||||
|
||||
return true;
|
||||
});
|
||||
|
||||
// In the field columns definition:
|
||||
cell: ({ row }) => {
|
||||
const rowErrors = validationErrors.get(row.index);
|
||||
const cellErrors = rowErrors?.[fieldKey] || [];
|
||||
|
||||
// Ensure cellErrors is always an array
|
||||
const normalizedErrors = Array.isArray(cellErrors) ? cellErrors : [cellErrors];
|
||||
|
||||
return <MemoizedCell {...props} errors={normalizedErrors} />;
|
||||
}
|
||||
```
|
||||
|
||||
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
|
||||
|
||||
## Root Causes (Revised Hypothesis)
|
||||
|
||||
After multiple attempts, the issue appears more complex than initially thought. Possible root causes:
|
||||
|
||||
1. **Error Object Structure**: Error objects might not have the expected structure or properties
|
||||
2. **Error Propagation**: Errors might be getting filtered out before reaching cell components
|
||||
3. **Validation Rules Configuration**: Validation rules in import.tsx might be incorrectly configured
|
||||
4. **Error State Management**: Error state might not be properly updated or might be reset incorrectly
|
||||
5. **Component Rendering Logic**: Components might not re-render when validation state changes
|
||||
6. **CSS/Styling Issues**: Validation icons might be rendered but hidden due to styling issues
|
||||
7. **Validation Timing**: Validation might be happening at the wrong time or getting overridden
|
||||
138
docs/validate-table-changes-implementation-issue8.md
Normal file
138
docs/validate-table-changes-implementation-issue8.md
Normal file
@@ -0,0 +1,138 @@
|
||||
# Multiple Cell Edit Issue Implementation
|
||||
|
||||
## Issue Being Addressed
|
||||
|
||||
**Multiple Cell Edit Issue**: When you enter values in 2+ cells before validation finishes, contents from all edited cells get erased when validation finishes.
|
||||
|
||||
## Implementation Attempts
|
||||
|
||||
### Attempt 1: Fix Multiple Cell Edit Issue (First Approach)
|
||||
|
||||
**Approach**:
|
||||
- Added a tracking mechanism using a Set to keep track of cells that are currently being edited
|
||||
- Modified the `flushPendingUpdates` function to preserve values of cells being edited
|
||||
- Added cleanup of editing state after validation completes
|
||||
|
||||
**Changes Made**:
|
||||
```typescript
|
||||
// Add ref to track cells currently being edited
|
||||
const currentlyEditingCellsRef = useRef(new Set<string>());
|
||||
|
||||
// Update a row's field value
|
||||
const updateRow = useCallback((rowIndex: number, key: T, value: any) => {
|
||||
// Add this cell to currently editing cells
|
||||
const cellKey = `${rowIndex}-${key}`;
|
||||
currentlyEditingCellsRef.current.add(cellKey);
|
||||
|
||||
// ...existing code...
|
||||
|
||||
// After validation completes, remove this cell from currently editing list
|
||||
setTimeout(() => {
|
||||
currentlyEditingCellsRef.current.delete(cellKey);
|
||||
}, 100);
|
||||
}, []);
|
||||
|
||||
// Modify flushPendingUpdates to respect currently editing cells
|
||||
const flushPendingUpdates = useCallback(() => {
|
||||
// ...existing code...
|
||||
|
||||
if (dataUpdates.length > 0) {
|
||||
setData(prev => {
|
||||
// ...existing code...
|
||||
|
||||
dataUpdates.forEach((row, index) => {
|
||||
if (index < newData.length) {
|
||||
const updatedRow = { ...row };
|
||||
|
||||
// Check if any fields in this row are currently being edited
|
||||
// If so, preserve their current values in the previous data
|
||||
Object.keys(prev[index] || {}).forEach(key => {
|
||||
const cellKey = `${index}-${key}`;
|
||||
if (currentlyEditingCellsRef.current.has(cellKey)) {
|
||||
// Keep the value from the previous state for this field
|
||||
updatedRow[key] = prev[index][key];
|
||||
}
|
||||
});
|
||||
|
||||
newData[index] = updatedRow;
|
||||
}
|
||||
});
|
||||
|
||||
return newData;
|
||||
});
|
||||
}
|
||||
}, []);
|
||||
```
|
||||
|
||||
**Result**:
|
||||
- Slight improvement - the first value entered was saved, but any subsequent values still got erased
|
||||
|
||||
### Attempt 2: Fix Multiple Cell Edit Issue (Second Approach)
|
||||
|
||||
**Approach**:
|
||||
- Completely revised the cell editing tracking system
|
||||
- Used a Map with timestamps to track editing cells more accurately
|
||||
- Added proper Promise-based tracking for cell validation
|
||||
- Increased timeout from 100ms to 1000ms
|
||||
- Made cleanup more robust by checking if it's still the same editing session
|
||||
|
||||
**Changes Made**:
|
||||
```typescript
|
||||
// Add ref to track cells currently being edited with timestamps
|
||||
const currentlyEditingCellsRef = useRef(new Map<string, number>());
|
||||
|
||||
// Add ref to track validation promises
|
||||
const validationPromisesRef = useRef<Map<string, Promise<void>>>(new Map());
|
||||
|
||||
// Update a row's field value
|
||||
const updateRow = useCallback((rowIndex: number, key: T, value: any) => {
|
||||
// Mark this cell as being edited with the current timestamp
|
||||
const cellKey = `${rowIndex}-${key}`;
|
||||
currentlyEditingCellsRef.current.set(cellKey, Date.now());
|
||||
|
||||
// ...existing code...
|
||||
|
||||
// Create a validation promise
|
||||
const validationPromise = new Promise<void>((resolve) => {
|
||||
setTimeout(() => {
|
||||
try {
|
||||
validateRow(rowIndex);
|
||||
} finally {
|
||||
resolve();
|
||||
}
|
||||
}, 0);
|
||||
});
|
||||
|
||||
validationPromisesRef.current.set(cellKey, validationPromise);
|
||||
|
||||
// When validation is complete, remove from validating cells
|
||||
validationPromise.then(() => {
|
||||
// ...existing code...
|
||||
|
||||
// Keep this cell in the editing state for a longer time
|
||||
setTimeout(() => {
|
||||
if (currentlyEditingCellsRef.current.has(cellKey)) {
|
||||
currentlyEditingCellsRef.current.delete(cellKey);
|
||||
}
|
||||
}, 1000); // Keep as "editing" for 1 second
|
||||
});
|
||||
}, []);
|
||||
```
|
||||
|
||||
**Result**:
|
||||
- Worse than the first approach - now all values get erased, including the first one
|
||||
|
||||
## Root Causes (Hypothesized)
|
||||
|
||||
- The validation process might be updating the entire data state, causing race conditions with cell edits
|
||||
- The timing of validation completions might be problematic
|
||||
- State updates might be happening in a way that overwrites user changes
|
||||
- The cell state tracking system is not robust enough to prevent overwrites
|
||||
|
||||
## Next Steps
|
||||
|
||||
The issue requires a more fundamental approach than just tweaking the editing logic. We need to:
|
||||
|
||||
1. Implement a more robust state management system for cell edits that can survive validation cycles
|
||||
2. Consider disabling validation during active editing
|
||||
3. Implement a proper "dirty state" tracking system for cells
|
||||
305
docs/validate-table-changes.md
Normal file
305
docs/validate-table-changes.md
Normal file
@@ -0,0 +1,305 @@
|
||||
# Current Issues to Address
|
||||
4. Validation isn't happening beyond checking if a cell is required or not - needs to respect rules in import.tsx
|
||||
* Red cell outline if cell is required and it's empty
|
||||
* Red outline + alert circle icon with tooltip if cell is NOT empty and isn't valid
|
||||
8. When you enter a value in 2+ cells before validation finishes, contents from all edited cells get erased when validation finishes
|
||||
|
||||
## Do NOT change or edit
|
||||
* Anything related to AI validation
|
||||
* Anything about how templates or UPC validation work (only focus on specific issues described above)
|
||||
* Anything outside of the ValidationStepNew folder
|
||||
|
||||
## Issues already fixed - do not work on these
|
||||
✅FIXED 1. The red row background should go away when all cells in the row are valid and all required cells are populated
|
||||
✅FIXED 2. Columns alignment with header is slightly off, gets worse the further right you go
|
||||
✅FIXED 3. The copy down button is in the way of the validation error icon and the select open trigger - all three need to be in unique locations
|
||||
✅FIXED 5. Description column needs to have an expanded view of some sort, maybe a popover to allow for easier editing
|
||||
* Don't distort table to make it happen
|
||||
✅FIXED 6. Need to ensure all cell's contents don't overflow the input (truncate). COO does this currently, probably more
|
||||
✅FIXED 7. The template select cell is expanding, needs to be fixed size and truncate
|
||||
✅FIXED 9. Import dialog state not fully reset when closing? (validate data step appears scrolled to the middle of the table where I left it)
|
||||
✅FIXED 10. UPC column doesn't need to show loading state when Item Number is being processed, only show on item number column
|
||||
✅FIXED 11. Copy down needs to show a loading state on the cells that it will copy to
|
||||
✅FIXED 12. Shipping restrictions/tax category should default to ID 0 if we didn't get it elsewhere
|
||||
✅FIXED 13. Header row should be sticky (both up/down and left/right)
|
||||
✅FIXED 14. Need a way to scroll around table if user doesn't have mouse wheel for left/right
|
||||
✅FIXED 15. Enhance copy down feature by allowing user to choose the last cell to copy to, instead of going all the way to the bottom
|
||||
|
||||
---------
|
||||
|
||||
# Validation Step Components Overview
|
||||
|
||||
## Core Components
|
||||
|
||||
### ValidationContainer
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx`
|
||||
- Main wrapper component for the validation step
|
||||
- Manages global state and coordinates between subcomponents
|
||||
- Handles navigation events (next, back)
|
||||
- Manages template application and validation state
|
||||
- Coordinates UPC validation and product line loading
|
||||
- Manages row selection and filtering
|
||||
- Contains cache management for UPC validation results
|
||||
- Maintains item number references separate from main data
|
||||
|
||||
### ValidationTable
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx`
|
||||
- Handles data display and column configuration
|
||||
- Uses TanStack Table for core functionality
|
||||
- Features:
|
||||
- Sticky header (both vertical and horizontal) - currently doesn't work properly
|
||||
- Row selection with checkboxes
|
||||
- Template selection column
|
||||
- Dynamic column widths based on field types - specified in import.tsx component
|
||||
- Copy down functionality for cell values
|
||||
- Error highlighting for rows and cells
|
||||
- Loading states for cells being validated
|
||||
|
||||
### ValidationCell
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx`
|
||||
- Base cell component that renders different cell types based on field configuration
|
||||
- Handles error display with tooltips
|
||||
- Manages copy down button visibility
|
||||
- Supports loading states during validation
|
||||
- Cell Types:
|
||||
1. InputCell: For single-value text input
|
||||
2. SelectCell: For dropdown selection
|
||||
3. MultiInputCell: For multiple value inputs
|
||||
4. Template selection cells with SearchableTemplateSelect component
|
||||
|
||||
### SearchableTemplateSelect
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx`
|
||||
- Advanced template selection component with search functionality
|
||||
- Features:
|
||||
- Real-time search filtering of templates
|
||||
- Customizable display text for templates
|
||||
- Support for default brand selection
|
||||
- Accessible popover interface
|
||||
- Keyboard navigation support
|
||||
- Custom styling through className props
|
||||
- Scroll event handling for nested scrollable areas
|
||||
|
||||
### TemplateManager
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/TemplateManager.tsx`
|
||||
- Comprehensive template management interface
|
||||
- Features:
|
||||
- Template selection with search functionality
|
||||
- Save template dialog with name and type inputs
|
||||
- Batch template application to selected rows
|
||||
- Template count tracking
|
||||
- Toast notifications for user feedback
|
||||
- Dialog-based interface for template operations
|
||||
|
||||
### AiValidationDialogs
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx`
|
||||
- Manages AI-assisted validation dialogs and interactions
|
||||
|
||||
### SaveTemplateDialog
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SaveTemplateDialog.tsx`
|
||||
- Dialog component for saving new templates
|
||||
|
||||
## Cell Components
|
||||
|
||||
### InputCell
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx`
|
||||
- Handles single value text input
|
||||
- Features:
|
||||
- Inline/edit mode switching
|
||||
- Multiline support
|
||||
- Price formatting
|
||||
- Error state display
|
||||
- Loading state during validation
|
||||
- Width constraints
|
||||
- Automated cleanPriceFields processing for "$" formatting
|
||||
|
||||
### SelectCell
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx`
|
||||
- Handles dropdown selection
|
||||
- Features:
|
||||
- Searchable dropdown
|
||||
- Custom option rendering
|
||||
- Error state display
|
||||
- Loading state during validation
|
||||
- Width constraints
|
||||
- Disabled state support
|
||||
- Deferred search query handling for performance
|
||||
|
||||
### MultiInputCell
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiInputCell.tsx`
|
||||
- Handles multiple value inputs
|
||||
- Features:
|
||||
- Comma-separated input support
|
||||
- Multi-select dropdown for predefined options
|
||||
- Custom separators
|
||||
- Badge display for selected count
|
||||
- Truncation for long values
|
||||
- Width constraints
|
||||
- Price formatting support
|
||||
- Internal state management to avoid excessive re-renders
|
||||
|
||||
## Validation System
|
||||
|
||||
### useValidation Hook
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx`
|
||||
- Provides core validation logic
|
||||
- Validates at multiple levels:
|
||||
1. Field-level validation (required, regex, unique)
|
||||
2. Row-level validation (supplier, company fields)
|
||||
3. Table-level validation
|
||||
4. Custom validation hooks support
|
||||
- Error object structure includes message, level, and source properties
|
||||
- Handles debounced validation updates to avoid UI freezing
|
||||
|
||||
### useAiValidation Hook
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx`
|
||||
- Manages AI-assisted validation logic and state
|
||||
- Features:
|
||||
- Tracks detailed changes per product
|
||||
- Manages validation progress with estimated completion time
|
||||
- Handles warnings and change suggestions
|
||||
- Supports diff generation for changes
|
||||
- Progress tracking with step indicators
|
||||
- Prompt management for AI interactions
|
||||
- Timer management for long-running operations
|
||||
|
||||
### useTemplates Hook
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx`
|
||||
- Comprehensive template management system
|
||||
- Features:
|
||||
- Template CRUD operations
|
||||
- Template application logic
|
||||
- Default value handling
|
||||
- Template search and filtering
|
||||
- Batch template operations
|
||||
- Template validation
|
||||
|
||||
### useUpcValidation Hook
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx`
|
||||
- Dedicated UPC validation management
|
||||
- Features:
|
||||
- UPC format validation
|
||||
- Supplier data validation
|
||||
- Cache management for validation results
|
||||
- Batch processing of UPC validations
|
||||
- Item number generation logic
|
||||
- Loading state management
|
||||
|
||||
### useFilters Hook
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useFilters.tsx`
|
||||
- Advanced filtering system for table data
|
||||
- Features:
|
||||
- Multiple filter criteria support
|
||||
- Dynamic filter updates
|
||||
- Filter persistence
|
||||
- Filter combination logic
|
||||
- Performance optimized filtering
|
||||
|
||||
### useValidationState Hook
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx`
|
||||
- Manages global validation state
|
||||
- Handles:
|
||||
- Data updates
|
||||
- Template management
|
||||
- Error tracking using Map objects
|
||||
- Row selection
|
||||
- Filtering
|
||||
- UPC validation with caching to prevent duplicate API calls
|
||||
- Product line loading
|
||||
- Batch processing of updates
|
||||
- Default value application for tax_cat and ship_restrictions (defaulting to "0")
|
||||
- Price field auto-formatting to remove "$" symbols
|
||||
|
||||
### Utility Files
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts`
|
||||
- Core validation utility functions
|
||||
- Includes:
|
||||
- Field validation logic
|
||||
- Error message formatting
|
||||
- Validation rule processing
|
||||
- Type checking utilities
|
||||
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts`
|
||||
- Error handling and formatting utilities
|
||||
- Includes:
|
||||
- Error object creation
|
||||
- Error message formatting
|
||||
- Error source tracking
|
||||
- Error level management
|
||||
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts`
|
||||
- Data transformation and mutation utilities
|
||||
- Includes:
|
||||
- Row data updates
|
||||
- Batch data processing
|
||||
- Data structure conversions
|
||||
- Change tracking
|
||||
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js`
|
||||
- Helper functions for validation
|
||||
- Includes:
|
||||
- Common validation patterns
|
||||
- Validation state management
|
||||
- Validation result processing
|
||||
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts`
|
||||
- UPC-specific validation utilities
|
||||
- Includes:
|
||||
- UPC format checking
|
||||
- Checksum validation
|
||||
- Supplier data matching
|
||||
- Cache management
|
||||
|
||||
### Types
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts`
|
||||
- Core type definitions for the validation step
|
||||
|
||||
### Validation Types
|
||||
1. Required field validation
|
||||
2. Regex pattern validation
|
||||
3. Unique value validation
|
||||
4. Custom field validation
|
||||
5. Row-level validation
|
||||
6. Table-level validation
|
||||
|
||||
## State Management
|
||||
|
||||
### useValidationState Hook
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx`
|
||||
- Manages global validation state
|
||||
- Handles:
|
||||
- Data updates
|
||||
- Template management
|
||||
- Error tracking using Map objects
|
||||
- Row selection
|
||||
- Filtering
|
||||
- UPC validation with caching to prevent duplicate API calls
|
||||
- Product line loading
|
||||
- Batch processing of updates
|
||||
- Default value application for tax_cat and ship_restrictions (defaulting to "0")
|
||||
- Price field auto-formatting to remove "$" symbols
|
||||
|
||||
## UPC Validation System
|
||||
|
||||
### UPC Processing
|
||||
- Validates UPCs against supplier data
|
||||
- Cache system for UPC validation results
|
||||
- Batch processing of UPC validation requests
|
||||
- Auto-generation of item numbers based on UPC
|
||||
- Special loading states for UPC/item number fields
|
||||
- Separate state tracking to avoid unnecessary data structure updates
|
||||
|
||||
## Template System
|
||||
|
||||
### Template Management
|
||||
- Supports saving and loading templates
|
||||
- Template application to single/multiple rows
|
||||
- Default template values
|
||||
- Template search and filtering
|
||||
|
||||
## Performance Optimizations
|
||||
1. Memoized components to prevent unnecessary renders
|
||||
2. Virtualized table for large datasets
|
||||
3. Deferred value updates for search inputs
|
||||
4. Efficient error state management
|
||||
5. Optimized cell update handling
|
||||
|
||||
131
docs/validation-hook-refactor.md
Normal file
131
docs/validation-hook-refactor.md
Normal file
@@ -0,0 +1,131 @@
|
||||
|
||||
|
||||
# Refactoring Plan for Validation Code
|
||||
|
||||
## Current Structure Analysis
|
||||
- **useValidationState.tsx**: ~1650 lines - Core validation state management
|
||||
- **useValidation.tsx**: ~425 lines - Field/data validation utility
|
||||
- **useUpcValidation.tsx**: ~410 lines - UPC-specific validation
|
||||
|
||||
## Proposed New Structure
|
||||
|
||||
### 1. Core Types & Utilities (150-200 lines)
|
||||
**File: `validation/types.ts`**
|
||||
- All interfaces and types (RowData, ValidationError, FilterState, Template, etc.)
|
||||
- Shared utility functions (isEmpty, getCellKey, etc.)
|
||||
|
||||
**File: `validation/utils.ts`**
|
||||
- Generic validation utility functions
|
||||
- Caching mechanism and cache clearing helpers
|
||||
- API URL helpers
|
||||
|
||||
### 2. Field Validation (300-350 lines)
|
||||
**File: `validation/hooks/useFieldValidation.ts`**
|
||||
- `validateField` function
|
||||
- Field-level validation logic
|
||||
- Required, regex, and other field validations
|
||||
|
||||
### 3. Uniqueness Validation (250-300 lines)
|
||||
**File: `validation/hooks/useUniquenessValidation.ts`**
|
||||
- `validateUniqueField` function
|
||||
- `validateUniqueItemNumbers` function
|
||||
- All uniqueness checking logic
|
||||
|
||||
### 4. UPC Validation (300-350 lines)
|
||||
**File: `validation/hooks/useUpcValidation.ts`**
|
||||
- `fetchProductByUpc` function
|
||||
- `validateUpc` function
|
||||
- `applyItemNumbersToData` function
|
||||
- UPC validation state management
|
||||
|
||||
### 5. Validation Status Management (300-350 lines)
|
||||
**File: `validation/hooks/useValidationStatus.ts`**
|
||||
- Error state management
|
||||
- Row validation status tracking
|
||||
- Validation indicators and refs
|
||||
- Batch validation processing
|
||||
|
||||
### 6. Data Management (300-350 lines)
|
||||
**File: `validation/hooks/useValidationData.ts`**
|
||||
- Data state management
|
||||
- Row updates
|
||||
- Data filtering
|
||||
- Initial data processing
|
||||
|
||||
### 7. Template Management (250-300 lines)
|
||||
**File: `validation/hooks/useTemplateManagement.ts`**
|
||||
- Template saving
|
||||
- Template application
|
||||
- Template loading
|
||||
- Template display helpers
|
||||
|
||||
### 8. Main Validation Hook (300-350 lines)
|
||||
**File: `validation/hooks/useValidation.ts`**
|
||||
- Main hook that composes all other hooks
|
||||
- Public API export
|
||||
- Initialization logic
|
||||
- Core validation flow
|
||||
|
||||
## Function Distribution
|
||||
|
||||
### Core Types & Utilities
|
||||
- All interfaces (InfoWithSource, ValidationState, etc.)
|
||||
- `isEmpty` utility
|
||||
- `getApiUrl` helper
|
||||
|
||||
### Field Validation
|
||||
- `validateField`
|
||||
- `validateRow`
|
||||
- `validateData` (partial)
|
||||
- All validation result caching
|
||||
|
||||
### Uniqueness Validation
|
||||
- `validateUniqueField`
|
||||
- `validateUniqueItemNumbers`
|
||||
- Uniqueness caching mechanisms
|
||||
|
||||
### UPC Validation
|
||||
- `fetchProductByUpc`
|
||||
- `validateUpc`
|
||||
- `validateAllUPCs`
|
||||
- `applyItemNumbersToData`
|
||||
- UPC validation state tracking (cells, rows)
|
||||
|
||||
### Validation Status Management
|
||||
- `startValidatingCell`/`stopValidatingCell`
|
||||
- `startValidatingRow`/`stopValidatingRow`
|
||||
- `isValidatingCell`/`isRowValidatingUpc`
|
||||
- Error state management
|
||||
- `revalidateRows`
|
||||
|
||||
### Data Management
|
||||
- Initial data cleaning/processing
|
||||
- `updateRow`
|
||||
- `copyDown`
|
||||
- Search/filter functionality
|
||||
- `filteredData` calculation
|
||||
|
||||
### Template Management
|
||||
- `saveTemplate`
|
||||
- `applyTemplate`
|
||||
- `applyTemplateToSelected`
|
||||
- `getTemplateDisplayText`
|
||||
- `loadTemplates`/`refreshTemplates`
|
||||
|
||||
### Main Validation Hook
|
||||
- Composition of all other hooks
|
||||
- Initialization logic
|
||||
- Button/navigation handling
|
||||
- Field options management
|
||||
|
||||
## Implementation Approach
|
||||
|
||||
1. **Start with Types**: Create the types file first, as all other files will depend on it
|
||||
2. **Create Utility Functions**: Move shared utilities next
|
||||
3. **Build Core Validation**: Extract the field validation and uniqueness validation
|
||||
4. **Separate UPC Logic**: Move all UPC-specific code to its own module
|
||||
5. **Extract State Management**: Move data and status management to separate files
|
||||
6. **Move Template Logic**: Extract template functionality
|
||||
7. **Create Composition Hook**: Build the main hook that uses all other hooks
|
||||
|
||||
This approach will give you more maintainable code with clearer separation of concerns, making it easier to understand, test, and modify each component independently.
|
||||
354
docs/validation-process-issues.md
Normal file
354
docs/validation-process-issues.md
Normal file
@@ -0,0 +1,354 @@
|
||||
## 1. ✅ Error Filtering Logic Inconsistency (RESOLVED)
|
||||
|
||||
> **Note: This issue has been resolved by implementing a type-based error system.**
|
||||
|
||||
The filtering logic in `ValidationCell.tsx` previously relied on string matching, which was fragile:
|
||||
|
||||
```typescript
|
||||
// Old implementation (string-based matching)
|
||||
const filteredErrors = React.useMemo(() => {
|
||||
return !isEmpty(value)
|
||||
? errors.filter(error => !error.message?.toLowerCase().includes('required'))
|
||||
: errors;
|
||||
}, [value, errors]);
|
||||
|
||||
// New implementation (type-based filtering)
|
||||
const filteredErrors = React.useMemo(() => {
|
||||
return !isEmpty(value)
|
||||
? errors.filter(error => error.type !== ErrorType.Required)
|
||||
: errors;
|
||||
}, [value, errors]);
|
||||
```
|
||||
|
||||
The solution implemented:
|
||||
- Added an `ErrorType` enum in `types.ts` to standardize error categorization
|
||||
- Updated all error creation to include the appropriate error type
|
||||
- Modified error filtering to use the type property instead of string matching
|
||||
- Ensured consistent error handling across the application
|
||||
|
||||
**Guidelines for future development:**
|
||||
- Always use the `ErrorType` enum when creating errors
|
||||
- Never rely on string matching for error filtering
|
||||
- Ensure all error objects include the `type` property
|
||||
- Use the appropriate error type for each validation rule:
|
||||
- `ErrorType.Required` for required field validations
|
||||
- `ErrorType.Regex` for regex validations
|
||||
- `ErrorType.Unique` for uniqueness validations
|
||||
- `ErrorType.Custom` for custom validations
|
||||
- `ErrorType.Api` for API-based validations
|
||||
|
||||
## 2. ⚠️ Redundant Error Processing (PARTIALLY RESOLVED)
|
||||
|
||||
> **Note: This issue has been partially resolved by the re-rendering optimizations.**
|
||||
|
||||
The system still processes errors in multiple places:
|
||||
- In `ValidationCell.tsx`, errors are filtered by the optimized `processErrors` function
|
||||
- In `useValidation.tsx`, errors are generated at the field level
|
||||
- In `ValidationContainer.tsx`, errors are manipulated at the container level
|
||||
|
||||
While the error processing has been optimized to be more efficient, there is still some redundancy in how errors are handled across components. However, the current implementation has mitigated the performance impact.
|
||||
|
||||
**Improvements made:**
|
||||
- Created a central `processErrors` function in ValidationCell that efficiently handles error filtering
|
||||
- Implemented a batched update system to reduce redundant error processing
|
||||
- Added better memoization to avoid reprocessing errors when not needed
|
||||
|
||||
**Future improvement opportunities:**
|
||||
- Further consolidate error processing logic into a single location
|
||||
- Create a dedicated error handling service or hook
|
||||
- Implement a more declarative approach to error handling
|
||||
|
||||
## 3. Race Conditions in Async Validation
|
||||
|
||||
async validations could create race conditions:
|
||||
- If a user types quickly, multiple validation requests might be in flight
|
||||
- Later responses could overwrite more recent ones if they complete out of order
|
||||
- The debouncing helps but doesn't fully solve this issue
|
||||
|
||||
## 4. Memory Leaks in Timeout Management
|
||||
|
||||
The validation timeouts are stored in refs:
|
||||
```typescript
|
||||
const validationTimeoutsRef = useRef<Record<number, NodeJS.Timeout>>({});
|
||||
```
|
||||
|
||||
While there is cleanup on unmount, if rows are added/removed dynamically, timeouts for deleted rows might not be properly cleared.
|
||||
|
||||
## 5. ✅ Inefficient Error Storage (RESOLVED)
|
||||
|
||||
**Status: RESOLVED**
|
||||
|
||||
### Problem
|
||||
|
||||
Previously, validation errors were stored in multiple locations:
|
||||
- In the `validationErrors` Map in `useValidationState`
|
||||
- In the row data itself as `__errors`
|
||||
|
||||
This redundancy caused several issues:
|
||||
- Inconsistent error states between the two storage locations
|
||||
- Increased memory usage by storing the same information twice
|
||||
- Complex state management to keep both sources in sync
|
||||
- Difficulty reasoning about where errors should be accessed from
|
||||
|
||||
### Solution
|
||||
|
||||
We've implemented a unified error storage approach by:
|
||||
- Making the `validationErrors` Map in `useValidationState` the single source of truth for all validation errors
|
||||
- Removed the `__errors` property from row data
|
||||
- Updated all validation functions to interact with the central error store instead of modifying row data
|
||||
- Modified UPC validation to use the central error store
|
||||
- Updated all components to read errors from the `validationErrors` Map instead of row data
|
||||
|
||||
### Key Changes
|
||||
|
||||
1. Modified `dataMutations.ts` to stop storing errors in row data
|
||||
2. Updated the `Meta` type to remove the `__errors` property
|
||||
3. Modified the `RowData` type to remove the `__errors` property
|
||||
4. Updated the `useValidation` hook to return errors separately from row data
|
||||
5. Modified the `useAiValidation` hook to work with the central error store
|
||||
6. Updated the `useFilters` hook to check for errors in the `validationErrors` Map
|
||||
7. Modified the `ValidationTable` and `ValidationCell` components to read errors from the `validationErrors` Map
|
||||
|
||||
### Benefits
|
||||
|
||||
- **Single Source of Truth**: All validation errors are now stored in one place
|
||||
- **Reduced Memory Usage**: No duplicate storage of error information
|
||||
- **Simplified State Management**: Only one state to update when errors change
|
||||
- **Cleaner Data Structure**: Row data no longer contains validation metadata
|
||||
- **More Maintainable Code**: Clearer separation of concerns between data and validation
|
||||
|
||||
### Future Improvements
|
||||
|
||||
While this refactoring addresses the core issue of inefficient error storage, there are still opportunities for further optimization:
|
||||
|
||||
1. ✅ **Redundant Error Processing**: ~~The validation process still performs some redundant calculations that could be optimized.~~ This has been largely addressed by the re-rendering optimizations.
|
||||
2. **Race Conditions**: Async validation can lead to race conditions when multiple validations are triggered in quick succession.
|
||||
3. **Memory Leaks**: The timeout management for validation could be improved to prevent potential memory leaks.
|
||||
4. **Tight Coupling**: Components are still tightly coupled to the validation state structure.
|
||||
5. **Error Prioritization**: The system doesn't prioritize errors well, showing all errors at once rather than focusing on the most critical ones first.
|
||||
|
||||
### Validation Flow
|
||||
|
||||
The validation process now works as follows:
|
||||
|
||||
1. **Error Generation**:
|
||||
- Field-level validations generate errors based on validation rules
|
||||
- Row-level hooks add custom validation errors
|
||||
- Table-level validations (like uniqueness checks) add errors across rows
|
||||
|
||||
2. **Error Storage**:
|
||||
- All errors are stored in the `validationErrors` Map in `useValidationState`
|
||||
- The Map uses row indices as keys and objects of field errors as values
|
||||
|
||||
3. **Error Display**:
|
||||
- The `ValidationTable` component checks the `validationErrors` Map to highlight rows with errors
|
||||
- The `ValidationCell` component receives errors for specific fields from the `validationErrors` Map
|
||||
- Errors are filtered in the UI to avoid showing "required" errors for fields with values
|
||||
|
||||
This focused refactoring approach has successfully addressed a critical issue while keeping changes manageable and targeted.
|
||||
|
||||
## 6. ✅ Excessive Re-rendering (RESOLVED)
|
||||
|
||||
**Status: RESOLVED**
|
||||
|
||||
### Problem
|
||||
|
||||
The validation system was suffering from excessive re-renders due to several key issues:
|
||||
|
||||
- **Inefficient Error Filtering**: The ValidationCell component was filtering errors on every render
|
||||
- **Redundant Error Processing**: The same validation work was repeated in multiple components
|
||||
- **Poor Memoization**: Components were inadequately memoized, causing unnecessary re-renders
|
||||
- **Inefficient Batch Updates**: The state update system wasn't optimally batching changes
|
||||
|
||||
These issues led to performance problems, especially with large datasets, and affected the user experience.
|
||||
|
||||
### Solution
|
||||
|
||||
We've implemented a comprehensive optimization approach:
|
||||
|
||||
- **Optimized Error Processing**: Created an efficient `processErrors` function in ValidationCell that calculates all derived state in one pass
|
||||
- **Enhanced Memoization**: Improved memo comparison functions to avoid unnecessary rerenders
|
||||
- **Improved Batch Updates**: Redesigned the batching system to aggregate multiple changes before state updates
|
||||
- **Single Update Pattern**: Implemented a queue-based update mechanism that applies multiple state changes at once
|
||||
|
||||
### Key Changes
|
||||
|
||||
1. Added a more efficient error processing function in ValidationCell
|
||||
2. Created an enhanced error comparison function to properly compare error arrays
|
||||
3. Improved the memo comparison function in ValidationCell
|
||||
4. Added a batch update system in useValidationState
|
||||
5. Implemented a queue-based update mechanism for row modifications
|
||||
|
||||
### Benefits
|
||||
|
||||
- **Improved Performance**: Reduced render cycles = faster UI response
|
||||
- **Better User Experience**: Less lag when editing large datasets
|
||||
- **Reduced Memory Usage**: Fewer component instantiations and temporary objects
|
||||
- **Increased Scalability**: The application can now handle larger datasets without slowdown
|
||||
- **Maintainable Code**: More predictable update flow that's easier to debug and extend
|
||||
|
||||
### Guidelines for future development
|
||||
|
||||
- Use the `processErrors` function for error filtering and processing
|
||||
- Ensure React.memo components have proper comparison functions
|
||||
- Use the batched update system for state changes
|
||||
- Maintain stable references to objects and functions
|
||||
- Use appropriate React hooks (useMemo, useCallback) with correct dependencies
|
||||
- Avoid unnecessary recreations of arrays, objects, and functions
|
||||
|
||||
## 7. Complex Error Merging Logic
|
||||
|
||||
When merging errors from different sources, the logic is complex and potentially error-prone:
|
||||
```typescript
|
||||
// Merge field errors and row hook errors
|
||||
const mergedErrors: Record<string, InfoWithSource> = {}
|
||||
|
||||
// Convert field errors to InfoWithSource
|
||||
Object.entries(fieldErrors).forEach(([key, errors]) => {
|
||||
if (errors.length > 0) {
|
||||
mergedErrors[key] = {
|
||||
message: errors[0].message,
|
||||
level: errors[0].level,
|
||||
source: ErrorSources.Row,
|
||||
type: errors[0].type || ErrorType.Custom
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
This only takes the first error for each field, potentially hiding important validation issues.
|
||||
|
||||
## 8. ✅ Inconsistent Error Handling for Empty Values (PARTIALLY RESOLVED)
|
||||
|
||||
> **Note: This issue has been partially resolved by standardizing the isEmpty function and error type system.**
|
||||
|
||||
The system previously had different approaches to handling empty values:
|
||||
- Some validations skipped empty values unless they're required
|
||||
- Others processed empty values differently
|
||||
- The `isEmpty` function was defined multiple times with slight variations
|
||||
|
||||
The solution implemented:
|
||||
- Standardized the `isEmpty` function implementation
|
||||
- Ensured consistent error type usage for required field validations
|
||||
- Made error filtering consistent across the application
|
||||
|
||||
**Guidelines for future development:**
|
||||
- Always use the shared `isEmpty` function for checking empty values
|
||||
- Ensure consistent handling of empty values across all validation rules
|
||||
- Use the `ErrorType.Required` type for all required field validations
|
||||
|
||||
## 9. Tight Coupling Between Components
|
||||
|
||||
The validation system is tightly coupled across components:
|
||||
- `ValidationCell` needs to understand the structure of errors
|
||||
- `ValidationTable` needs to extract and pass the right errors
|
||||
- `ValidationContainer` directly manipulates the error structure
|
||||
|
||||
This makes it harder to refactor or reuse components independently.
|
||||
|
||||
## 10. Limited Error Prioritization
|
||||
|
||||
There's no clear prioritization of errors:
|
||||
- When multiple errors exist for a field, which one should be shown first?
|
||||
- Are some errors more important than others?
|
||||
- The current system mostly shows the first error it finds
|
||||
|
||||
A more robust approach would be to have a consistent error source identification system and a clear prioritization strategy for displaying errors.
|
||||
|
||||
------------
|
||||
|
||||
Let me explain how these hooks fit together to create the validation errors that eventually get filtered in the `ValidationCell` component:
|
||||
|
||||
## The Validation Flow
|
||||
|
||||
1. **useValidationState Hook**:
|
||||
This is the main state management hook used by the `ValidationContainer` component. It:
|
||||
- Manages the core data state (`data`)
|
||||
- Tracks validation errors in a Map (`validationErrors`)
|
||||
- Provides functions to update and validate rows
|
||||
|
||||
2. **useValidation Hook**:
|
||||
This is a utility hook that provides the core validation logic:
|
||||
- `validateField`: Validates a single field against its validation rules
|
||||
- `validateRow`: Validates an entire row, field by field
|
||||
- `validateTable`: Runs table-level validations
|
||||
- `validateUnique`: Checks for uniqueness constraints
|
||||
- `validateData`: Orchestrates the complete validation process
|
||||
|
||||
## How Errors Are Generated
|
||||
|
||||
Validation errors come from multiple sources:
|
||||
|
||||
1. **Field-Level Validations**:
|
||||
In `useValidation.tsx`, the `validateField` function checks individual fields against rules like:
|
||||
- `required`: Field must have a value
|
||||
- `regex`: Value must match a pattern
|
||||
- `min`/`max`: Numeric constraints
|
||||
|
||||
2. **Row-Level Validations**:
|
||||
The `validateRow` function in `useValidation.tsx` runs:
|
||||
- Field validations for each field in the row
|
||||
- Special validations for required fields like supplier and company
|
||||
- Custom row hooks provided by the application
|
||||
|
||||
3. **Table-Level Validations**:
|
||||
- `validateUnique` checks for duplicate values in fields marked as unique
|
||||
- `validateTable` runs custom table hooks for cross-row validations
|
||||
|
||||
4. **API-Based Validations**:
|
||||
In `useValidationState.tsx` and `ValidationContainer.tsx`:
|
||||
- UPC validation via API calls
|
||||
- Item number uniqueness checks
|
||||
|
||||
## The Error Flow
|
||||
|
||||
1. Errors are collected in the `validationErrors` Map in `useValidationState`
|
||||
2. This Map is passed to `ValidationTable` as a prop
|
||||
3. `ValidationTable` extracts the relevant errors for each cell and passes them to `ValidationCell`
|
||||
4. In `ValidationCell`, the errors are filtered based on whether the cell has a value:
|
||||
```typescript
|
||||
// Updated implementation using type-based filtering
|
||||
const filteredErrors = React.useMemo(() => {
|
||||
return !isEmpty(value)
|
||||
? errors.filter(error => error.type !== ErrorType.Required)
|
||||
: errors;
|
||||
}, [value, errors]);
|
||||
```
|
||||
|
||||
## Key Insights
|
||||
|
||||
1. **Error Structure**:
|
||||
Errors now have a consistent structure with type information:
|
||||
```typescript
|
||||
type ErrorObject = {
|
||||
message: string;
|
||||
level: string; // 'error', 'warning', etc.
|
||||
source?: ErrorSources; // Where the error came from
|
||||
type: ErrorType; // The type of error (Required, Regex, Unique, etc.)
|
||||
}
|
||||
```
|
||||
|
||||
2. **Error Sources**:
|
||||
Errors can come from:
|
||||
- Field validations (required, regex, etc.)
|
||||
- Row validations (custom business logic)
|
||||
- Table validations (uniqueness checks)
|
||||
- API validations (UPC checks)
|
||||
|
||||
3. **Error Types**:
|
||||
Errors are now categorized by type:
|
||||
- `ErrorType.Required`: Field is required but empty
|
||||
- `ErrorType.Regex`: Value doesn't match the regex pattern
|
||||
- `ErrorType.Unique`: Value must be unique across rows
|
||||
- `ErrorType.Custom`: Custom validation errors
|
||||
- `ErrorType.Api`: Errors from API calls
|
||||
|
||||
4. **Error Filtering**:
|
||||
The filtering in `ValidationCell` is now more robust:
|
||||
- When a field has a value, errors of type `ErrorType.Required` are filtered out
|
||||
- When a field is empty, all errors are shown
|
||||
|
||||
5. **Performance Optimizations**:
|
||||
- Batch processing of validations
|
||||
- Debounced updates to avoid excessive re-renders
|
||||
- Memoization of computed values
|
||||
538
docs/validation-table-scroll-issue.md
Normal file
538
docs/validation-table-scroll-issue.md
Normal file
@@ -0,0 +1,538 @@
|
||||
# ValidationTable Scroll Position Issue
|
||||
|
||||
## Problem Description
|
||||
|
||||
The `ValidationTable` component in the inventory application suffers from a persistent scroll position issue. When the table content updates or re-renders, the scroll position resets to the top left corner. This creates a poor user experience, especially when users are working with large datasets and need to maintain their position while making edits or filtering data.
|
||||
|
||||
Specific behaviors:
|
||||
- Scroll position resets to the top left corner during re-renders
|
||||
- User loses their place in the table when data is updated
|
||||
- The table does not preserve vertical or horizontal scroll position
|
||||
|
||||
## Relevant Files
|
||||
|
||||
- **`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx`**
|
||||
- Main component that renders the validation table
|
||||
- Handles scroll position management
|
||||
|
||||
- **`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx`**
|
||||
- Parent component that wraps ValidationTable
|
||||
- Creates an EnhancedValidationTable wrapper component
|
||||
- Manages data and state for the validation table
|
||||
|
||||
- **`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx`**
|
||||
- Provides state management and data manipulation functions
|
||||
- Contains scroll-related code in the `updateRow` function
|
||||
|
||||
- **`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx`**
|
||||
- Renders individual cells in the table
|
||||
- May influence re-renders that affect scroll position
|
||||
|
||||
## Failed Attempts
|
||||
|
||||
We've tried multiple approaches to fix the scroll position issue, none of which have been successful:
|
||||
|
||||
### 1. Using Refs for Scroll Position
|
||||
|
||||
```typescript
|
||||
const scrollPosition = useRef({
|
||||
left: 0,
|
||||
top: 0
|
||||
});
|
||||
|
||||
// Capture position on scroll
|
||||
const handleScroll = useCallback(() => {
|
||||
if (tableContainerRef.current) {
|
||||
scrollPosition.current = {
|
||||
left: tableContainerRef.current.scrollLeft,
|
||||
top: tableContainerRef.current.scrollTop
|
||||
};
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Restore in useLayoutEffect
|
||||
useLayoutEffect(() => {
|
||||
const container = tableContainerRef.current;
|
||||
if (container) {
|
||||
const { left, top } = scrollPosition.current;
|
||||
if (left || top) {
|
||||
container.scrollLeft = left;
|
||||
container.scrollTop = top;
|
||||
}
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
Result: Scroll position was still lost during updates.
|
||||
|
||||
### 2. Multiple Restoration Attempts with Timeouts
|
||||
|
||||
```typescript
|
||||
// Multiple timeouts at different intervals
|
||||
setTimeout(() => {
|
||||
if (tableContainerRef.current) {
|
||||
tableContainerRef.current.scrollTop = savedPosition.top;
|
||||
tableContainerRef.current.scrollLeft = savedPosition.left;
|
||||
}
|
||||
}, 0);
|
||||
|
||||
setTimeout(() => {
|
||||
if (tableContainerRef.current) {
|
||||
tableContainerRef.current.scrollTop = savedPosition.top;
|
||||
tableContainerRef.current.scrollLeft = savedPosition.left;
|
||||
}
|
||||
}, 50);
|
||||
|
||||
// Additional timeouts at 100ms, 300ms
|
||||
```
|
||||
|
||||
Result: Still not reliable, scroll position would reset between timeouts or after all timeouts completed.
|
||||
|
||||
### 3. Using MutationObserver and ResizeObserver
|
||||
|
||||
```typescript
|
||||
// Create a mutation observer to detect DOM changes
|
||||
const mutationObserver = new MutationObserver(() => {
|
||||
if (shouldPreserveScroll) {
|
||||
restoreScrollPosition();
|
||||
}
|
||||
});
|
||||
|
||||
// Start observing the table for DOM changes
|
||||
mutationObserver.observe(scrollableContainer, {
|
||||
childList: true,
|
||||
subtree: true,
|
||||
attributes: true,
|
||||
attributeFilter: ['style', 'class']
|
||||
});
|
||||
|
||||
// Create a resize observer
|
||||
const resizeObserver = new ResizeObserver(() => {
|
||||
if (shouldPreserveScroll) {
|
||||
restoreScrollPosition();
|
||||
}
|
||||
});
|
||||
|
||||
// Observe the table container
|
||||
resizeObserver.observe(scrollableContainer);
|
||||
```
|
||||
|
||||
Result: Did not reliably maintain scroll position, and sometimes caused other rendering issues.
|
||||
|
||||
### 4. Recursive Restoration Approach
|
||||
|
||||
```typescript
|
||||
let attempts = 0;
|
||||
const maxAttempts = 5;
|
||||
|
||||
const restore = () => {
|
||||
if (tableContainerRef.current) {
|
||||
tableContainerRef.current.scrollTop = y;
|
||||
tableContainerRef.current.scrollLeft = x;
|
||||
|
||||
attempts++;
|
||||
if (attempts < maxAttempts) {
|
||||
setTimeout(restore, 50 * attempts);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
restore();
|
||||
```
|
||||
|
||||
Result: No improvement, scroll position still reset.
|
||||
|
||||
### 5. Using React State for Scroll Position
|
||||
|
||||
```typescript
|
||||
const [scrollPos, setScrollPos] = useState<{top: number; left: number}>({top: 0, left: 0});
|
||||
|
||||
// Track the scroll event
|
||||
useEffect(() => {
|
||||
const handleScroll = () => {
|
||||
if (scrollContainerRef.current) {
|
||||
setScrollPos({
|
||||
top: scrollContainerRef.current.scrollTop,
|
||||
left: scrollContainerRef.current.scrollLeft
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
// Add scroll listener...
|
||||
}, []);
|
||||
|
||||
// Restore scroll position
|
||||
useLayoutEffect(() => {
|
||||
const container = scrollContainerRef.current;
|
||||
const { top, left } = scrollPos;
|
||||
|
||||
if (top > 0 || left > 0) {
|
||||
requestAnimationFrame(() => {
|
||||
if (container) {
|
||||
container.scrollTop = top;
|
||||
container.scrollLeft = left;
|
||||
}
|
||||
});
|
||||
}
|
||||
}, [scrollPos, data]);
|
||||
```
|
||||
|
||||
Result: Caused the screen to shake violently when scrolling and did not preserve position.
|
||||
|
||||
### 6. Using Key Attribute for Stability
|
||||
|
||||
```typescript
|
||||
return (
|
||||
<div
|
||||
key="validation-table-container"
|
||||
ref={scrollContainerRef}
|
||||
className="overflow-auto max-h-[calc(100vh-300px)]"
|
||||
>
|
||||
{/* Table content */}
|
||||
</div>
|
||||
);
|
||||
```
|
||||
|
||||
Result: Did not resolve the issue and may have contributed to rendering instability.
|
||||
|
||||
### 7. Removing Scroll Management from Other Components
|
||||
|
||||
We removed scroll position management code from:
|
||||
- `useValidationState.tsx` (in the updateRow function)
|
||||
- `ValidationContainer.tsx` (in the enhancedUpdateRow function)
|
||||
|
||||
Result: This did not fix the issue either.
|
||||
|
||||
### 8. Simple Scroll Position Management with Event Listeners
|
||||
|
||||
```typescript
|
||||
// Create a ref to store scroll position
|
||||
const scrollPosition = useRef({ left: 0, top: 0 });
|
||||
const tableContainerRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
// Save scroll position when scrolling
|
||||
const handleScroll = useCallback(() => {
|
||||
if (tableContainerRef.current) {
|
||||
scrollPosition.current = {
|
||||
left: tableContainerRef.current.scrollLeft,
|
||||
top: tableContainerRef.current.scrollTop
|
||||
};
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Add scroll listener
|
||||
useEffect(() => {
|
||||
const container = tableContainerRef.current;
|
||||
if (container) {
|
||||
container.addEventListener('scroll', handleScroll);
|
||||
return () => container.removeEventListener('scroll', handleScroll);
|
||||
}
|
||||
}, [handleScroll]);
|
||||
|
||||
// Restore scroll position after data changes
|
||||
useLayoutEffect(() => {
|
||||
const container = tableContainerRef.current;
|
||||
if (container) {
|
||||
const { left, top } = scrollPosition.current;
|
||||
if (left > 0 || top > 0) {
|
||||
container.scrollLeft = left;
|
||||
container.scrollTop = top;
|
||||
}
|
||||
}
|
||||
}, [data]);
|
||||
```
|
||||
|
||||
Result: Still did not maintain scroll position during updates.
|
||||
|
||||
### 9. Memoized Scroll Container Component
|
||||
|
||||
```typescript
|
||||
// Create a stable scroll container that won't re-render with the table
|
||||
const ScrollContainer = React.memo(({ children }: { children: React.ReactNode }) => {
|
||||
const containerRef = useRef<HTMLDivElement>(null);
|
||||
const scrollPosition = useRef({ left: 0, top: 0 });
|
||||
|
||||
const handleScroll = useCallback(() => {
|
||||
if (containerRef.current) {
|
||||
scrollPosition.current = {
|
||||
left: containerRef.current.scrollLeft,
|
||||
top: containerRef.current.scrollTop
|
||||
};
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const container = containerRef.current;
|
||||
if (container) {
|
||||
// Set initial scroll position if it exists
|
||||
if (scrollPosition.current.left > 0 || scrollPosition.current.top > 0) {
|
||||
container.scrollLeft = scrollPosition.current.left;
|
||||
container.scrollTop = scrollPosition.current.top;
|
||||
}
|
||||
container.addEventListener('scroll', handleScroll);
|
||||
return () => container.removeEventListener('scroll', handleScroll);
|
||||
}
|
||||
}, [handleScroll]);
|
||||
|
||||
return (
|
||||
<div ref={containerRef} className="overflow-auto max-h-[calc(100vh-300px)]">
|
||||
{children}
|
||||
</div>
|
||||
);
|
||||
});
|
||||
```
|
||||
|
||||
Result: Still did not maintain scroll position during updates, even with a memoized container.
|
||||
|
||||
### 10. Using TanStack Table State Management
|
||||
|
||||
```typescript
|
||||
// Track scroll state in the table instance
|
||||
const [scrollState, setScrollState] = useState({ scrollLeft: 0, scrollTop: 0 });
|
||||
|
||||
const table = useReactTable({
|
||||
data,
|
||||
columns,
|
||||
getCoreRowModel: getCoreRowModel(),
|
||||
state: {
|
||||
rowSelection,
|
||||
// Include scroll position in table state
|
||||
scrollLeft: scrollState.scrollLeft,
|
||||
scrollTop: scrollState.scrollTop
|
||||
},
|
||||
onStateChange: (updater) => {
|
||||
if (typeof updater === 'function') {
|
||||
const newState = updater({
|
||||
rowSelection,
|
||||
scrollLeft: scrollState.scrollLeft,
|
||||
scrollTop: scrollState.scrollTop
|
||||
});
|
||||
if ('scrollLeft' in newState || 'scrollTop' in newState) {
|
||||
setScrollState({
|
||||
scrollLeft: newState.scrollLeft ?? scrollState.scrollLeft,
|
||||
scrollTop: newState.scrollTop ?? scrollState.scrollTop
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Handle scroll events
|
||||
const handleScroll = useCallback((event: React.UIEvent<HTMLDivElement>) => {
|
||||
const target = event.target as HTMLDivElement;
|
||||
setScrollState({
|
||||
scrollLeft: target.scrollLeft,
|
||||
scrollTop: target.scrollTop
|
||||
});
|
||||
}, []);
|
||||
|
||||
// Restore scroll position after updates
|
||||
useLayoutEffect(() => {
|
||||
if (tableContainerRef.current) {
|
||||
tableContainerRef.current.scrollLeft = scrollState.scrollLeft;
|
||||
tableContainerRef.current.scrollTop = scrollState.scrollTop;
|
||||
}
|
||||
}, [data, scrollState]);
|
||||
```
|
||||
|
||||
Result: Still did not maintain scroll position during updates, even with table state management.
|
||||
|
||||
### 11. Using CSS Sticky Positioning
|
||||
|
||||
```typescript
|
||||
return (
|
||||
<div className="relative max-h-[calc(100vh-300px)] overflow-auto">
|
||||
<Table>
|
||||
<TableHeader className="sticky top-0 z-10 bg-background">
|
||||
<TableRow>
|
||||
{table.getFlatHeaders().map((header) => (
|
||||
<TableHead
|
||||
key={header.id}
|
||||
style={{
|
||||
width: `${header.getSize()}px`,
|
||||
minWidth: `${header.getSize()}px`,
|
||||
position: 'sticky',
|
||||
top: 0,
|
||||
backgroundColor: 'inherit'
|
||||
}}
|
||||
>
|
||||
{/* Header content */}
|
||||
</TableHead>
|
||||
))}
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{/* Table body content */}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
);
|
||||
```
|
||||
|
||||
Result: Still did not maintain scroll position during updates, even with native CSS scrolling.
|
||||
|
||||
### 12. Optimized Memoization with Object.is
|
||||
|
||||
```typescript
|
||||
// Memoize data structures to prevent unnecessary re-renders
|
||||
const memoizedData = useMemo(() => data, [data]);
|
||||
const memoizedValidationErrors = useMemo(() => validationErrors, [validationErrors]);
|
||||
const memoizedValidatingCells = useMemo(() => validatingCells, [validatingCells]);
|
||||
const memoizedItemNumbers = useMemo(() => itemNumbers, [itemNumbers]);
|
||||
|
||||
// Use Object.is for more efficient comparisons
|
||||
export default React.memo(ValidationTable, (prev, next) => {
|
||||
if (!Object.is(prev.data.length, next.data.length)) return false;
|
||||
|
||||
if (prev.validationErrors.size !== next.validationErrors.size) return false;
|
||||
for (const [key, value] of prev.validationErrors) {
|
||||
if (!next.validationErrors.has(key)) return false;
|
||||
if (!Object.is(value, next.validationErrors.get(key))) return false;
|
||||
}
|
||||
|
||||
// ... more optimized comparisons ...
|
||||
});
|
||||
```
|
||||
|
||||
Result: Caused the page to crash with "TypeError: undefined has no properties" in the MemoizedCell component.
|
||||
|
||||
### 13. Simplified Component Structure
|
||||
|
||||
```typescript
|
||||
const ValidationTable = <T extends string>({
|
||||
data,
|
||||
fields,
|
||||
rowSelection,
|
||||
setRowSelection,
|
||||
updateRow,
|
||||
validationErrors,
|
||||
// ... other props
|
||||
}) => {
|
||||
const tableContainerRef = useRef<HTMLDivElement>(null);
|
||||
const lastScrollPosition = useRef({ left: 0, top: 0 });
|
||||
|
||||
// Simple scroll position management
|
||||
const handleScroll = useCallback(() => {
|
||||
if (tableContainerRef.current) {
|
||||
lastScrollPosition.current = {
|
||||
left: tableContainerRef.current.scrollLeft,
|
||||
top: tableContainerRef.current.scrollTop
|
||||
};
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const container = tableContainerRef.current;
|
||||
if (container) {
|
||||
container.addEventListener('scroll', handleScroll);
|
||||
return () => container.removeEventListener('scroll', handleScroll);
|
||||
}
|
||||
}, [handleScroll]);
|
||||
|
||||
useLayoutEffect(() => {
|
||||
const container = tableContainerRef.current;
|
||||
if (container) {
|
||||
const { left, top } = lastScrollPosition.current;
|
||||
if (left > 0 || top > 0) {
|
||||
requestAnimationFrame(() => {
|
||||
if (container) {
|
||||
container.scrollLeft = left;
|
||||
container.scrollTop = top;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}, [data]);
|
||||
|
||||
return (
|
||||
<div ref={tableContainerRef} className="overflow-auto max-h-[calc(100vh-300px)]">
|
||||
<Table>
|
||||
{/* ... table content ... */}
|
||||
<TableBody>
|
||||
{table.getRowModel().rows.map((row) => (
|
||||
<TableRow
|
||||
key={row.id}
|
||||
className={cn(
|
||||
"hover:bg-muted/50",
|
||||
row.getIsSelected() ? "bg-muted/50" : "",
|
||||
validationErrors.get(data.indexOf(row.original)) ? "bg-red-50/40" : ""
|
||||
)}
|
||||
>
|
||||
{/* ... row content ... */}
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
```
|
||||
|
||||
Result: Still did not maintain scroll position during updates. However, this implementation restored the subtle red highlight on rows with validation errors, which is a useful visual indicator that should be preserved in future attempts.
|
||||
|
||||
### 14. Portal-Based Scroll Container
|
||||
|
||||
```typescript
|
||||
// Create a stable container outside of React's control
|
||||
const createStableContainer = () => {
|
||||
const containerId = 'validation-table-container';
|
||||
let container = document.getElementById(containerId);
|
||||
|
||||
if (!container) {
|
||||
container = document.createElement('div');
|
||||
container.id = containerId;
|
||||
container.className = 'overflow-auto';
|
||||
container.style.maxHeight = 'calc(100vh - 300px)';
|
||||
document.body.appendChild(container);
|
||||
}
|
||||
|
||||
return container;
|
||||
};
|
||||
|
||||
const ValidationTable = <T extends string>({...props}) => {
|
||||
const [container] = useState(createStableContainer);
|
||||
const [mounted, setMounted] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
setMounted(true);
|
||||
return () => {
|
||||
if (container && container.parentNode) {
|
||||
container.parentNode.removeChild(container);
|
||||
}
|
||||
};
|
||||
}, [container]);
|
||||
|
||||
// ... table configuration ...
|
||||
|
||||
return createPortal(content, container);
|
||||
};
|
||||
```
|
||||
|
||||
Result: The table contents failed to render at all. The portal-based approach to maintain scroll position by moving the scroll container outside of React's control was unsuccessful.
|
||||
|
||||
## Current Understanding
|
||||
|
||||
The scroll position issue appears to be complex and likely stems from multiple factors:
|
||||
|
||||
1. React's virtual DOM reconciliation may be replacing the scroll container element during updates
|
||||
2. The table uses complex memo patterns with custom equality checks that may not be working as expected
|
||||
3. The data structure may be changing in ways that cause complete re-renders
|
||||
4. The component hierarchy (with EnhancedValidationTable wrapper) may be affecting DOM stability
|
||||
|
||||
## Next Steps to Consider
|
||||
|
||||
At this point, we have tried multiple approaches without success:
|
||||
1. Various scroll position management techniques
|
||||
2. Memoization and optimization strategies
|
||||
3. Different component structures
|
||||
4. Portal-based rendering
|
||||
|
||||
Given that none of these approaches have fully resolved the issue, it may be worth:
|
||||
1. Investigating if there are any parent component updates forcing re-renders
|
||||
2. Profiling the application to identify the exact timing of scroll position resets
|
||||
3. Considering if the current table implementation could be simplified
|
||||
4. Exploring if the data update patterns could be optimized to reduce re-renders
|
||||
|
||||
## Conclusion
|
||||
|
||||
The scroll position issue has proven resistant to multiple solution attempts. Each approach has either failed to maintain scroll position, introduced new issues, or in some cases (like the portal-based approach) prevented the table from rendering entirely. A deeper investigation into the component lifecycle and data flow may be necessary to identify the root cause.
|
||||
@@ -1,22 +0,0 @@
|
||||
module.exports = {
|
||||
apps: [
|
||||
{
|
||||
...commonSettings,
|
||||
name: 'new-auth-server',
|
||||
script: './inventory-server/auth/server.js',
|
||||
env: {
|
||||
NODE_ENV: 'production',
|
||||
AUTH_PORT: 3011,
|
||||
JWT_SECRET: process.env.JWT_SECRET
|
||||
},
|
||||
error_file: 'inventory-server/auth/logs/pm2/err.log',
|
||||
out_file: 'inventory-server/auth/logs/pm2/out.log',
|
||||
log_file: 'inventory-server/auth/logs/pm2/combined.log',
|
||||
env_production: {
|
||||
NODE_ENV: 'production',
|
||||
AUTH_PORT: 3011,
|
||||
JWT_SECRET: process.env.JWT_SECRET
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
@@ -1,41 +0,0 @@
|
||||
const bcrypt = require('bcrypt');
|
||||
const mysql = require('mysql2/promise');
|
||||
const readline = require('readline').createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
require('dotenv').config({ path: '../.env' });
|
||||
|
||||
const dbConfig = {
|
||||
host: process.env.DB_HOST,
|
||||
user: process.env.DB_USER,
|
||||
password: process.env.DB_PASSWORD,
|
||||
database: process.env.DB_NAME,
|
||||
};
|
||||
|
||||
async function addUser() {
|
||||
const username = await askQuestion('Enter username: ');
|
||||
const password = await askQuestion('Enter password: ');
|
||||
|
||||
const hashedPassword = await bcrypt.hash(password, 10);
|
||||
|
||||
const connection = await mysql.createConnection(dbConfig);
|
||||
|
||||
try {
|
||||
await connection.query('INSERT INTO users (username, password) VALUES (?, ?)', [username, hashedPassword]);
|
||||
console.log(`User ${username} added successfully.`);
|
||||
} catch (error) {
|
||||
console.error('Error adding user:', error);
|
||||
} finally {
|
||||
connection.end();
|
||||
readline.close();
|
||||
}
|
||||
}
|
||||
|
||||
function askQuestion(query) {
|
||||
return new Promise(resolve => readline.question(query, ans => {
|
||||
resolve(ans);
|
||||
}));
|
||||
}
|
||||
|
||||
addUser();
|
||||
1915
inventory-server/auth/package-lock.json
generated
1915
inventory-server/auth/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -1,21 +0,0 @@
|
||||
{
|
||||
"name": "auth-server",
|
||||
"version": "1.0.0",
|
||||
"description": "Authentication server for inventory management",
|
||||
"main": "server.js",
|
||||
"scripts": {
|
||||
"start": "node server.js",
|
||||
"dev": "nodemon server.js",
|
||||
"add_user": "node add_user.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"bcrypt": "^5.1.1",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^16.4.5",
|
||||
"express": "^4.18.2",
|
||||
"jsonwebtoken": "^9.0.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nodemon": "^3.1.0"
|
||||
}
|
||||
}
|
||||
@@ -1,6 +0,0 @@
|
||||
CREATE TABLE `users` (
|
||||
`id` INT AUTO_INCREMENT PRIMARY KEY,
|
||||
`username` VARCHAR(255) NOT NULL UNIQUE,
|
||||
`password` VARCHAR(255) NOT NULL,
|
||||
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
@@ -1,135 +1,171 @@
|
||||
require('dotenv').config({ path: '../.env' });
|
||||
const express = require('express');
|
||||
const cors = require('cors');
|
||||
const bcrypt = require('bcrypt');
|
||||
const jwt = require('jsonwebtoken');
|
||||
const cors = require('cors');
|
||||
const mysql = require('mysql2/promise');
|
||||
require('dotenv').config({ path: '../.env' });
|
||||
const { Pool } = require('pg');
|
||||
const morgan = require('morgan');
|
||||
const authRoutes = require('./routes');
|
||||
|
||||
// Log startup configuration
|
||||
console.log('Starting auth server with config:', {
|
||||
host: process.env.DB_HOST,
|
||||
user: process.env.DB_USER,
|
||||
database: process.env.DB_NAME,
|
||||
port: process.env.DB_PORT,
|
||||
auth_port: process.env.AUTH_PORT
|
||||
});
|
||||
|
||||
const app = express();
|
||||
const PORT = process.env.AUTH_PORT || 3011;
|
||||
const port = process.env.AUTH_PORT || 3011;
|
||||
|
||||
// Database configuration
|
||||
const dbConfig = {
|
||||
const pool = new Pool({
|
||||
host: process.env.DB_HOST,
|
||||
user: process.env.DB_USER,
|
||||
password: process.env.DB_PASSWORD,
|
||||
database: process.env.DB_NAME,
|
||||
};
|
||||
port: process.env.DB_PORT,
|
||||
});
|
||||
|
||||
// Create a connection pool
|
||||
const pool = mysql.createPool(dbConfig);
|
||||
// Make pool available globally
|
||||
global.pool = pool;
|
||||
|
||||
app.use(cors({
|
||||
origin: [
|
||||
'https://inventory.kent.pw',
|
||||
'http://localhost:5173',
|
||||
'http://127.0.0.1:5173',
|
||||
/^http:\/\/192\.168\.\d+\.\d+(:\d+)?$/,
|
||||
/^http:\/\/10\.\d+\.\d+\.\d+(:\d+)?$/
|
||||
],
|
||||
methods: ['GET', 'POST', 'OPTIONS'],
|
||||
allowedHeaders: ['Content-Type', 'Authorization', 'X-Requested-With'],
|
||||
credentials: true,
|
||||
exposedHeaders: ['set-cookie']
|
||||
}));
|
||||
// Middleware
|
||||
app.use(express.json());
|
||||
|
||||
// Debug middleware to log request details
|
||||
app.use((req, res, next) => {
|
||||
console.log('Request details:', {
|
||||
method: req.method,
|
||||
url: req.url,
|
||||
origin: req.get('Origin'),
|
||||
headers: req.headers,
|
||||
body: req.body,
|
||||
});
|
||||
next();
|
||||
});
|
||||
|
||||
// Registration endpoint
|
||||
app.post('/register', async (req, res) => {
|
||||
try {
|
||||
const { username, password } = req.body;
|
||||
const hashedPassword = await bcrypt.hash(password, 10);
|
||||
|
||||
const connection = await pool.getConnection();
|
||||
await connection.query('INSERT INTO users (username, password) VALUES (?, ?)', [username, hashedPassword]);
|
||||
connection.release();
|
||||
|
||||
res.status(201).json({ message: 'User registered successfully' });
|
||||
} catch (error) {
|
||||
console.error('Registration error:', error);
|
||||
res.status(500).json({ error: 'Registration failed' });
|
||||
}
|
||||
});
|
||||
app.use(morgan('combined'));
|
||||
app.use(cors({
|
||||
origin: ['http://localhost:5175', 'http://localhost:5174', 'https://inventory.kent.pw', 'https://tools.acherryontop.com', 'https://tools.acherryontop.com'],
|
||||
credentials: true
|
||||
}));
|
||||
|
||||
// Login endpoint
|
||||
app.post('/login', async (req, res) => {
|
||||
try {
|
||||
const { username, password } = req.body;
|
||||
console.log(`Login attempt for user: ${username}`);
|
||||
|
||||
const connection = await pool.getConnection();
|
||||
const [rows] = await connection.query(
|
||||
'SELECT * FROM users WHERE username = ?',
|
||||
[username],
|
||||
try {
|
||||
// Get user from database
|
||||
const result = await pool.query(
|
||||
'SELECT id, username, password, is_admin, is_active FROM users WHERE username = $1',
|
||||
[username]
|
||||
);
|
||||
connection.release();
|
||||
|
||||
if (rows.length === 1) {
|
||||
const user = rows[0];
|
||||
const passwordMatch = await bcrypt.compare(password, user.password);
|
||||
const user = result.rows[0];
|
||||
|
||||
if (passwordMatch) {
|
||||
console.log(`User ${username} authenticated successfully`);
|
||||
// Check if user exists and password is correct
|
||||
if (!user || !(await bcrypt.compare(password, user.password))) {
|
||||
return res.status(401).json({ error: 'Invalid username or password' });
|
||||
}
|
||||
|
||||
// Check if user is active
|
||||
if (!user.is_active) {
|
||||
return res.status(403).json({ error: 'Account is inactive' });
|
||||
}
|
||||
|
||||
// Update last login timestamp
|
||||
await pool.query(
|
||||
'UPDATE users SET last_login = CURRENT_TIMESTAMP WHERE id = $1',
|
||||
[user.id]
|
||||
);
|
||||
|
||||
// Generate JWT token
|
||||
const token = jwt.sign(
|
||||
{ username: user.username },
|
||||
{ userId: user.id, username: user.username },
|
||||
process.env.JWT_SECRET,
|
||||
{ expiresIn: '1h' },
|
||||
{ expiresIn: '24h' }
|
||||
);
|
||||
res.json({ token });
|
||||
} else {
|
||||
console.error(`Invalid password for user: ${username}`);
|
||||
res.status(401).json({ error: 'Invalid credentials' });
|
||||
}
|
||||
} else {
|
||||
console.error(`User not found: ${username}`);
|
||||
res.status(401).json({ error: 'Invalid credentials' });
|
||||
|
||||
// Get user permissions for the response
|
||||
const permissionsResult = await pool.query(`
|
||||
SELECT code
|
||||
FROM permissions p
|
||||
JOIN user_permissions up ON p.id = up.permission_id
|
||||
WHERE up.user_id = $1
|
||||
`, [user.id]);
|
||||
|
||||
const permissions = permissionsResult.rows.map(row => row.code);
|
||||
|
||||
res.json({
|
||||
token,
|
||||
user: {
|
||||
id: user.id,
|
||||
username: user.username,
|
||||
is_admin: user.is_admin,
|
||||
permissions: user.is_admin ? [] : permissions
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Login error:', error);
|
||||
res.status(500).json({ error: 'Login failed' });
|
||||
res.status(500).json({ error: 'Internal server error' });
|
||||
}
|
||||
});
|
||||
|
||||
// Protected endpoint example
|
||||
app.get('/protected', async (req, res) => {
|
||||
// User info endpoint
|
||||
app.get('/me', async (req, res) => {
|
||||
const authHeader = req.headers.authorization;
|
||||
if (!authHeader) {
|
||||
return res.status(401).json({ error: 'Unauthorized' });
|
||||
|
||||
if (!authHeader || !authHeader.startsWith('Bearer ')) {
|
||||
return res.status(401).json({ error: 'No token provided' });
|
||||
}
|
||||
|
||||
const token = authHeader.split(' ')[1];
|
||||
try {
|
||||
const token = authHeader.split(' ')[1];
|
||||
const decoded = jwt.verify(token, process.env.JWT_SECRET);
|
||||
|
||||
// Optionally, you can fetch the user from the database here
|
||||
// to verify that the user still exists or to get more user information
|
||||
const connection = await pool.getConnection();
|
||||
const [rows] = await connection.query('SELECT * FROM users WHERE username = ?', [decoded.username]);
|
||||
connection.release();
|
||||
// Get user details from database
|
||||
const userResult = await pool.query(
|
||||
'SELECT id, username, email, is_admin, rocket_chat_user_id, is_active FROM users WHERE id = $1',
|
||||
[decoded.userId]
|
||||
);
|
||||
|
||||
if (rows.length === 0) {
|
||||
return res.status(401).json({ error: 'User not found' });
|
||||
if (userResult.rows.length === 0) {
|
||||
return res.status(404).json({ error: 'User not found' });
|
||||
}
|
||||
|
||||
res.json({ message: 'Protected resource accessed', user: decoded });
|
||||
const user = userResult.rows[0];
|
||||
|
||||
// Get user permissions
|
||||
let permissions = [];
|
||||
if (!user.is_admin) {
|
||||
const permissionsResult = await pool.query(`
|
||||
SELECT code
|
||||
FROM permissions p
|
||||
JOIN user_permissions up ON p.id = up.permission_id
|
||||
WHERE up.user_id = $1
|
||||
`, [user.id]);
|
||||
|
||||
permissions = permissionsResult.rows.map(row => row.code);
|
||||
}
|
||||
|
||||
res.json({
|
||||
id: user.id,
|
||||
username: user.username,
|
||||
email: user.email,
|
||||
rocket_chat_user_id: user.rocket_chat_user_id,
|
||||
is_admin: user.is_admin,
|
||||
permissions: permissions
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Protected endpoint error:', error);
|
||||
res.status(403).json({ error: 'Invalid token' });
|
||||
console.error('Token verification error:', error);
|
||||
res.status(401).json({ error: 'Invalid token' });
|
||||
}
|
||||
});
|
||||
|
||||
app.listen(PORT, "0.0.0.0", () => {
|
||||
console.log(`Auth server running on port ${PORT}`);
|
||||
// Mount all routes from routes.js
|
||||
app.use('/', authRoutes);
|
||||
|
||||
// Health check endpoint
|
||||
app.get('/health', (req, res) => {
|
||||
res.json({ status: 'healthy' });
|
||||
});
|
||||
|
||||
// Error handling middleware
|
||||
app.use((err, req, res, next) => {
|
||||
console.error(err.stack);
|
||||
res.status(500).json({ error: 'Something broke!' });
|
||||
});
|
||||
|
||||
// Start server
|
||||
app.listen(port, () => {
|
||||
console.log(`Auth server running on port ${port}`);
|
||||
});
|
||||
83
inventory-server/chat/server.js
Normal file
83
inventory-server/chat/server.js
Normal file
@@ -0,0 +1,83 @@
|
||||
require('dotenv').config({ path: '../.env' });
|
||||
const express = require('express');
|
||||
const cors = require('cors');
|
||||
const { Pool } = require('pg');
|
||||
const morgan = require('morgan');
|
||||
const chatRoutes = require('./routes');
|
||||
|
||||
// Log startup configuration
|
||||
console.log('Starting chat server with config:', {
|
||||
host: process.env.CHAT_DB_HOST,
|
||||
user: process.env.CHAT_DB_USER,
|
||||
database: process.env.CHAT_DB_NAME || 'rocketchat_converted',
|
||||
port: process.env.CHAT_DB_PORT,
|
||||
chat_port: process.env.CHAT_PORT || 3014
|
||||
});
|
||||
|
||||
const app = express();
|
||||
const port = process.env.CHAT_PORT || 3014;
|
||||
|
||||
// Database configuration for rocketchat_converted database
|
||||
const pool = new Pool({
|
||||
host: process.env.CHAT_DB_HOST,
|
||||
user: process.env.CHAT_DB_USER,
|
||||
password: process.env.CHAT_DB_PASSWORD,
|
||||
database: process.env.CHAT_DB_NAME || 'rocketchat_converted',
|
||||
port: process.env.CHAT_DB_PORT,
|
||||
});
|
||||
|
||||
// Make pool available globally
|
||||
global.pool = pool;
|
||||
|
||||
// Middleware
|
||||
app.use(express.json());
|
||||
app.use(morgan('combined'));
|
||||
app.use(cors({
|
||||
origin: ['http://localhost:5175', 'http://localhost:5174', 'https://inventory.kent.pw', 'https://tools.acherryontop.com', 'https://tools.acherryontop.com'],
|
||||
credentials: true
|
||||
}));
|
||||
|
||||
// Test database connection endpoint
|
||||
app.get('/test-db', async (req, res) => {
|
||||
try {
|
||||
const result = await pool.query('SELECT COUNT(*) as user_count FROM users WHERE active = true');
|
||||
const messageResult = await pool.query('SELECT COUNT(*) as message_count FROM message');
|
||||
const roomResult = await pool.query('SELECT COUNT(*) as room_count FROM room');
|
||||
|
||||
res.json({
|
||||
status: 'success',
|
||||
database: 'rocketchat_converted',
|
||||
stats: {
|
||||
active_users: parseInt(result.rows[0].user_count),
|
||||
total_messages: parseInt(messageResult.rows[0].message_count),
|
||||
total_rooms: parseInt(roomResult.rows[0].room_count)
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Database test error:', error);
|
||||
res.status(500).json({
|
||||
status: 'error',
|
||||
error: 'Database connection failed',
|
||||
details: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Mount all routes from routes.js
|
||||
app.use('/', chatRoutes);
|
||||
|
||||
// Health check endpoint
|
||||
app.get('/health', (req, res) => {
|
||||
res.json({ status: 'healthy' });
|
||||
});
|
||||
|
||||
// Error handling middleware
|
||||
app.use((err, req, res, next) => {
|
||||
console.error(err.stack);
|
||||
res.status(500).json({ error: 'Something broke!' });
|
||||
});
|
||||
|
||||
// Start server
|
||||
app.listen(port, () => {
|
||||
console.log(`Chat server running on port ${port}`);
|
||||
});
|
||||
973
inventory-server/dashboard/acot-server/routes/events.js
Normal file
973
inventory-server/dashboard/acot-server/routes/events.js
Normal file
@@ -0,0 +1,973 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const { getDbConnection, getPoolStatus } = require('../db/connection');
|
||||
const { getTimeRangeConditions, formatBusinessDate, getBusinessDayBounds } = require('../utils/timeUtils');
|
||||
|
||||
// Image URL generation utility
|
||||
const getImageUrls = (pid, iid = 1) => {
|
||||
const imageUrlBase = 'https://sbing.com/i/products/0000/';
|
||||
const paddedPid = pid.toString().padStart(6, '0');
|
||||
const prefix = paddedPid.slice(0, 3);
|
||||
const basePath = `${imageUrlBase}${prefix}/${pid}`;
|
||||
return {
|
||||
image: `${basePath}-t-${iid}.jpg`,
|
||||
image_175: `${basePath}-175x175-${iid}.jpg`,
|
||||
image_full: `${basePath}-o-${iid}.jpg`,
|
||||
ImgThumb: `${basePath}-175x175-${iid}.jpg` // For ProductGrid component
|
||||
};
|
||||
};
|
||||
|
||||
// Main stats endpoint - replaces /api/klaviyo/events/stats
|
||||
router.get('/stats', async (req, res) => {
|
||||
const startTime = Date.now();
|
||||
console.log(`[STATS] Starting request for timeRange: ${req.query.timeRange}`);
|
||||
|
||||
// Set a timeout for the entire operation
|
||||
const timeoutPromise = new Promise((_, reject) => {
|
||||
setTimeout(() => reject(new Error('Request timeout after 15 seconds')), 15000);
|
||||
});
|
||||
|
||||
try {
|
||||
const mainOperation = async () => {
|
||||
const { timeRange, startDate, endDate } = req.query;
|
||||
console.log(`[STATS] Getting DB connection...`);
|
||||
const { connection, release } = await getDbConnection();
|
||||
console.log(`[STATS] DB connection obtained in ${Date.now() - startTime}ms`);
|
||||
|
||||
const { whereClause, params, dateRange } = getTimeRangeConditions(timeRange, startDate, endDate);
|
||||
|
||||
// Main order stats query
|
||||
const mainStatsQuery = `
|
||||
SELECT
|
||||
COUNT(*) as orderCount,
|
||||
SUM(summary_total) as revenue,
|
||||
SUM(stats_prod_pieces) as itemCount,
|
||||
AVG(summary_total) as averageOrderValue,
|
||||
AVG(stats_prod_pieces) as averageItemsPerOrder,
|
||||
SUM(CASE WHEN stats_waiting_preorder > 0 THEN 1 ELSE 0 END) as preOrderCount,
|
||||
SUM(CASE WHEN ship_method_selected = 'localpickup' THEN 1 ELSE 0 END) as localPickupCount,
|
||||
SUM(CASE WHEN ship_method_selected = 'holdit' THEN 1 ELSE 0 END) as onHoldCount,
|
||||
SUM(CASE WHEN order_status IN (100, 92) THEN 1 ELSE 0 END) as shippedCount,
|
||||
SUM(CASE WHEN order_status = 15 THEN 1 ELSE 0 END) as cancelledCount,
|
||||
SUM(CASE WHEN order_status = 15 THEN summary_total ELSE 0 END) as cancelledTotal
|
||||
FROM _order
|
||||
WHERE order_status > 15 AND ${whereClause}
|
||||
`;
|
||||
|
||||
const [mainStats] = await connection.execute(mainStatsQuery, params);
|
||||
const stats = mainStats[0];
|
||||
|
||||
// Refunds query
|
||||
const refundsQuery = `
|
||||
SELECT
|
||||
COUNT(*) as refundCount,
|
||||
ABS(SUM(payment_amount)) as refundTotal
|
||||
FROM order_payment op
|
||||
JOIN _order o ON op.order_id = o.order_id
|
||||
WHERE payment_amount < 0 AND o.order_status > 15 AND ${whereClause.replace('date_placed', 'o.date_placed')}
|
||||
`;
|
||||
|
||||
const [refundStats] = await connection.execute(refundsQuery, params);
|
||||
|
||||
// Best revenue day query
|
||||
const bestDayQuery = `
|
||||
SELECT
|
||||
DATE(date_placed) as date,
|
||||
SUM(summary_total) as revenue,
|
||||
COUNT(*) as orders
|
||||
FROM _order
|
||||
WHERE order_status > 15 AND ${whereClause}
|
||||
GROUP BY DATE(date_placed)
|
||||
ORDER BY revenue DESC
|
||||
LIMIT 1
|
||||
`;
|
||||
|
||||
const [bestDayResult] = await connection.execute(bestDayQuery, params);
|
||||
|
||||
// Peak hour query (for single day periods)
|
||||
let peakHour = null;
|
||||
if (['today', 'yesterday'].includes(timeRange)) {
|
||||
const peakHourQuery = `
|
||||
SELECT
|
||||
HOUR(date_placed) as hour,
|
||||
COUNT(*) as count
|
||||
FROM _order
|
||||
WHERE order_status > 15 AND ${whereClause}
|
||||
GROUP BY HOUR(date_placed)
|
||||
ORDER BY count DESC
|
||||
LIMIT 1
|
||||
`;
|
||||
|
||||
const [peakHourResult] = await connection.execute(peakHourQuery, params);
|
||||
if (peakHourResult.length > 0) {
|
||||
const hour = peakHourResult[0].hour;
|
||||
const date = new Date();
|
||||
date.setHours(hour, 0, 0);
|
||||
peakHour = {
|
||||
hour,
|
||||
count: peakHourResult[0].count,
|
||||
displayHour: date.toLocaleString("en-US", { hour: "numeric", hour12: true })
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Brands and categories query - simplified for now since we don't have the category tables
|
||||
// We'll use a simple approach without company table for now
|
||||
const brandsQuery = `
|
||||
SELECT
|
||||
'Various Brands' as brandName,
|
||||
COUNT(DISTINCT oi.order_id) as orderCount,
|
||||
SUM(oi.qty_ordered) as itemCount,
|
||||
SUM(oi.qty_ordered * oi.prod_price) as revenue
|
||||
FROM order_items oi
|
||||
JOIN _order o ON oi.order_id = o.order_id
|
||||
JOIN products p ON oi.prod_pid = p.pid
|
||||
WHERE o.order_status > 15 AND ${whereClause.replace('date_placed', 'o.date_placed')}
|
||||
HAVING revenue > 0
|
||||
`;
|
||||
|
||||
const [brandsResult] = await connection.execute(brandsQuery, params);
|
||||
|
||||
// For categories, we'll use a simplified approach
|
||||
const categoriesQuery = `
|
||||
SELECT
|
||||
'General' as categoryName,
|
||||
COUNT(DISTINCT oi.order_id) as orderCount,
|
||||
SUM(oi.qty_ordered) as itemCount,
|
||||
SUM(oi.qty_ordered * oi.prod_price) as revenue
|
||||
FROM order_items oi
|
||||
JOIN _order o ON oi.order_id = o.order_id
|
||||
JOIN products p ON oi.prod_pid = p.pid
|
||||
WHERE o.order_status > 15 AND ${whereClause.replace('date_placed', 'o.date_placed')}
|
||||
HAVING revenue > 0
|
||||
`;
|
||||
|
||||
const [categoriesResult] = await connection.execute(categoriesQuery, params);
|
||||
|
||||
// Shipping locations query
|
||||
const shippingQuery = `
|
||||
SELECT
|
||||
ship_country,
|
||||
ship_state,
|
||||
ship_method_selected,
|
||||
COUNT(*) as count
|
||||
FROM _order
|
||||
WHERE order_status IN (100, 92) AND ${whereClause}
|
||||
GROUP BY ship_country, ship_state, ship_method_selected
|
||||
`;
|
||||
|
||||
const [shippingResult] = await connection.execute(shippingQuery, params);
|
||||
|
||||
// Process shipping data
|
||||
const shippingStats = processShippingData(shippingResult, stats.shippedCount);
|
||||
|
||||
// Order value range query
|
||||
const orderRangeQuery = `
|
||||
SELECT
|
||||
MIN(summary_total) as smallest,
|
||||
MAX(summary_total) as largest
|
||||
FROM _order
|
||||
WHERE order_status > 15 AND ${whereClause}
|
||||
`;
|
||||
|
||||
const [orderRangeResult] = await connection.execute(orderRangeQuery, params);
|
||||
|
||||
// Calculate period progress for incomplete periods
|
||||
let periodProgress = 100;
|
||||
if (['today', 'thisWeek', 'thisMonth'].includes(timeRange)) {
|
||||
periodProgress = calculatePeriodProgress(timeRange);
|
||||
}
|
||||
|
||||
// Previous period comparison data
|
||||
const prevPeriodData = await getPreviousPeriodData(connection, timeRange, startDate, endDate);
|
||||
|
||||
const response = {
|
||||
timeRange: dateRange,
|
||||
stats: {
|
||||
revenue: parseFloat(stats.revenue || 0),
|
||||
orderCount: parseInt(stats.orderCount || 0),
|
||||
itemCount: parseInt(stats.itemCount || 0),
|
||||
averageOrderValue: parseFloat(stats.averageOrderValue || 0),
|
||||
averageItemsPerOrder: parseFloat(stats.averageItemsPerOrder || 0),
|
||||
|
||||
// Order types
|
||||
orderTypes: {
|
||||
preOrders: {
|
||||
count: parseInt(stats.preOrderCount || 0),
|
||||
percentage: stats.orderCount > 0 ? (stats.preOrderCount / stats.orderCount) * 100 : 0
|
||||
},
|
||||
localPickup: {
|
||||
count: parseInt(stats.localPickupCount || 0),
|
||||
percentage: stats.orderCount > 0 ? (stats.localPickupCount / stats.orderCount) * 100 : 0
|
||||
},
|
||||
heldItems: {
|
||||
count: parseInt(stats.onHoldCount || 0),
|
||||
percentage: stats.orderCount > 0 ? (stats.onHoldCount / stats.orderCount) * 100 : 0
|
||||
}
|
||||
},
|
||||
|
||||
// Shipping
|
||||
shipping: {
|
||||
shippedCount: parseInt(stats.shippedCount || 0),
|
||||
locations: shippingStats.locations,
|
||||
methodStats: shippingStats.methods
|
||||
},
|
||||
|
||||
// Brands and categories
|
||||
brands: {
|
||||
total: brandsResult.length,
|
||||
list: brandsResult.slice(0, 50).map(brand => ({
|
||||
name: brand.brandName,
|
||||
count: parseInt(brand.itemCount),
|
||||
revenue: parseFloat(brand.revenue)
|
||||
}))
|
||||
},
|
||||
|
||||
categories: {
|
||||
total: categoriesResult.length,
|
||||
list: categoriesResult.slice(0, 50).map(category => ({
|
||||
name: category.categoryName,
|
||||
count: parseInt(category.itemCount),
|
||||
revenue: parseFloat(category.revenue)
|
||||
}))
|
||||
},
|
||||
|
||||
// Refunds and cancellations
|
||||
refunds: {
|
||||
total: parseFloat(refundStats[0]?.refundTotal || 0),
|
||||
count: parseInt(refundStats[0]?.refundCount || 0)
|
||||
},
|
||||
|
||||
canceledOrders: {
|
||||
total: parseFloat(stats.cancelledTotal || 0),
|
||||
count: parseInt(stats.cancelledCount || 0)
|
||||
},
|
||||
|
||||
// Best day
|
||||
bestRevenueDay: bestDayResult.length > 0 ? {
|
||||
amount: parseFloat(bestDayResult[0].revenue),
|
||||
displayDate: bestDayResult[0].date,
|
||||
orders: parseInt(bestDayResult[0].orders)
|
||||
} : null,
|
||||
|
||||
// Peak hour (for single days)
|
||||
peakOrderHour: peakHour,
|
||||
|
||||
// Order value range
|
||||
orderValueRange: orderRangeResult.length > 0 ? {
|
||||
smallest: parseFloat(orderRangeResult[0].smallest || 0),
|
||||
largest: parseFloat(orderRangeResult[0].largest || 0)
|
||||
} : { smallest: 0, largest: 0 },
|
||||
|
||||
// Period progress and projections
|
||||
periodProgress,
|
||||
projectedRevenue: periodProgress < 100 ? (stats.revenue / (periodProgress / 100)) : stats.revenue,
|
||||
|
||||
// Previous period comparison
|
||||
prevPeriodRevenue: prevPeriodData.revenue,
|
||||
prevPeriodOrders: prevPeriodData.orderCount,
|
||||
prevPeriodAOV: prevPeriodData.averageOrderValue
|
||||
}
|
||||
};
|
||||
|
||||
return { response, release };
|
||||
};
|
||||
|
||||
// Race between the main operation and timeout
|
||||
let result;
|
||||
try {
|
||||
result = await Promise.race([mainOperation(), timeoutPromise]);
|
||||
} catch (error) {
|
||||
// If it's a timeout, we don't have a release function to call
|
||||
if (error.message.includes('timeout')) {
|
||||
console.log(`[STATS] Request timed out in ${Date.now() - startTime}ms`);
|
||||
throw error;
|
||||
}
|
||||
// For other errors, re-throw
|
||||
throw error;
|
||||
}
|
||||
|
||||
const { response, release } = result;
|
||||
|
||||
// Release connection back to pool
|
||||
if (release) release();
|
||||
|
||||
console.log(`[STATS] Request completed in ${Date.now() - startTime}ms`);
|
||||
res.json(response);
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error in /stats:', error);
|
||||
console.log(`[STATS] Request failed in ${Date.now() - startTime}ms`);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Daily details endpoint - replaces /api/klaviyo/events/stats/details
|
||||
router.get('/stats/details', async (req, res) => {
|
||||
let release;
|
||||
try {
|
||||
const { timeRange, startDate, endDate, metric, daily } = req.query;
|
||||
const { connection, release: releaseConn } = await getDbConnection();
|
||||
release = releaseConn;
|
||||
|
||||
const { whereClause, params } = getTimeRangeConditions(timeRange, startDate, endDate);
|
||||
|
||||
// Daily breakdown query
|
||||
const dailyQuery = `
|
||||
SELECT
|
||||
DATE(date_placed) as date,
|
||||
COUNT(*) as orders,
|
||||
SUM(summary_total) as revenue,
|
||||
AVG(summary_total) as averageOrderValue,
|
||||
SUM(stats_prod_pieces) as itemCount
|
||||
FROM _order
|
||||
WHERE order_status > 15 AND ${whereClause}
|
||||
GROUP BY DATE(date_placed)
|
||||
ORDER BY DATE(date_placed)
|
||||
`;
|
||||
|
||||
const [dailyResults] = await connection.execute(dailyQuery, params);
|
||||
|
||||
// Get previous period data using the same logic as main stats endpoint
|
||||
let prevWhereClause, prevParams;
|
||||
|
||||
if (timeRange && timeRange !== 'custom') {
|
||||
const prevTimeRange = getPreviousTimeRange(timeRange);
|
||||
const result = getTimeRangeConditions(prevTimeRange);
|
||||
prevWhereClause = result.whereClause;
|
||||
prevParams = result.params;
|
||||
} else {
|
||||
// Custom date range - go back by the same duration
|
||||
const start = new Date(startDate);
|
||||
const end = new Date(endDate);
|
||||
const duration = end.getTime() - start.getTime();
|
||||
|
||||
const prevEnd = new Date(start.getTime() - 1);
|
||||
const prevStart = new Date(prevEnd.getTime() - duration);
|
||||
|
||||
prevWhereClause = 'date_placed >= ? AND date_placed <= ?';
|
||||
prevParams = [prevStart.toISOString(), prevEnd.toISOString()];
|
||||
}
|
||||
|
||||
// Get previous period daily data
|
||||
const prevQuery = `
|
||||
SELECT
|
||||
DATE(date_placed) as date,
|
||||
COUNT(*) as prevOrders,
|
||||
SUM(summary_total) as prevRevenue,
|
||||
AVG(summary_total) as prevAvgOrderValue
|
||||
FROM _order
|
||||
WHERE order_status > 15 AND ${prevWhereClause}
|
||||
GROUP BY DATE(date_placed)
|
||||
`;
|
||||
|
||||
const [prevResults] = await connection.execute(prevQuery, prevParams);
|
||||
|
||||
// Create a map for quick lookup of previous period data
|
||||
const prevMap = new Map();
|
||||
prevResults.forEach(prev => {
|
||||
const key = new Date(prev.date).toISOString().split('T')[0];
|
||||
prevMap.set(key, prev);
|
||||
});
|
||||
|
||||
// For period-to-period comparison, we need to map days by relative position
|
||||
// since dates won't match exactly (e.g., current week vs previous week)
|
||||
const dailyArray = dailyResults.map(day => ({
|
||||
timestamp: day.date,
|
||||
date: day.date,
|
||||
orders: parseInt(day.orders),
|
||||
revenue: parseFloat(day.revenue),
|
||||
averageOrderValue: parseFloat(day.averageOrderValue || 0),
|
||||
itemCount: parseInt(day.itemCount)
|
||||
}));
|
||||
|
||||
const prevArray = prevResults.map(day => ({
|
||||
orders: parseInt(day.prevOrders),
|
||||
revenue: parseFloat(day.prevRevenue),
|
||||
averageOrderValue: parseFloat(day.prevAvgOrderValue || 0)
|
||||
}));
|
||||
|
||||
// Combine current and previous period data by matching relative positions
|
||||
const statsWithComparison = dailyArray.map((day, index) => {
|
||||
const prev = prevArray[index] || { orders: 0, revenue: 0, averageOrderValue: 0 };
|
||||
|
||||
return {
|
||||
...day,
|
||||
prevOrders: prev.orders,
|
||||
prevRevenue: prev.revenue,
|
||||
prevAvgOrderValue: prev.averageOrderValue
|
||||
};
|
||||
});
|
||||
|
||||
res.json({ stats: statsWithComparison });
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error in /stats/details:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
} finally {
|
||||
// Release connection back to pool
|
||||
if (release) release();
|
||||
}
|
||||
});
|
||||
|
||||
// Financial performance endpoint
|
||||
router.get('/financials', async (req, res) => {
|
||||
let release;
|
||||
try {
|
||||
const { timeRange, startDate, endDate } = req.query;
|
||||
const { connection, release: releaseConn } = await getDbConnection();
|
||||
release = releaseConn;
|
||||
|
||||
const { whereClause, params, dateRange } = getTimeRangeConditions(timeRange, startDate, endDate);
|
||||
const financialWhere = whereClause.replace(/date_placed/g, 'DATE_SUB(date_change, INTERVAL 1 HOUR)');
|
||||
|
||||
const [totalsRows] = await connection.execute(
|
||||
buildFinancialTotalsQuery(financialWhere),
|
||||
params
|
||||
);
|
||||
|
||||
const totals = normalizeFinancialTotals(totalsRows[0]);
|
||||
|
||||
const [trendRows] = await connection.execute(
|
||||
buildFinancialTrendQuery(financialWhere),
|
||||
params
|
||||
);
|
||||
|
||||
const trend = trendRows.map(normalizeFinancialTrendRow);
|
||||
|
||||
let previousTotals = null;
|
||||
let comparison = null;
|
||||
|
||||
const previousRange = getPreviousPeriodRange(timeRange, startDate, endDate);
|
||||
if (previousRange) {
|
||||
const prevWhere = previousRange.whereClause.replace(/date_placed/g, 'DATE_SUB(date_change, INTERVAL 1 HOUR)');
|
||||
const [previousRows] = await connection.execute(
|
||||
buildFinancialTotalsQuery(prevWhere),
|
||||
previousRange.params
|
||||
);
|
||||
previousTotals = normalizeFinancialTotals(previousRows[0]);
|
||||
comparison = {
|
||||
grossSales: calculateComparison(totals.grossSales, previousTotals.grossSales),
|
||||
refunds: calculateComparison(totals.refunds, previousTotals.refunds),
|
||||
taxCollected: calculateComparison(totals.taxCollected, previousTotals.taxCollected),
|
||||
discounts: calculateComparison(totals.discounts, previousTotals.discounts),
|
||||
cogs: calculateComparison(totals.cogs, previousTotals.cogs),
|
||||
income: calculateComparison(totals.income, previousTotals.income),
|
||||
profit: calculateComparison(totals.profit, previousTotals.profit),
|
||||
margin: calculateComparison(totals.margin, previousTotals.margin),
|
||||
};
|
||||
}
|
||||
|
||||
res.json({
|
||||
dateRange,
|
||||
totals,
|
||||
previousTotals,
|
||||
comparison,
|
||||
trend,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error in /financials:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
} finally {
|
||||
if (release) release();
|
||||
}
|
||||
});
|
||||
|
||||
// Products endpoint - replaces /api/klaviyo/events/products
|
||||
router.get('/products', async (req, res) => {
|
||||
let release;
|
||||
try {
|
||||
const { timeRange, startDate, endDate } = req.query;
|
||||
const { connection, release: releaseConn } = await getDbConnection();
|
||||
release = releaseConn;
|
||||
|
||||
const { whereClause, params } = getTimeRangeConditions(timeRange, startDate, endDate);
|
||||
|
||||
const productsQuery = `
|
||||
SELECT
|
||||
p.pid,
|
||||
p.description as name,
|
||||
SUM(oi.qty_ordered) as totalQuantity,
|
||||
SUM(oi.qty_ordered * oi.prod_price) as totalRevenue,
|
||||
COUNT(DISTINCT oi.order_id) as orderCount,
|
||||
(SELECT pi.iid FROM product_images pi WHERE pi.pid = p.pid AND pi.order = 255 LIMIT 1) as primary_iid
|
||||
FROM order_items oi
|
||||
JOIN _order o ON oi.order_id = o.order_id
|
||||
JOIN products p ON oi.prod_pid = p.pid
|
||||
WHERE o.order_status > 15 AND ${whereClause.replace('date_placed', 'o.date_placed')}
|
||||
GROUP BY p.pid, p.description
|
||||
ORDER BY totalRevenue DESC
|
||||
LIMIT 500
|
||||
`;
|
||||
|
||||
const [productsResult] = await connection.execute(productsQuery, params);
|
||||
|
||||
// Add image URLs to each product
|
||||
const productsWithImages = productsResult.map(product => {
|
||||
const imageUrls = getImageUrls(product.pid, product.primary_iid || 1);
|
||||
return {
|
||||
id: product.pid,
|
||||
name: product.name,
|
||||
totalQuantity: parseInt(product.totalQuantity),
|
||||
totalRevenue: parseFloat(product.totalRevenue),
|
||||
orderCount: parseInt(product.orderCount),
|
||||
...imageUrls
|
||||
};
|
||||
});
|
||||
|
||||
res.json({
|
||||
stats: {
|
||||
products: {
|
||||
total: productsWithImages.length,
|
||||
list: productsWithImages
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error in /products:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
} finally {
|
||||
// Release connection back to pool
|
||||
if (release) release();
|
||||
}
|
||||
});
|
||||
|
||||
// Projection endpoint - replaces /api/klaviyo/events/projection
|
||||
router.get('/projection', async (req, res) => {
|
||||
let release;
|
||||
try {
|
||||
const { timeRange, startDate, endDate } = req.query;
|
||||
|
||||
// Only provide projections for incomplete periods
|
||||
if (!['today', 'thisWeek', 'thisMonth'].includes(timeRange)) {
|
||||
return res.json({ projectedRevenue: 0, confidence: 0 });
|
||||
}
|
||||
|
||||
const { connection, release: releaseConn } = await getDbConnection();
|
||||
release = releaseConn;
|
||||
|
||||
// Get current period data
|
||||
const { whereClause, params } = getTimeRangeConditions(timeRange, startDate, endDate);
|
||||
|
||||
const currentQuery = `
|
||||
SELECT
|
||||
SUM(summary_total) as currentRevenue,
|
||||
COUNT(*) as currentOrders
|
||||
FROM _order
|
||||
WHERE order_status > 15 AND ${whereClause}
|
||||
`;
|
||||
|
||||
const [currentResult] = await connection.execute(currentQuery, params);
|
||||
const current = currentResult[0];
|
||||
|
||||
// Get historical data for the same period type
|
||||
const historicalQuery = await getHistoricalProjectionData(connection, timeRange);
|
||||
|
||||
// Calculate projection based on current progress and historical patterns
|
||||
const periodProgress = calculatePeriodProgress(timeRange);
|
||||
const projection = calculateSmartProjection(
|
||||
parseFloat(current.currentRevenue || 0),
|
||||
parseInt(current.currentOrders || 0),
|
||||
periodProgress,
|
||||
historicalQuery
|
||||
);
|
||||
|
||||
res.json(projection);
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error in /projection:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
} finally {
|
||||
// Release connection back to pool
|
||||
if (release) release();
|
||||
}
|
||||
});
|
||||
|
||||
// Debug endpoint to check connection pool status
|
||||
router.get('/debug/pool', (req, res) => {
|
||||
res.json(getPoolStatus());
|
||||
});
|
||||
|
||||
// Health check endpoint
|
||||
router.get('/health', async (req, res) => {
|
||||
try {
|
||||
const { connection, release } = await getDbConnection();
|
||||
|
||||
// Simple query to test connection
|
||||
const [result] = await connection.execute('SELECT 1 as test');
|
||||
release();
|
||||
|
||||
res.json({
|
||||
status: 'healthy',
|
||||
timestamp: new Date().toISOString(),
|
||||
pool: getPoolStatus(),
|
||||
dbTest: result[0]
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(500).json({
|
||||
status: 'unhealthy',
|
||||
error: error.message,
|
||||
timestamp: new Date().toISOString(),
|
||||
pool: getPoolStatus()
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Helper functions
|
||||
function processShippingData(shippingResult, totalShipped) {
|
||||
const countries = {};
|
||||
const states = {};
|
||||
const methods = {};
|
||||
|
||||
shippingResult.forEach(row => {
|
||||
// Countries
|
||||
if (row.ship_country) {
|
||||
countries[row.ship_country] = (countries[row.ship_country] || 0) + row.count;
|
||||
}
|
||||
|
||||
// States
|
||||
if (row.ship_state) {
|
||||
states[row.ship_state] = (states[row.ship_state] || 0) + row.count;
|
||||
}
|
||||
|
||||
// Methods
|
||||
if (row.ship_method_selected) {
|
||||
methods[row.ship_method_selected] = (methods[row.ship_method_selected] || 0) + row.count;
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
locations: {
|
||||
total: totalShipped,
|
||||
byCountry: Object.entries(countries)
|
||||
.map(([country, count]) => ({
|
||||
country,
|
||||
count,
|
||||
percentage: (count / totalShipped) * 100
|
||||
}))
|
||||
.sort((a, b) => b.count - a.count),
|
||||
byState: Object.entries(states)
|
||||
.map(([state, count]) => ({
|
||||
state,
|
||||
count,
|
||||
percentage: (count / totalShipped) * 100
|
||||
}))
|
||||
.sort((a, b) => b.count - a.count)
|
||||
},
|
||||
methods: Object.entries(methods)
|
||||
.map(([name, value]) => ({ name, value }))
|
||||
.sort((a, b) => b.value - a.value)
|
||||
};
|
||||
}
|
||||
|
||||
function calculatePeriodProgress(timeRange) {
|
||||
const now = new Date();
|
||||
const easternTime = new Date(now.getTime() - (5 * 60 * 60 * 1000)); // UTC-5
|
||||
|
||||
switch (timeRange) {
|
||||
case 'today': {
|
||||
const { start } = getBusinessDayBounds('today');
|
||||
const businessStart = new Date(start);
|
||||
const businessEnd = new Date(businessStart);
|
||||
businessEnd.setDate(businessEnd.getDate() + 1);
|
||||
businessEnd.setHours(0, 59, 59, 999); // 12:59 AM next day
|
||||
|
||||
const elapsed = easternTime.getTime() - businessStart.getTime();
|
||||
const total = businessEnd.getTime() - businessStart.getTime();
|
||||
return Math.min(100, Math.max(0, (elapsed / total) * 100));
|
||||
}
|
||||
case 'thisWeek': {
|
||||
const startOfWeek = new Date(easternTime);
|
||||
startOfWeek.setDate(easternTime.getDate() - easternTime.getDay()); // Sunday
|
||||
startOfWeek.setHours(1, 0, 0, 0); // 1 AM business day start
|
||||
|
||||
const endOfWeek = new Date(startOfWeek);
|
||||
endOfWeek.setDate(endOfWeek.getDate() + 7);
|
||||
|
||||
const elapsed = easternTime.getTime() - startOfWeek.getTime();
|
||||
const total = endOfWeek.getTime() - startOfWeek.getTime();
|
||||
return Math.min(100, Math.max(0, (elapsed / total) * 100));
|
||||
}
|
||||
case 'thisMonth': {
|
||||
const startOfMonth = new Date(easternTime.getFullYear(), easternTime.getMonth(), 1, 1, 0, 0, 0);
|
||||
const endOfMonth = new Date(easternTime.getFullYear(), easternTime.getMonth() + 1, 1, 0, 59, 59, 999);
|
||||
|
||||
const elapsed = easternTime.getTime() - startOfMonth.getTime();
|
||||
const total = endOfMonth.getTime() - startOfMonth.getTime();
|
||||
return Math.min(100, Math.max(0, (elapsed / total) * 100));
|
||||
}
|
||||
default:
|
||||
return 100;
|
||||
}
|
||||
}
|
||||
|
||||
function buildFinancialTotalsQuery(whereClause) {
|
||||
return `
|
||||
SELECT
|
||||
COALESCE(SUM(sale_amount), 0) as grossSales,
|
||||
COALESCE(SUM(refund_amount), 0) as refunds,
|
||||
COALESCE(SUM(shipping_collected_amount + small_order_fee_amount + rush_fee_amount), 0) as shippingFees,
|
||||
COALESCE(SUM(tax_collected_amount), 0) as taxCollected,
|
||||
COALESCE(SUM(discount_total_amount), 0) as discounts,
|
||||
COALESCE(SUM(cogs_amount), 0) as cogs
|
||||
FROM report_sales_data
|
||||
WHERE ${whereClause}
|
||||
AND action IN (1, 2, 3)
|
||||
`;
|
||||
}
|
||||
|
||||
function buildFinancialTrendQuery(whereClause) {
|
||||
return `
|
||||
SELECT
|
||||
DATE(DATE_SUB(date_change, INTERVAL 1 HOUR)) as date,
|
||||
SUM(sale_amount) as grossSales,
|
||||
SUM(refund_amount) as refunds,
|
||||
SUM(shipping_collected_amount + small_order_fee_amount + rush_fee_amount) as shippingFees,
|
||||
SUM(tax_collected_amount) as taxCollected,
|
||||
SUM(discount_total_amount) as discounts,
|
||||
SUM(cogs_amount) as cogs
|
||||
FROM report_sales_data
|
||||
WHERE ${whereClause}
|
||||
AND action IN (1, 2, 3)
|
||||
GROUP BY DATE(DATE_SUB(date_change, INTERVAL 1 HOUR))
|
||||
ORDER BY date ASC
|
||||
`;
|
||||
}
|
||||
|
||||
function normalizeFinancialTotals(row = {}) {
|
||||
const grossSales = parseFloat(row.grossSales || 0);
|
||||
const refunds = parseFloat(row.refunds || 0);
|
||||
const shippingFees = parseFloat(row.shippingFees || 0);
|
||||
const taxCollected = parseFloat(row.taxCollected || 0);
|
||||
const discounts = parseFloat(row.discounts || 0);
|
||||
const cogs = parseFloat(row.cogs || 0);
|
||||
const productNet = grossSales - refunds - discounts;
|
||||
const income = productNet + shippingFees;
|
||||
const profit = income - cogs;
|
||||
const margin = income !== 0 ? (profit / income) * 100 : 0;
|
||||
|
||||
return {
|
||||
grossSales,
|
||||
refunds,
|
||||
shippingFees,
|
||||
taxCollected,
|
||||
discounts,
|
||||
cogs,
|
||||
income,
|
||||
profit,
|
||||
margin,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeFinancialTrendRow(row = {}) {
|
||||
const grossSales = parseFloat(row.grossSales || 0);
|
||||
const refunds = parseFloat(row.refunds || 0);
|
||||
const shippingFees = parseFloat(row.shippingFees || 0);
|
||||
const taxCollected = parseFloat(row.taxCollected || 0);
|
||||
const discounts = parseFloat(row.discounts || 0);
|
||||
const cogs = parseFloat(row.cogs || 0);
|
||||
const productNet = grossSales - refunds - discounts;
|
||||
const income = productNet + shippingFees;
|
||||
const profit = income - cogs;
|
||||
const margin = income !== 0 ? (profit / income) * 100 : 0;
|
||||
let timestamp = null;
|
||||
let dateValue = null;
|
||||
|
||||
if (row.date instanceof Date) {
|
||||
dateValue = row.date.toISOString().slice(0, 10);
|
||||
} else if (typeof row.date === 'string') {
|
||||
dateValue = row.date;
|
||||
}
|
||||
|
||||
if (typeof dateValue === 'string') {
|
||||
timestamp = new Date(`${dateValue}T06:00:00.000Z`).toISOString();
|
||||
}
|
||||
|
||||
return {
|
||||
date: dateValue,
|
||||
grossSales,
|
||||
refunds,
|
||||
shippingFees,
|
||||
taxCollected,
|
||||
discounts,
|
||||
cogs,
|
||||
income,
|
||||
profit,
|
||||
margin,
|
||||
timestamp,
|
||||
};
|
||||
}
|
||||
|
||||
function calculateComparison(currentValue, previousValue) {
|
||||
if (typeof previousValue !== 'number') {
|
||||
return { absolute: null, percentage: null };
|
||||
}
|
||||
|
||||
const absolute = typeof currentValue === 'number' ? currentValue - previousValue : null;
|
||||
const percentage =
|
||||
absolute !== null && previousValue !== 0
|
||||
? (absolute / Math.abs(previousValue)) * 100
|
||||
: null;
|
||||
|
||||
return { absolute, percentage };
|
||||
}
|
||||
|
||||
function getPreviousPeriodRange(timeRange, startDate, endDate) {
|
||||
if (timeRange && timeRange !== 'custom') {
|
||||
const prevTimeRange = getPreviousTimeRange(timeRange);
|
||||
if (!prevTimeRange || prevTimeRange === timeRange) {
|
||||
return null;
|
||||
}
|
||||
return getTimeRangeConditions(prevTimeRange);
|
||||
}
|
||||
|
||||
const hasCustomDates = (timeRange === 'custom' || !timeRange) && startDate && endDate;
|
||||
if (!hasCustomDates) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const start = new Date(startDate);
|
||||
const end = new Date(endDate);
|
||||
|
||||
if (Number.isNaN(start.getTime()) || Number.isNaN(end.getTime())) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const duration = end.getTime() - start.getTime();
|
||||
if (!Number.isFinite(duration) || duration <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const prevEnd = new Date(start.getTime() - 1);
|
||||
const prevStart = new Date(prevEnd.getTime() - duration);
|
||||
|
||||
return getTimeRangeConditions('custom', prevStart.toISOString(), prevEnd.toISOString());
|
||||
}
|
||||
|
||||
async function getPreviousPeriodData(connection, timeRange, startDate, endDate) {
|
||||
// Calculate previous period dates
|
||||
let prevWhereClause, prevParams;
|
||||
|
||||
if (timeRange && timeRange !== 'custom') {
|
||||
const prevTimeRange = getPreviousTimeRange(timeRange);
|
||||
const result = getTimeRangeConditions(prevTimeRange);
|
||||
prevWhereClause = result.whereClause;
|
||||
prevParams = result.params;
|
||||
} else {
|
||||
// Custom date range - go back by the same duration
|
||||
const start = new Date(startDate);
|
||||
const end = new Date(endDate);
|
||||
const duration = end.getTime() - start.getTime();
|
||||
|
||||
const prevEnd = new Date(start.getTime() - 1);
|
||||
const prevStart = new Date(prevEnd.getTime() - duration);
|
||||
|
||||
prevWhereClause = 'date_placed >= ? AND date_placed <= ?';
|
||||
prevParams = [prevStart.toISOString(), prevEnd.toISOString()];
|
||||
}
|
||||
|
||||
const prevQuery = `
|
||||
SELECT
|
||||
COUNT(*) as orderCount,
|
||||
SUM(summary_total) as revenue,
|
||||
AVG(summary_total) as averageOrderValue
|
||||
FROM _order
|
||||
WHERE order_status > 15 AND ${prevWhereClause}
|
||||
`;
|
||||
|
||||
const [prevResult] = await connection.execute(prevQuery, prevParams);
|
||||
const prev = prevResult[0] || { orderCount: 0, revenue: 0, averageOrderValue: 0 };
|
||||
|
||||
return {
|
||||
orderCount: parseInt(prev.orderCount || 0),
|
||||
revenue: parseFloat(prev.revenue || 0),
|
||||
averageOrderValue: parseFloat(prev.averageOrderValue || 0)
|
||||
};
|
||||
}
|
||||
|
||||
function getPreviousTimeRange(timeRange) {
|
||||
const map = {
|
||||
today: 'yesterday',
|
||||
thisWeek: 'lastWeek',
|
||||
thisMonth: 'lastMonth',
|
||||
last7days: 'previous7days',
|
||||
last30days: 'previous30days',
|
||||
last90days: 'previous90days',
|
||||
yesterday: 'twoDaysAgo'
|
||||
};
|
||||
return map[timeRange] || timeRange;
|
||||
}
|
||||
|
||||
async function getHistoricalProjectionData(connection, timeRange) {
|
||||
// Get historical data for projection calculations
|
||||
// This is a simplified version - you could make this more sophisticated
|
||||
const historicalQuery = `
|
||||
SELECT
|
||||
SUM(summary_total) as revenue,
|
||||
COUNT(*) as orders
|
||||
FROM _order
|
||||
WHERE order_status > 15
|
||||
AND date_placed >= DATE_SUB(NOW(), INTERVAL 30 DAY)
|
||||
AND date_placed < DATE_SUB(NOW(), INTERVAL 1 DAY)
|
||||
`;
|
||||
|
||||
const [result] = await connection.execute(historicalQuery);
|
||||
return result;
|
||||
}
|
||||
|
||||
function calculateSmartProjection(currentRevenue, currentOrders, periodProgress, historicalData) {
|
||||
if (periodProgress >= 100) {
|
||||
return { projectedRevenue: currentRevenue, projectedOrders: currentOrders, confidence: 1.0 };
|
||||
}
|
||||
|
||||
// Simple linear projection with confidence based on how much of the period has elapsed
|
||||
const projectedRevenue = currentRevenue / (periodProgress / 100);
|
||||
const projectedOrders = Math.round(currentOrders / (periodProgress / 100));
|
||||
|
||||
// Confidence increases with more data (higher period progress)
|
||||
const confidence = Math.min(0.95, Math.max(0.1, periodProgress / 100));
|
||||
|
||||
return {
|
||||
projectedRevenue,
|
||||
projectedOrders,
|
||||
confidence
|
||||
};
|
||||
}
|
||||
|
||||
// Health check endpoint
|
||||
router.get('/health', async (req, res) => {
|
||||
try {
|
||||
const poolStatus = getPoolStatus();
|
||||
|
||||
// Test database connectivity
|
||||
const { connection, release } = await getDbConnection();
|
||||
await connection.execute('SELECT 1 as test');
|
||||
release();
|
||||
|
||||
res.json({
|
||||
status: 'healthy',
|
||||
timestamp: new Date().toISOString(),
|
||||
pool: poolStatus,
|
||||
database: 'connected'
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Health check failed:', error);
|
||||
res.status(500).json({
|
||||
status: 'unhealthy',
|
||||
timestamp: new Date().toISOString(),
|
||||
error: error.message,
|
||||
pool: getPoolStatus()
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Debug endpoint for pool status
|
||||
router.get('/debug/pool', (req, res) => {
|
||||
res.json({
|
||||
timestamp: new Date().toISOString(),
|
||||
pool: getPoolStatus()
|
||||
});
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -1,196 +0,0 @@
|
||||
-- Configuration tables schema
|
||||
|
||||
-- Stock threshold configurations
|
||||
CREATE TABLE IF NOT EXISTS stock_thresholds (
|
||||
id INT NOT NULL,
|
||||
category_id BIGINT, -- NULL means default/global threshold
|
||||
vendor VARCHAR(100), -- NULL means applies to all vendors
|
||||
critical_days INT NOT NULL DEFAULT 7,
|
||||
reorder_days INT NOT NULL DEFAULT 14,
|
||||
overstock_days INT NOT NULL DEFAULT 90,
|
||||
low_stock_threshold INT NOT NULL DEFAULT 5,
|
||||
min_reorder_quantity INT NOT NULL DEFAULT 1,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id),
|
||||
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
|
||||
UNIQUE KEY unique_category_vendor (category_id, vendor),
|
||||
INDEX idx_st_metrics (category_id, vendor)
|
||||
);
|
||||
|
||||
-- Lead time threshold configurations
|
||||
CREATE TABLE IF NOT EXISTS lead_time_thresholds (
|
||||
id INT NOT NULL,
|
||||
category_id BIGINT, -- NULL means default/global threshold
|
||||
vendor VARCHAR(100), -- NULL means applies to all vendors
|
||||
target_days INT NOT NULL DEFAULT 14,
|
||||
warning_days INT NOT NULL DEFAULT 21,
|
||||
critical_days INT NOT NULL DEFAULT 30,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id),
|
||||
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
|
||||
UNIQUE KEY unique_category_vendor (category_id, vendor)
|
||||
);
|
||||
|
||||
-- Sales velocity window configurations
|
||||
CREATE TABLE IF NOT EXISTS sales_velocity_config (
|
||||
id INT NOT NULL,
|
||||
category_id BIGINT, -- NULL means default/global threshold
|
||||
vendor VARCHAR(100), -- NULL means applies to all vendors
|
||||
daily_window_days INT NOT NULL DEFAULT 30,
|
||||
weekly_window_days INT NOT NULL DEFAULT 7,
|
||||
monthly_window_days INT NOT NULL DEFAULT 90,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id),
|
||||
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
|
||||
UNIQUE KEY unique_category_vendor (category_id, vendor),
|
||||
INDEX idx_sv_metrics (category_id, vendor)
|
||||
);
|
||||
|
||||
-- ABC Classification configurations
|
||||
CREATE TABLE IF NOT EXISTS abc_classification_config (
|
||||
id INT NOT NULL PRIMARY KEY,
|
||||
a_threshold DECIMAL(5,2) NOT NULL DEFAULT 20.0,
|
||||
b_threshold DECIMAL(5,2) NOT NULL DEFAULT 50.0,
|
||||
classification_period_days INT NOT NULL DEFAULT 90,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Safety stock configurations
|
||||
CREATE TABLE IF NOT EXISTS safety_stock_config (
|
||||
id INT NOT NULL,
|
||||
category_id BIGINT, -- NULL means default/global threshold
|
||||
vendor VARCHAR(100), -- NULL means applies to all vendors
|
||||
coverage_days INT NOT NULL DEFAULT 14,
|
||||
service_level DECIMAL(5,2) NOT NULL DEFAULT 95.0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id),
|
||||
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
|
||||
UNIQUE KEY unique_category_vendor (category_id, vendor),
|
||||
INDEX idx_ss_metrics (category_id, vendor)
|
||||
);
|
||||
|
||||
-- Turnover rate configurations
|
||||
CREATE TABLE IF NOT EXISTS turnover_config (
|
||||
id INT NOT NULL,
|
||||
category_id BIGINT, -- NULL means default/global threshold
|
||||
vendor VARCHAR(100), -- NULL means applies to all vendors
|
||||
calculation_period_days INT NOT NULL DEFAULT 30,
|
||||
target_rate DECIMAL(10,2) NOT NULL DEFAULT 1.0,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id),
|
||||
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
|
||||
UNIQUE KEY unique_category_vendor (category_id, vendor)
|
||||
);
|
||||
|
||||
-- Create table for sales seasonality factors
|
||||
CREATE TABLE IF NOT EXISTS sales_seasonality (
|
||||
month INT NOT NULL,
|
||||
seasonality_factor DECIMAL(5,3) DEFAULT 0,
|
||||
last_updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (month),
|
||||
CHECK (month BETWEEN 1 AND 12),
|
||||
CHECK (seasonality_factor BETWEEN -1.0 AND 1.0)
|
||||
);
|
||||
|
||||
-- Insert default global thresholds if not exists
|
||||
INSERT INTO stock_thresholds (id, category_id, vendor, critical_days, reorder_days, overstock_days)
|
||||
VALUES (1, NULL, NULL, 7, 14, 90)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
critical_days = VALUES(critical_days),
|
||||
reorder_days = VALUES(reorder_days),
|
||||
overstock_days = VALUES(overstock_days);
|
||||
|
||||
INSERT INTO lead_time_thresholds (id, category_id, vendor, target_days, warning_days, critical_days)
|
||||
VALUES (1, NULL, NULL, 14, 21, 30)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
target_days = VALUES(target_days),
|
||||
warning_days = VALUES(warning_days),
|
||||
critical_days = VALUES(critical_days);
|
||||
|
||||
INSERT INTO sales_velocity_config (id, category_id, vendor, daily_window_days, weekly_window_days, monthly_window_days)
|
||||
VALUES (1, NULL, NULL, 30, 7, 90)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
daily_window_days = VALUES(daily_window_days),
|
||||
weekly_window_days = VALUES(weekly_window_days),
|
||||
monthly_window_days = VALUES(monthly_window_days);
|
||||
|
||||
INSERT INTO abc_classification_config (id, a_threshold, b_threshold, classification_period_days)
|
||||
VALUES (1, 20.0, 50.0, 90)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
a_threshold = VALUES(a_threshold),
|
||||
b_threshold = VALUES(b_threshold),
|
||||
classification_period_days = VALUES(classification_period_days);
|
||||
|
||||
INSERT INTO safety_stock_config (id, category_id, vendor, coverage_days, service_level)
|
||||
VALUES (1, NULL, NULL, 14, 95.0)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
coverage_days = VALUES(coverage_days),
|
||||
service_level = VALUES(service_level);
|
||||
|
||||
INSERT INTO turnover_config (id, category_id, vendor, calculation_period_days, target_rate)
|
||||
VALUES (1, NULL, NULL, 30, 1.0)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
calculation_period_days = VALUES(calculation_period_days),
|
||||
target_rate = VALUES(target_rate);
|
||||
|
||||
-- Insert default seasonality factors (neutral)
|
||||
INSERT INTO sales_seasonality (month, seasonality_factor)
|
||||
VALUES
|
||||
(1, 0), (2, 0), (3, 0), (4, 0), (5, 0), (6, 0),
|
||||
(7, 0), (8, 0), (9, 0), (10, 0), (11, 0), (12, 0)
|
||||
ON DUPLICATE KEY UPDATE last_updated = CURRENT_TIMESTAMP;
|
||||
|
||||
-- View to show thresholds with category names
|
||||
CREATE OR REPLACE VIEW stock_thresholds_view AS
|
||||
SELECT
|
||||
st.*,
|
||||
c.name as category_name,
|
||||
CASE
|
||||
WHEN st.category_id IS NULL AND st.vendor IS NULL THEN 'Global Default'
|
||||
WHEN st.category_id IS NULL THEN CONCAT('Vendor: ', st.vendor)
|
||||
WHEN st.vendor IS NULL THEN CONCAT('Category: ', c.name)
|
||||
ELSE CONCAT('Category: ', c.name, ' / Vendor: ', st.vendor)
|
||||
END as threshold_scope
|
||||
FROM
|
||||
stock_thresholds st
|
||||
LEFT JOIN
|
||||
categories c ON st.category_id = c.cat_id
|
||||
ORDER BY
|
||||
CASE
|
||||
WHEN st.category_id IS NULL AND st.vendor IS NULL THEN 1
|
||||
WHEN st.category_id IS NULL THEN 2
|
||||
WHEN st.vendor IS NULL THEN 3
|
||||
ELSE 4
|
||||
END,
|
||||
c.name,
|
||||
st.vendor;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sync_status (
|
||||
table_name VARCHAR(50) PRIMARY KEY,
|
||||
last_sync_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
last_sync_id BIGINT,
|
||||
INDEX idx_last_sync (last_sync_timestamp)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS import_history (
|
||||
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
table_name VARCHAR(50) NOT NULL,
|
||||
start_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
end_time TIMESTAMP NULL,
|
||||
duration_seconds INT,
|
||||
duration_minutes DECIMAL(10,2) GENERATED ALWAYS AS (duration_seconds / 60.0) STORED,
|
||||
records_added INT DEFAULT 0,
|
||||
records_updated INT DEFAULT 0,
|
||||
is_incremental BOOLEAN DEFAULT FALSE,
|
||||
status ENUM('running', 'completed', 'failed', 'cancelled') DEFAULT 'running',
|
||||
error_message TEXT,
|
||||
additional_info JSON,
|
||||
INDEX idx_table_time (table_name, start_time),
|
||||
INDEX idx_status (status)
|
||||
);
|
||||
@@ -1,430 +0,0 @@
|
||||
-- Disable foreign key checks
|
||||
SET FOREIGN_KEY_CHECKS = 0;
|
||||
|
||||
-- Temporary tables for batch metrics processing
|
||||
CREATE TABLE IF NOT EXISTS temp_sales_metrics (
|
||||
pid BIGINT NOT NULL,
|
||||
daily_sales_avg DECIMAL(10,3),
|
||||
weekly_sales_avg DECIMAL(10,3),
|
||||
monthly_sales_avg DECIMAL(10,3),
|
||||
total_revenue DECIMAL(10,3),
|
||||
avg_margin_percent DECIMAL(10,3),
|
||||
first_sale_date DATE,
|
||||
last_sale_date DATE,
|
||||
PRIMARY KEY (pid)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS temp_purchase_metrics (
|
||||
pid BIGINT NOT NULL,
|
||||
avg_lead_time_days INT,
|
||||
last_purchase_date DATE,
|
||||
first_received_date DATE,
|
||||
last_received_date DATE,
|
||||
PRIMARY KEY (pid)
|
||||
);
|
||||
|
||||
-- New table for product metrics
|
||||
CREATE TABLE IF NOT EXISTS product_metrics (
|
||||
pid BIGINT NOT NULL,
|
||||
last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
-- Sales velocity metrics
|
||||
daily_sales_avg DECIMAL(10,3),
|
||||
weekly_sales_avg DECIMAL(10,3),
|
||||
monthly_sales_avg DECIMAL(10,3),
|
||||
avg_quantity_per_order DECIMAL(10,3),
|
||||
number_of_orders INT,
|
||||
first_sale_date DATE,
|
||||
last_sale_date DATE,
|
||||
-- Stock metrics
|
||||
days_of_inventory INT,
|
||||
weeks_of_inventory INT,
|
||||
reorder_point INT,
|
||||
safety_stock INT,
|
||||
reorder_qty INT DEFAULT 0,
|
||||
overstocked_amt INT DEFAULT 0,
|
||||
-- Financial metrics
|
||||
avg_margin_percent DECIMAL(10,3),
|
||||
total_revenue DECIMAL(10,3),
|
||||
inventory_value DECIMAL(10,3),
|
||||
cost_of_goods_sold DECIMAL(10,3),
|
||||
gross_profit DECIMAL(10,3),
|
||||
gmroi DECIMAL(10,3),
|
||||
-- Purchase metrics
|
||||
avg_lead_time_days INT,
|
||||
last_purchase_date DATE,
|
||||
first_received_date DATE,
|
||||
last_received_date DATE,
|
||||
-- Classification metrics
|
||||
abc_class CHAR(1),
|
||||
stock_status VARCHAR(20),
|
||||
-- Turnover metrics
|
||||
turnover_rate DECIMAL(12,3),
|
||||
-- Lead time metrics
|
||||
current_lead_time INT,
|
||||
target_lead_time INT,
|
||||
lead_time_status VARCHAR(20),
|
||||
-- Forecast metrics
|
||||
forecast_accuracy DECIMAL(5,2) DEFAULT NULL,
|
||||
forecast_bias DECIMAL(5,2) DEFAULT NULL,
|
||||
last_forecast_date DATE DEFAULT NULL,
|
||||
PRIMARY KEY (pid),
|
||||
FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE,
|
||||
INDEX idx_metrics_revenue (total_revenue),
|
||||
INDEX idx_metrics_stock_status (stock_status),
|
||||
INDEX idx_metrics_lead_time (lead_time_status),
|
||||
INDEX idx_metrics_turnover (turnover_rate),
|
||||
INDEX idx_metrics_last_calculated (last_calculated_at),
|
||||
INDEX idx_metrics_abc (abc_class),
|
||||
INDEX idx_metrics_sales (daily_sales_avg, weekly_sales_avg, monthly_sales_avg),
|
||||
INDEX idx_metrics_forecast (forecast_accuracy, forecast_bias)
|
||||
);
|
||||
|
||||
-- New table for time-based aggregates
|
||||
CREATE TABLE IF NOT EXISTS product_time_aggregates (
|
||||
pid BIGINT NOT NULL,
|
||||
year INT NOT NULL,
|
||||
month INT NOT NULL,
|
||||
-- Sales metrics
|
||||
total_quantity_sold INT DEFAULT 0,
|
||||
total_revenue DECIMAL(10,3) DEFAULT 0,
|
||||
total_cost DECIMAL(10,3) DEFAULT 0,
|
||||
order_count INT DEFAULT 0,
|
||||
-- Stock changes
|
||||
stock_received INT DEFAULT 0,
|
||||
stock_ordered INT DEFAULT 0,
|
||||
-- Calculated fields
|
||||
avg_price DECIMAL(10,3),
|
||||
profit_margin DECIMAL(10,3),
|
||||
inventory_value DECIMAL(10,3),
|
||||
gmroi DECIMAL(10,3),
|
||||
PRIMARY KEY (pid, year, month),
|
||||
FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE,
|
||||
INDEX idx_date (year, month)
|
||||
);
|
||||
|
||||
-- Create vendor details table
|
||||
CREATE TABLE IF NOT EXISTS vendor_details (
|
||||
vendor VARCHAR(100) NOT NULL,
|
||||
contact_name VARCHAR(100),
|
||||
email VARCHAR(100),
|
||||
phone VARCHAR(20),
|
||||
status VARCHAR(20) DEFAULT 'active',
|
||||
notes TEXT,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (vendor),
|
||||
INDEX idx_vendor_status (status)
|
||||
);
|
||||
|
||||
-- New table for vendor metrics
|
||||
CREATE TABLE IF NOT EXISTS vendor_metrics (
|
||||
vendor VARCHAR(100) NOT NULL,
|
||||
last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
-- Performance metrics
|
||||
avg_lead_time_days DECIMAL(10,3),
|
||||
on_time_delivery_rate DECIMAL(5,2),
|
||||
order_fill_rate DECIMAL(5,2),
|
||||
total_orders INT DEFAULT 0,
|
||||
total_late_orders INT DEFAULT 0,
|
||||
total_purchase_value DECIMAL(10,3) DEFAULT 0,
|
||||
avg_order_value DECIMAL(10,3),
|
||||
-- Product metrics
|
||||
active_products INT DEFAULT 0,
|
||||
total_products INT DEFAULT 0,
|
||||
-- Financial metrics
|
||||
total_revenue DECIMAL(10,3) DEFAULT 0,
|
||||
avg_margin_percent DECIMAL(5,2),
|
||||
-- Status
|
||||
status VARCHAR(20) DEFAULT 'active',
|
||||
PRIMARY KEY (vendor),
|
||||
FOREIGN KEY (vendor) REFERENCES vendor_details(vendor) ON DELETE CASCADE,
|
||||
INDEX idx_vendor_performance (on_time_delivery_rate),
|
||||
INDEX idx_vendor_status (status),
|
||||
INDEX idx_metrics_last_calculated (last_calculated_at),
|
||||
INDEX idx_vendor_metrics_orders (total_orders, total_late_orders)
|
||||
);
|
||||
|
||||
-- New table for category metrics
|
||||
CREATE TABLE IF NOT EXISTS category_metrics (
|
||||
category_id BIGINT NOT NULL,
|
||||
last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
-- Product metrics
|
||||
product_count INT DEFAULT 0,
|
||||
active_products INT DEFAULT 0,
|
||||
-- Financial metrics
|
||||
total_value DECIMAL(15,3) DEFAULT 0,
|
||||
avg_margin DECIMAL(5,2),
|
||||
turnover_rate DECIMAL(12,3),
|
||||
growth_rate DECIMAL(5,2),
|
||||
-- Status
|
||||
status VARCHAR(20) DEFAULT 'active',
|
||||
PRIMARY KEY (category_id),
|
||||
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
|
||||
INDEX idx_category_status (status),
|
||||
INDEX idx_category_growth (growth_rate),
|
||||
INDEX idx_metrics_last_calculated (last_calculated_at),
|
||||
INDEX idx_category_metrics_products (product_count, active_products)
|
||||
);
|
||||
|
||||
-- New table for vendor time-based metrics
|
||||
CREATE TABLE IF NOT EXISTS vendor_time_metrics (
|
||||
vendor VARCHAR(100) NOT NULL,
|
||||
year INT NOT NULL,
|
||||
month INT NOT NULL,
|
||||
-- Order metrics
|
||||
total_orders INT DEFAULT 0,
|
||||
late_orders INT DEFAULT 0,
|
||||
avg_lead_time_days DECIMAL(10,3),
|
||||
-- Financial metrics
|
||||
total_purchase_value DECIMAL(10,3) DEFAULT 0,
|
||||
total_revenue DECIMAL(10,3) DEFAULT 0,
|
||||
avg_margin_percent DECIMAL(5,2),
|
||||
PRIMARY KEY (vendor, year, month),
|
||||
FOREIGN KEY (vendor) REFERENCES vendor_details(vendor) ON DELETE CASCADE,
|
||||
INDEX idx_vendor_date (year, month)
|
||||
);
|
||||
|
||||
-- New table for category time-based metrics
|
||||
CREATE TABLE IF NOT EXISTS category_time_metrics (
|
||||
category_id BIGINT NOT NULL,
|
||||
year INT NOT NULL,
|
||||
month INT NOT NULL,
|
||||
-- Product metrics
|
||||
product_count INT DEFAULT 0,
|
||||
active_products INT DEFAULT 0,
|
||||
-- Financial metrics
|
||||
total_value DECIMAL(15,3) DEFAULT 0,
|
||||
total_revenue DECIMAL(15,3) DEFAULT 0,
|
||||
avg_margin DECIMAL(5,2),
|
||||
turnover_rate DECIMAL(12,3),
|
||||
PRIMARY KEY (category_id, year, month),
|
||||
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
|
||||
INDEX idx_category_date (year, month)
|
||||
);
|
||||
|
||||
-- New table for category-based sales metrics
|
||||
CREATE TABLE IF NOT EXISTS category_sales_metrics (
|
||||
category_id BIGINT NOT NULL,
|
||||
brand VARCHAR(100) NOT NULL,
|
||||
period_start DATE NOT NULL,
|
||||
period_end DATE NOT NULL,
|
||||
avg_daily_sales DECIMAL(10,3) DEFAULT 0,
|
||||
total_sold INT DEFAULT 0,
|
||||
num_products INT DEFAULT 0,
|
||||
avg_price DECIMAL(10,3) DEFAULT 0,
|
||||
last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (category_id, brand, period_start, period_end),
|
||||
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
|
||||
INDEX idx_category_brand (category_id, brand),
|
||||
INDEX idx_period (period_start, period_end)
|
||||
);
|
||||
|
||||
-- New table for brand metrics
|
||||
CREATE TABLE IF NOT EXISTS brand_metrics (
|
||||
brand VARCHAR(100) NOT NULL,
|
||||
last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
-- Product metrics
|
||||
product_count INT DEFAULT 0,
|
||||
active_products INT DEFAULT 0,
|
||||
-- Stock metrics
|
||||
total_stock_units INT DEFAULT 0,
|
||||
total_stock_cost DECIMAL(15,2) DEFAULT 0,
|
||||
total_stock_retail DECIMAL(15,2) DEFAULT 0,
|
||||
-- Sales metrics
|
||||
total_revenue DECIMAL(15,2) DEFAULT 0,
|
||||
avg_margin DECIMAL(5,2) DEFAULT 0,
|
||||
growth_rate DECIMAL(5,2) DEFAULT 0,
|
||||
PRIMARY KEY (brand),
|
||||
INDEX idx_brand_metrics_last_calculated (last_calculated_at),
|
||||
INDEX idx_brand_metrics_revenue (total_revenue),
|
||||
INDEX idx_brand_metrics_growth (growth_rate)
|
||||
);
|
||||
|
||||
-- New table for brand time-based metrics
|
||||
CREATE TABLE IF NOT EXISTS brand_time_metrics (
|
||||
brand VARCHAR(100) NOT NULL,
|
||||
year INT NOT NULL,
|
||||
month INT NOT NULL,
|
||||
-- Product metrics
|
||||
product_count INT DEFAULT 0,
|
||||
active_products INT DEFAULT 0,
|
||||
-- Stock metrics
|
||||
total_stock_units INT DEFAULT 0,
|
||||
total_stock_cost DECIMAL(15,2) DEFAULT 0,
|
||||
total_stock_retail DECIMAL(15,2) DEFAULT 0,
|
||||
-- Sales metrics
|
||||
total_revenue DECIMAL(15,2) DEFAULT 0,
|
||||
avg_margin DECIMAL(5,2) DEFAULT 0,
|
||||
PRIMARY KEY (brand, year, month),
|
||||
INDEX idx_brand_date (year, month)
|
||||
);
|
||||
|
||||
-- New table for sales forecasts
|
||||
CREATE TABLE IF NOT EXISTS sales_forecasts (
|
||||
pid BIGINT NOT NULL,
|
||||
forecast_date DATE NOT NULL,
|
||||
forecast_units DECIMAL(10,2) DEFAULT 0,
|
||||
forecast_revenue DECIMAL(10,2) DEFAULT 0,
|
||||
confidence_level DECIMAL(5,2) DEFAULT 0,
|
||||
last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (pid, forecast_date),
|
||||
FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE,
|
||||
INDEX idx_forecast_date (forecast_date),
|
||||
INDEX idx_forecast_last_calculated (last_calculated_at)
|
||||
);
|
||||
|
||||
-- New table for category forecasts
|
||||
CREATE TABLE IF NOT EXISTS category_forecasts (
|
||||
category_id BIGINT NOT NULL,
|
||||
forecast_date DATE NOT NULL,
|
||||
forecast_units DECIMAL(10,2) DEFAULT 0,
|
||||
forecast_revenue DECIMAL(10,2) DEFAULT 0,
|
||||
confidence_level DECIMAL(5,2) DEFAULT 0,
|
||||
last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (category_id, forecast_date),
|
||||
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
|
||||
INDEX idx_category_forecast_date (forecast_date),
|
||||
INDEX idx_category_forecast_last_calculated (last_calculated_at)
|
||||
);
|
||||
|
||||
-- Create view for inventory health
|
||||
CREATE OR REPLACE VIEW inventory_health AS
|
||||
WITH product_thresholds AS (
|
||||
SELECT
|
||||
p.pid,
|
||||
COALESCE(
|
||||
-- Try category+vendor specific
|
||||
(SELECT critical_days FROM stock_thresholds st
|
||||
JOIN product_categories pc ON st.category_id = pc.cat_id
|
||||
WHERE pc.pid = p.pid
|
||||
AND st.vendor = p.vendor LIMIT 1),
|
||||
-- Try category specific
|
||||
(SELECT critical_days FROM stock_thresholds st
|
||||
JOIN product_categories pc ON st.category_id = pc.cat_id
|
||||
WHERE pc.pid = p.pid
|
||||
AND st.vendor IS NULL LIMIT 1),
|
||||
-- Try vendor specific
|
||||
(SELECT critical_days FROM stock_thresholds st
|
||||
WHERE st.category_id IS NULL
|
||||
AND st.vendor = p.vendor LIMIT 1),
|
||||
-- Fall back to default
|
||||
(SELECT critical_days FROM stock_thresholds st
|
||||
WHERE st.category_id IS NULL
|
||||
AND st.vendor IS NULL LIMIT 1),
|
||||
7
|
||||
) as critical_days,
|
||||
COALESCE(
|
||||
-- Try category+vendor specific
|
||||
(SELECT reorder_days FROM stock_thresholds st
|
||||
JOIN product_categories pc ON st.category_id = pc.cat_id
|
||||
WHERE pc.pid = p.pid
|
||||
AND st.vendor = p.vendor LIMIT 1),
|
||||
-- Try category specific
|
||||
(SELECT reorder_days FROM stock_thresholds st
|
||||
JOIN product_categories pc ON st.category_id = pc.cat_id
|
||||
WHERE pc.pid = p.pid
|
||||
AND st.vendor IS NULL LIMIT 1),
|
||||
-- Try vendor specific
|
||||
(SELECT reorder_days FROM stock_thresholds st
|
||||
WHERE st.category_id IS NULL
|
||||
AND st.vendor = p.vendor LIMIT 1),
|
||||
-- Fall back to default
|
||||
(SELECT reorder_days FROM stock_thresholds st
|
||||
WHERE st.category_id IS NULL
|
||||
AND st.vendor IS NULL LIMIT 1),
|
||||
14
|
||||
) as reorder_days,
|
||||
COALESCE(
|
||||
-- Try category+vendor specific
|
||||
(SELECT overstock_days FROM stock_thresholds st
|
||||
JOIN product_categories pc ON st.category_id = pc.cat_id
|
||||
WHERE pc.pid = p.pid
|
||||
AND st.vendor = p.vendor LIMIT 1),
|
||||
-- Try category specific
|
||||
(SELECT overstock_days FROM stock_thresholds st
|
||||
JOIN product_categories pc ON st.category_id = pc.cat_id
|
||||
WHERE pc.pid = p.pid
|
||||
AND st.vendor IS NULL LIMIT 1),
|
||||
-- Try vendor specific
|
||||
(SELECT overstock_days FROM stock_thresholds st
|
||||
WHERE st.category_id IS NULL
|
||||
AND st.vendor = p.vendor LIMIT 1),
|
||||
-- Fall back to default
|
||||
(SELECT overstock_days FROM stock_thresholds st
|
||||
WHERE st.category_id IS NULL
|
||||
AND st.vendor IS NULL LIMIT 1),
|
||||
90
|
||||
) as overstock_days
|
||||
FROM products p
|
||||
)
|
||||
SELECT
|
||||
p.pid,
|
||||
p.SKU,
|
||||
p.title,
|
||||
p.stock_quantity,
|
||||
COALESCE(pm.daily_sales_avg, 0) as daily_sales_avg,
|
||||
COALESCE(pm.days_of_inventory, 0) as days_of_inventory,
|
||||
COALESCE(pm.reorder_point, 0) as reorder_point,
|
||||
COALESCE(pm.safety_stock, 0) as safety_stock,
|
||||
CASE
|
||||
WHEN pm.daily_sales_avg = 0 THEN 'New'
|
||||
WHEN p.stock_quantity <= CEIL(pm.daily_sales_avg * pt.critical_days) THEN 'Critical'
|
||||
WHEN p.stock_quantity <= CEIL(pm.daily_sales_avg * pt.reorder_days) THEN 'Reorder'
|
||||
WHEN p.stock_quantity > (pm.daily_sales_avg * pt.overstock_days) THEN 'Overstocked'
|
||||
ELSE 'Healthy'
|
||||
END as stock_status
|
||||
FROM
|
||||
products p
|
||||
LEFT JOIN
|
||||
product_metrics pm ON p.pid = pm.pid
|
||||
LEFT JOIN
|
||||
product_thresholds pt ON p.pid = pt.pid
|
||||
WHERE
|
||||
p.managing_stock = true;
|
||||
|
||||
-- Create view for category performance trends
|
||||
CREATE OR REPLACE VIEW category_performance_trends AS
|
||||
SELECT
|
||||
c.cat_id as category_id,
|
||||
c.name,
|
||||
c.description,
|
||||
p.name as parent_name,
|
||||
c.status,
|
||||
cm.product_count,
|
||||
cm.active_products,
|
||||
cm.total_value,
|
||||
cm.avg_margin,
|
||||
cm.turnover_rate,
|
||||
cm.growth_rate,
|
||||
CASE
|
||||
WHEN cm.growth_rate >= 20 THEN 'High Growth'
|
||||
WHEN cm.growth_rate >= 5 THEN 'Growing'
|
||||
WHEN cm.growth_rate >= -5 THEN 'Stable'
|
||||
ELSE 'Declining'
|
||||
END as performance_rating
|
||||
FROM
|
||||
categories c
|
||||
LEFT JOIN
|
||||
categories p ON c.parent_id = p.cat_id
|
||||
LEFT JOIN
|
||||
category_metrics cm ON c.cat_id = cm.category_id;
|
||||
|
||||
-- Re-enable foreign key checks
|
||||
SET FOREIGN_KEY_CHECKS = 1;
|
||||
|
||||
-- Create table for sales seasonality factors
|
||||
CREATE TABLE IF NOT EXISTS sales_seasonality (
|
||||
month INT NOT NULL,
|
||||
seasonality_factor DECIMAL(5,3) DEFAULT 0,
|
||||
last_updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (month),
|
||||
CHECK (month BETWEEN 1 AND 12),
|
||||
CHECK (seasonality_factor BETWEEN -1.0 AND 1.0)
|
||||
);
|
||||
|
||||
-- Insert default seasonality factors (neutral)
|
||||
INSERT INTO sales_seasonality (month, seasonality_factor)
|
||||
VALUES
|
||||
(1, 0), (2, 0), (3, 0), (4, 0), (5, 0), (6, 0),
|
||||
(7, 0), (8, 0), (9, 0), (10, 0), (11, 0), (12, 0)
|
||||
ON DUPLICATE KEY UPDATE last_updated = CURRENT_TIMESTAMP;
|
||||
@@ -1,168 +0,0 @@
|
||||
-- Enable strict error reporting
|
||||
SET sql_mode = 'STRICT_ALL_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ZERO_DATE,NO_ZERO_IN_DATE,NO_ENGINE_SUBSTITUTION';
|
||||
SET FOREIGN_KEY_CHECKS = 0;
|
||||
|
||||
-- Create tables
|
||||
CREATE TABLE products (
|
||||
pid BIGINT NOT NULL,
|
||||
title VARCHAR(255) NOT NULL,
|
||||
description TEXT,
|
||||
SKU VARCHAR(50) NOT NULL,
|
||||
created_at TIMESTAMP NULL,
|
||||
first_received TIMESTAMP NULL,
|
||||
stock_quantity INT DEFAULT 0,
|
||||
preorder_count INT DEFAULT 0,
|
||||
notions_inv_count INT DEFAULT 0,
|
||||
price DECIMAL(10, 3) NOT NULL,
|
||||
regular_price DECIMAL(10, 3) NOT NULL,
|
||||
cost_price DECIMAL(10, 3),
|
||||
landing_cost_price DECIMAL(10, 3),
|
||||
barcode VARCHAR(50),
|
||||
harmonized_tariff_code VARCHAR(20),
|
||||
updated_at TIMESTAMP,
|
||||
visible BOOLEAN DEFAULT true,
|
||||
managing_stock BOOLEAN DEFAULT true,
|
||||
replenishable BOOLEAN DEFAULT true,
|
||||
vendor VARCHAR(100),
|
||||
vendor_reference VARCHAR(100),
|
||||
notions_reference VARCHAR(100),
|
||||
permalink VARCHAR(255),
|
||||
categories TEXT,
|
||||
image VARCHAR(255),
|
||||
image_175 VARCHAR(255),
|
||||
image_full VARCHAR(255),
|
||||
brand VARCHAR(100),
|
||||
line VARCHAR(100),
|
||||
subline VARCHAR(100),
|
||||
artist VARCHAR(100),
|
||||
options TEXT,
|
||||
tags TEXT,
|
||||
moq INT DEFAULT 1,
|
||||
uom INT DEFAULT 1,
|
||||
rating DECIMAL(10,2) DEFAULT 0.00,
|
||||
reviews INT UNSIGNED DEFAULT 0,
|
||||
weight DECIMAL(10,3),
|
||||
length DECIMAL(10,3),
|
||||
width DECIMAL(10,3),
|
||||
height DECIMAL(10,3),
|
||||
country_of_origin VARCHAR(5),
|
||||
location VARCHAR(50),
|
||||
total_sold INT UNSIGNED DEFAULT 0,
|
||||
baskets INT UNSIGNED DEFAULT 0,
|
||||
notifies INT UNSIGNED DEFAULT 0,
|
||||
date_last_sold DATE,
|
||||
PRIMARY KEY (pid),
|
||||
INDEX idx_sku (SKU),
|
||||
INDEX idx_vendor (vendor),
|
||||
INDEX idx_brand (brand),
|
||||
INDEX idx_location (location),
|
||||
INDEX idx_total_sold (total_sold),
|
||||
INDEX idx_date_last_sold (date_last_sold)
|
||||
) ENGINE=InnoDB;
|
||||
|
||||
-- Create categories table with hierarchy support
|
||||
CREATE TABLE categories (
|
||||
cat_id BIGINT PRIMARY KEY,
|
||||
name VARCHAR(100) NOT NULL,
|
||||
type SMALLINT NOT NULL COMMENT '10=section, 11=category, 12=subcategory, 13=subsubcategory, 1=company, 2=line, 3=subline, 40=artist',
|
||||
parent_id BIGINT,
|
||||
description TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
status VARCHAR(20) DEFAULT 'active',
|
||||
FOREIGN KEY (parent_id) REFERENCES categories(cat_id),
|
||||
INDEX idx_parent (parent_id),
|
||||
INDEX idx_type (type),
|
||||
INDEX idx_status (status),
|
||||
INDEX idx_name_type (name, type)
|
||||
) ENGINE=InnoDB;
|
||||
|
||||
-- Create vendor_details table
|
||||
CREATE TABLE vendor_details (
|
||||
vendor VARCHAR(100) PRIMARY KEY,
|
||||
contact_name VARCHAR(100),
|
||||
email VARCHAR(255),
|
||||
phone VARCHAR(50),
|
||||
status VARCHAR(20) DEFAULT 'active',
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
INDEX idx_status (status)
|
||||
) ENGINE=InnoDB;
|
||||
|
||||
-- Create product_categories junction table
|
||||
CREATE TABLE product_categories (
|
||||
cat_id BIGINT NOT NULL,
|
||||
pid BIGINT NOT NULL,
|
||||
PRIMARY KEY (pid, cat_id),
|
||||
FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE,
|
||||
FOREIGN KEY (cat_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
|
||||
INDEX idx_category (cat_id),
|
||||
INDEX idx_product (pid)
|
||||
) ENGINE=InnoDB;
|
||||
|
||||
-- Create orders table with its indexes
|
||||
CREATE TABLE IF NOT EXISTS orders (
|
||||
id BIGINT NOT NULL AUTO_INCREMENT,
|
||||
order_number VARCHAR(50) NOT NULL,
|
||||
pid BIGINT NOT NULL,
|
||||
SKU VARCHAR(50) NOT NULL,
|
||||
date DATE NOT NULL,
|
||||
price DECIMAL(10,3) NOT NULL,
|
||||
quantity INT NOT NULL,
|
||||
discount DECIMAL(10,3) DEFAULT 0.000,
|
||||
tax DECIMAL(10,3) DEFAULT 0.000,
|
||||
tax_included TINYINT(1) DEFAULT 0,
|
||||
shipping DECIMAL(10,3) DEFAULT 0.000,
|
||||
costeach DECIMAL(10,3) DEFAULT 0.000,
|
||||
customer VARCHAR(50) NOT NULL,
|
||||
customer_name VARCHAR(100),
|
||||
status VARCHAR(20) DEFAULT 'pending',
|
||||
canceled TINYINT(1) DEFAULT 0,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY unique_order_line (order_number, pid),
|
||||
KEY order_number (order_number),
|
||||
KEY pid (pid),
|
||||
KEY customer (customer),
|
||||
KEY date (date),
|
||||
KEY status (status),
|
||||
INDEX idx_orders_metrics (pid, date, canceled)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
|
||||
-- Create purchase_orders table with its indexes
|
||||
CREATE TABLE purchase_orders (
|
||||
id BIGINT AUTO_INCREMENT PRIMARY KEY,
|
||||
po_id VARCHAR(50) NOT NULL,
|
||||
vendor VARCHAR(100) NOT NULL,
|
||||
date DATE NOT NULL,
|
||||
expected_date DATE,
|
||||
pid BIGINT NOT NULL,
|
||||
sku VARCHAR(50) NOT NULL,
|
||||
name VARCHAR(100) NOT NULL COMMENT 'Product name from products.description',
|
||||
cost_price DECIMAL(10, 3) NOT NULL,
|
||||
po_cost_price DECIMAL(10, 3) NOT NULL COMMENT 'Original cost from PO, before receiving adjustments',
|
||||
status TINYINT UNSIGNED DEFAULT 1 COMMENT '0=canceled,1=created,10=electronically_ready_send,11=ordered,12=preordered,13=electronically_sent,15=receiving_started,50=done',
|
||||
receiving_status TINYINT UNSIGNED DEFAULT 1 COMMENT '0=canceled,1=created,30=partial_received,40=full_received,50=paid',
|
||||
notes TEXT,
|
||||
long_note TEXT,
|
||||
ordered INT NOT NULL,
|
||||
received INT DEFAULT 0,
|
||||
received_date DATE COMMENT 'Date of first receiving',
|
||||
last_received_date DATE COMMENT 'Date of most recent receiving',
|
||||
received_by INT,
|
||||
receiving_history JSON COMMENT 'Array of receiving records with qty, date, cost, receiving_id, and alt_po flag',
|
||||
FOREIGN KEY (pid) REFERENCES products(pid),
|
||||
INDEX idx_po_id (po_id),
|
||||
INDEX idx_vendor (vendor),
|
||||
INDEX idx_status (status),
|
||||
INDEX idx_receiving_status (receiving_status),
|
||||
INDEX idx_purchase_orders_metrics (pid, date, status, ordered, received),
|
||||
INDEX idx_po_metrics (pid, date, receiving_status, received_date),
|
||||
INDEX idx_po_product_date (pid, date),
|
||||
INDEX idx_po_product_status (pid, status),
|
||||
UNIQUE KEY unique_po_product (po_id, pid)
|
||||
) ENGINE=InnoDB;
|
||||
|
||||
SET FOREIGN_KEY_CHECKS = 1;
|
||||
|
||||
-- Create views for common calculations
|
||||
-- product_sales_trends view moved to metrics-schema.sql
|
||||
File diff suppressed because it is too large
Load Diff
3172
inventory-server/package-lock.json
generated
3172
inventory-server/package-lock.json
generated
File diff suppressed because it is too large
Load Diff
@@ -1,34 +0,0 @@
|
||||
{
|
||||
"name": "inventory-server",
|
||||
"version": "1.0.0",
|
||||
"description": "Backend server for inventory management system",
|
||||
"main": "src/server.js",
|
||||
"scripts": {
|
||||
"start": "node src/server.js",
|
||||
"dev": "nodemon src/server.js",
|
||||
"prod": "pm2 start ecosystem.config.js",
|
||||
"prod:stop": "pm2 stop inventory-server",
|
||||
"prod:restart": "pm2 restart inventory-server",
|
||||
"prod:logs": "pm2 logs inventory-server",
|
||||
"prod:status": "pm2 status inventory-server",
|
||||
"setup": "mkdir -p logs uploads",
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"cors": "^2.8.5",
|
||||
"csv-parse": "^5.6.0",
|
||||
"dotenv": "^16.4.7",
|
||||
"express": "^4.18.2",
|
||||
"multer": "^1.4.5-lts.1",
|
||||
"mysql2": "^3.12.0",
|
||||
"pm2": "^5.3.0",
|
||||
"ssh2": "^1.16.0",
|
||||
"uuid": "^9.0.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nodemon": "^3.0.2"
|
||||
}
|
||||
}
|
||||
@@ -1,395 +0,0 @@
|
||||
const path = require('path');
|
||||
|
||||
// Change working directory to script directory
|
||||
process.chdir(path.dirname(__filename));
|
||||
|
||||
require('dotenv').config({ path: path.resolve(__dirname, '..', '.env') });
|
||||
|
||||
// Configuration flags for controlling which metrics to calculate
|
||||
// Set to 1 to skip the corresponding calculation, 0 to run it
|
||||
const SKIP_PRODUCT_METRICS = 1; // Skip all product metrics
|
||||
const SKIP_TIME_AGGREGATES = 1; // Skip time aggregates
|
||||
const SKIP_FINANCIAL_METRICS = 1; // Skip financial metrics
|
||||
const SKIP_VENDOR_METRICS = 1; // Skip vendor metrics
|
||||
const SKIP_CATEGORY_METRICS = 1; // Skip category metrics
|
||||
const SKIP_BRAND_METRICS = 1; // Skip brand metrics
|
||||
const SKIP_SALES_FORECASTS = 1; // Skip sales forecasts
|
||||
|
||||
// Add error handler for uncaught exceptions
|
||||
process.on('uncaughtException', (error) => {
|
||||
console.error('Uncaught Exception:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
// Add error handler for unhandled promise rejections
|
||||
process.on('unhandledRejection', (reason, promise) => {
|
||||
console.error('Unhandled Rejection at:', promise, 'reason:', reason);
|
||||
process.exit(1);
|
||||
});
|
||||
|
||||
const progress = require('./metrics/utils/progress');
|
||||
console.log('Progress module loaded:', {
|
||||
modulePath: require.resolve('./metrics/utils/progress'),
|
||||
exports: Object.keys(progress),
|
||||
currentDir: process.cwd(),
|
||||
scriptDir: __dirname
|
||||
});
|
||||
|
||||
// Store progress functions in global scope to ensure availability
|
||||
global.formatElapsedTime = progress.formatElapsedTime;
|
||||
global.estimateRemaining = progress.estimateRemaining;
|
||||
global.calculateRate = progress.calculateRate;
|
||||
global.outputProgress = progress.outputProgress;
|
||||
global.clearProgress = progress.clearProgress;
|
||||
global.getProgress = progress.getProgress;
|
||||
global.logError = progress.logError;
|
||||
|
||||
const { getConnection, closePool } = require('./metrics/utils/db');
|
||||
const calculateProductMetrics = require('./metrics/product-metrics');
|
||||
const calculateTimeAggregates = require('./metrics/time-aggregates');
|
||||
const calculateFinancialMetrics = require('./metrics/financial-metrics');
|
||||
const calculateVendorMetrics = require('./metrics/vendor-metrics');
|
||||
const calculateCategoryMetrics = require('./metrics/category-metrics');
|
||||
const calculateBrandMetrics = require('./metrics/brand-metrics');
|
||||
const calculateSalesForecasts = require('./metrics/sales-forecasts');
|
||||
|
||||
// Add cancel handler
|
||||
let isCancelled = false;
|
||||
|
||||
function cancelCalculation() {
|
||||
isCancelled = true;
|
||||
global.clearProgress();
|
||||
// Format as SSE event
|
||||
const event = {
|
||||
progress: {
|
||||
status: 'cancelled',
|
||||
operation: 'Calculation cancelled',
|
||||
current: 0,
|
||||
total: 0,
|
||||
elapsed: null,
|
||||
remaining: null,
|
||||
rate: 0,
|
||||
timestamp: Date.now()
|
||||
}
|
||||
};
|
||||
process.stdout.write(JSON.stringify(event) + '\n');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Handle SIGTERM signal for cancellation
|
||||
process.on('SIGTERM', cancelCalculation);
|
||||
|
||||
// Update the main calculation function to use the new modular structure
|
||||
async function calculateMetrics() {
|
||||
let connection;
|
||||
const startTime = Date.now();
|
||||
let processedCount = 0;
|
||||
let totalProducts = 0;
|
||||
|
||||
try {
|
||||
// Add debug logging for the progress functions
|
||||
console.log('Debug - Progress functions:', {
|
||||
formatElapsedTime: typeof global.formatElapsedTime,
|
||||
estimateRemaining: typeof global.estimateRemaining,
|
||||
calculateRate: typeof global.calculateRate,
|
||||
startTime: startTime
|
||||
});
|
||||
|
||||
try {
|
||||
const elapsed = global.formatElapsedTime(startTime);
|
||||
console.log('Debug - formatElapsedTime test successful:', elapsed);
|
||||
} catch (err) {
|
||||
console.error('Debug - Error testing formatElapsedTime:', err);
|
||||
throw err;
|
||||
}
|
||||
|
||||
isCancelled = false;
|
||||
connection = await getConnection();
|
||||
|
||||
try {
|
||||
global.outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Starting metrics calculation',
|
||||
current: 0,
|
||||
total: 100,
|
||||
elapsed: '0s',
|
||||
remaining: 'Calculating...',
|
||||
rate: 0,
|
||||
percentage: '0'
|
||||
});
|
||||
|
||||
// Get total number of products
|
||||
const [countResult] = await connection.query('SELECT COUNT(*) as total FROM products')
|
||||
.catch(err => {
|
||||
global.logError(err, 'Failed to count products');
|
||||
throw err;
|
||||
});
|
||||
totalProducts = countResult[0].total;
|
||||
|
||||
if (!SKIP_PRODUCT_METRICS) {
|
||||
processedCount = await calculateProductMetrics(startTime, totalProducts);
|
||||
} else {
|
||||
console.log('Skipping product metrics calculation...');
|
||||
processedCount = Math.floor(totalProducts * 0.6);
|
||||
global.outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Skipping product metrics calculation',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: global.formatElapsedTime(startTime),
|
||||
remaining: global.estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: global.calculateRate(startTime, processedCount),
|
||||
percentage: '60'
|
||||
});
|
||||
}
|
||||
|
||||
// Calculate time-based aggregates
|
||||
if (!SKIP_TIME_AGGREGATES) {
|
||||
processedCount = await calculateTimeAggregates(startTime, totalProducts, processedCount);
|
||||
} else {
|
||||
console.log('Skipping time aggregates calculation');
|
||||
}
|
||||
|
||||
// Calculate financial metrics
|
||||
if (!SKIP_FINANCIAL_METRICS) {
|
||||
processedCount = await calculateFinancialMetrics(startTime, totalProducts, processedCount);
|
||||
} else {
|
||||
console.log('Skipping financial metrics calculation');
|
||||
}
|
||||
|
||||
// Calculate vendor metrics
|
||||
if (!SKIP_VENDOR_METRICS) {
|
||||
processedCount = await calculateVendorMetrics(startTime, totalProducts, processedCount);
|
||||
} else {
|
||||
console.log('Skipping vendor metrics calculation');
|
||||
}
|
||||
|
||||
// Calculate category metrics
|
||||
if (!SKIP_CATEGORY_METRICS) {
|
||||
processedCount = await calculateCategoryMetrics(startTime, totalProducts, processedCount);
|
||||
} else {
|
||||
console.log('Skipping category metrics calculation');
|
||||
}
|
||||
|
||||
// Calculate brand metrics
|
||||
if (!SKIP_BRAND_METRICS) {
|
||||
processedCount = await calculateBrandMetrics(startTime, totalProducts, processedCount);
|
||||
} else {
|
||||
console.log('Skipping brand metrics calculation');
|
||||
}
|
||||
|
||||
// Calculate sales forecasts
|
||||
if (!SKIP_SALES_FORECASTS) {
|
||||
processedCount = await calculateSalesForecasts(startTime, totalProducts, processedCount);
|
||||
} else {
|
||||
console.log('Skipping sales forecasts calculation');
|
||||
}
|
||||
|
||||
// Calculate ABC classification
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Starting ABC classification',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
if (isCancelled) return processedCount;
|
||||
|
||||
const [abcConfig] = await connection.query('SELECT a_threshold, b_threshold FROM abc_classification_config WHERE id = 1');
|
||||
const abcThresholds = abcConfig[0] || { a_threshold: 20, b_threshold: 50 };
|
||||
|
||||
// First, create and populate the rankings table with an index
|
||||
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_revenue_ranks');
|
||||
await connection.query(`
|
||||
CREATE TEMPORARY TABLE temp_revenue_ranks (
|
||||
pid BIGINT NOT NULL,
|
||||
total_revenue DECIMAL(10,3),
|
||||
rank_num INT,
|
||||
total_count INT,
|
||||
PRIMARY KEY (pid),
|
||||
INDEX (rank_num)
|
||||
) ENGINE=MEMORY
|
||||
`);
|
||||
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Creating revenue rankings',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
if (isCancelled) return processedCount;
|
||||
|
||||
await connection.query(`
|
||||
INSERT INTO temp_revenue_ranks
|
||||
SELECT
|
||||
pid,
|
||||
total_revenue,
|
||||
@rank := @rank + 1 as rank_num,
|
||||
@total_count := @rank as total_count
|
||||
FROM (
|
||||
SELECT pid, total_revenue
|
||||
FROM product_metrics
|
||||
WHERE total_revenue > 0
|
||||
ORDER BY total_revenue DESC
|
||||
) ranked,
|
||||
(SELECT @rank := 0) r
|
||||
`);
|
||||
|
||||
// Get total count for percentage calculation
|
||||
const [rankingCount] = await connection.query('SELECT MAX(rank_num) as total_count FROM temp_revenue_ranks');
|
||||
const totalCount = rankingCount[0].total_count || 1;
|
||||
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Updating ABC classifications',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
if (isCancelled) return processedCount;
|
||||
|
||||
// Process updates in batches
|
||||
let abcProcessedCount = 0;
|
||||
const batchSize = 5000;
|
||||
|
||||
while (true) {
|
||||
if (isCancelled) return processedCount;
|
||||
|
||||
// First get a batch of PIDs that need updating
|
||||
const [pids] = await connection.query(`
|
||||
SELECT pm.pid
|
||||
FROM product_metrics pm
|
||||
LEFT JOIN temp_revenue_ranks tr ON pm.pid = tr.pid
|
||||
WHERE pm.abc_class IS NULL
|
||||
OR pm.abc_class !=
|
||||
CASE
|
||||
WHEN tr.rank_num IS NULL THEN 'C'
|
||||
WHEN (tr.rank_num / ?) * 100 <= ? THEN 'A'
|
||||
WHEN (tr.rank_num / ?) * 100 <= ? THEN 'B'
|
||||
ELSE 'C'
|
||||
END
|
||||
LIMIT ?
|
||||
`, [totalCount, abcThresholds.a_threshold,
|
||||
totalCount, abcThresholds.b_threshold,
|
||||
batchSize]);
|
||||
|
||||
if (pids.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Then update just those PIDs
|
||||
const [result] = await connection.query(`
|
||||
UPDATE product_metrics pm
|
||||
LEFT JOIN temp_revenue_ranks tr ON pm.pid = tr.pid
|
||||
SET pm.abc_class =
|
||||
CASE
|
||||
WHEN tr.rank_num IS NULL THEN 'C'
|
||||
WHEN (tr.rank_num / ?) * 100 <= ? THEN 'A'
|
||||
WHEN (tr.rank_num / ?) * 100 <= ? THEN 'B'
|
||||
ELSE 'C'
|
||||
END,
|
||||
pm.last_calculated_at = NOW()
|
||||
WHERE pm.pid IN (?)
|
||||
`, [totalCount, abcThresholds.a_threshold,
|
||||
totalCount, abcThresholds.b_threshold,
|
||||
pids.map(row => row.pid)]);
|
||||
|
||||
abcProcessedCount += result.affectedRows;
|
||||
processedCount = Math.floor(totalProducts * (0.99 + (abcProcessedCount / totalCount) * 0.01));
|
||||
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'ABC classification progress',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
// Small delay between batches to allow other transactions
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
}
|
||||
|
||||
// Clean up
|
||||
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_revenue_ranks');
|
||||
|
||||
// Final success message
|
||||
outputProgress({
|
||||
status: 'complete',
|
||||
operation: 'Metrics calculation complete',
|
||||
current: totalProducts,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: '0s',
|
||||
rate: calculateRate(startTime, totalProducts),
|
||||
percentage: '100'
|
||||
});
|
||||
|
||||
// Clear progress file on successful completion
|
||||
global.clearProgress();
|
||||
|
||||
} catch (error) {
|
||||
if (isCancelled) {
|
||||
global.outputProgress({
|
||||
status: 'cancelled',
|
||||
operation: 'Calculation cancelled',
|
||||
current: processedCount,
|
||||
total: totalProducts || 0,
|
||||
elapsed: global.formatElapsedTime(startTime),
|
||||
remaining: null,
|
||||
rate: global.calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / (totalProducts || 1)) * 100).toFixed(1)
|
||||
});
|
||||
} else {
|
||||
global.outputProgress({
|
||||
status: 'error',
|
||||
operation: 'Error: ' + error.message,
|
||||
current: processedCount,
|
||||
total: totalProducts || 0,
|
||||
elapsed: global.formatElapsedTime(startTime),
|
||||
remaining: null,
|
||||
rate: global.calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / (totalProducts || 1)) * 100).toFixed(1)
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
if (connection) {
|
||||
connection.release();
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
// Close the connection pool when we're done
|
||||
await closePool();
|
||||
}
|
||||
}
|
||||
|
||||
// Export both functions and progress checker
|
||||
module.exports = calculateMetrics;
|
||||
module.exports.cancelCalculation = cancelCalculation;
|
||||
module.exports.getProgress = global.getProgress;
|
||||
|
||||
// Run directly if called from command line
|
||||
if (require.main === module) {
|
||||
calculateMetrics().catch(error => {
|
||||
if (!error.message.includes('Operation cancelled')) {
|
||||
console.error('Error:', error);
|
||||
}
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -1,308 +0,0 @@
|
||||
const dotenv = require("dotenv");
|
||||
const path = require("path");
|
||||
const { outputProgress, formatElapsedTime } = require('./metrics/utils/progress');
|
||||
const { setupConnections, closeConnections } = require('./import/utils');
|
||||
const importCategories = require('./import/categories');
|
||||
const { importProducts } = require('./import/products');
|
||||
const importOrders = require('./import/orders');
|
||||
const importPurchaseOrders = require('./import/purchase-orders');
|
||||
|
||||
dotenv.config({ path: path.join(__dirname, "../.env") });
|
||||
|
||||
// Constants to control which imports run
|
||||
const IMPORT_CATEGORIES = true;
|
||||
const IMPORT_PRODUCTS = true;
|
||||
const IMPORT_ORDERS = true;
|
||||
const IMPORT_PURCHASE_ORDERS = true;
|
||||
|
||||
// Add flag for incremental updates
|
||||
const INCREMENTAL_UPDATE = process.env.INCREMENTAL_UPDATE !== 'false'; // Default to true unless explicitly set to false
|
||||
|
||||
// SSH configuration
|
||||
// In import-from-prod.js
|
||||
const sshConfig = {
|
||||
ssh: {
|
||||
host: process.env.PROD_SSH_HOST,
|
||||
port: process.env.PROD_SSH_PORT || 22,
|
||||
username: process.env.PROD_SSH_USER,
|
||||
privateKey: process.env.PROD_SSH_KEY_PATH
|
||||
? require("fs").readFileSync(process.env.PROD_SSH_KEY_PATH)
|
||||
: undefined,
|
||||
compress: true, // Enable SSH compression
|
||||
},
|
||||
prodDbConfig: {
|
||||
host: process.env.PROD_DB_HOST || "localhost",
|
||||
user: process.env.PROD_DB_USER,
|
||||
password: process.env.PROD_DB_PASSWORD,
|
||||
database: process.env.PROD_DB_NAME,
|
||||
port: process.env.PROD_DB_PORT || 3306,
|
||||
timezone: 'Z',
|
||||
},
|
||||
localDbConfig: {
|
||||
host: process.env.DB_HOST,
|
||||
user: process.env.DB_USER,
|
||||
password: process.env.DB_PASSWORD,
|
||||
database: process.env.DB_NAME,
|
||||
multipleStatements: true,
|
||||
waitForConnections: true,
|
||||
connectionLimit: 10,
|
||||
queueLimit: 0,
|
||||
namedPlaceholders: true,
|
||||
connectTimeout: 60000,
|
||||
enableKeepAlive: true,
|
||||
keepAliveInitialDelay: 10000,
|
||||
compress: true,
|
||||
timezone: 'Z',
|
||||
stringifyObjects: false,
|
||||
}
|
||||
};
|
||||
|
||||
let isImportCancelled = false;
|
||||
|
||||
// Add cancel function
|
||||
function cancelImport() {
|
||||
isImportCancelled = true;
|
||||
outputProgress({
|
||||
status: 'cancelled',
|
||||
operation: 'Import process',
|
||||
message: 'Import cancelled by user',
|
||||
current: 0,
|
||||
total: 0,
|
||||
elapsed: null,
|
||||
remaining: null,
|
||||
rate: 0
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const startTime = Date.now();
|
||||
let connections;
|
||||
let completedSteps = 0;
|
||||
let importHistoryId;
|
||||
const totalSteps = [
|
||||
IMPORT_CATEGORIES,
|
||||
IMPORT_PRODUCTS,
|
||||
IMPORT_ORDERS,
|
||||
IMPORT_PURCHASE_ORDERS
|
||||
].filter(Boolean).length;
|
||||
|
||||
try {
|
||||
// Initial progress update
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Import process",
|
||||
message: `Initializing SSH tunnel for ${INCREMENTAL_UPDATE ? 'incremental' : 'full'} import...`,
|
||||
current: completedSteps,
|
||||
total: totalSteps,
|
||||
elapsed: formatElapsedTime(startTime)
|
||||
});
|
||||
|
||||
connections = await setupConnections(sshConfig);
|
||||
const { prodConnection, localConnection } = connections;
|
||||
|
||||
if (isImportCancelled) throw new Error("Import cancelled");
|
||||
|
||||
// Clean up any previously running imports that weren't completed
|
||||
await localConnection.query(`
|
||||
UPDATE import_history
|
||||
SET
|
||||
status = 'cancelled',
|
||||
end_time = NOW(),
|
||||
duration_seconds = TIMESTAMPDIFF(SECOND, start_time, NOW()),
|
||||
error_message = 'Previous import was not completed properly'
|
||||
WHERE status = 'running'
|
||||
`);
|
||||
|
||||
// Initialize sync_status table if it doesn't exist
|
||||
await localConnection.query(`
|
||||
CREATE TABLE IF NOT EXISTS sync_status (
|
||||
table_name VARCHAR(50) PRIMARY KEY,
|
||||
last_sync_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
last_sync_id BIGINT,
|
||||
INDEX idx_last_sync (last_sync_timestamp)
|
||||
);
|
||||
`);
|
||||
|
||||
// Create import history record for the overall session
|
||||
const [historyResult] = await localConnection.query(`
|
||||
INSERT INTO import_history (
|
||||
table_name,
|
||||
start_time,
|
||||
is_incremental,
|
||||
status,
|
||||
additional_info
|
||||
) VALUES (
|
||||
'all_tables',
|
||||
NOW(),
|
||||
?,
|
||||
'running',
|
||||
JSON_OBJECT(
|
||||
'categories_enabled', ?,
|
||||
'products_enabled', ?,
|
||||
'orders_enabled', ?,
|
||||
'purchase_orders_enabled', ?
|
||||
)
|
||||
)
|
||||
`, [INCREMENTAL_UPDATE, IMPORT_CATEGORIES, IMPORT_PRODUCTS, IMPORT_ORDERS, IMPORT_PURCHASE_ORDERS]);
|
||||
importHistoryId = historyResult.insertId;
|
||||
|
||||
const results = {
|
||||
categories: null,
|
||||
products: null,
|
||||
orders: null,
|
||||
purchaseOrders: null
|
||||
};
|
||||
|
||||
let totalRecordsAdded = 0;
|
||||
let totalRecordsUpdated = 0;
|
||||
|
||||
// Run each import based on constants
|
||||
if (IMPORT_CATEGORIES) {
|
||||
results.categories = await importCategories(prodConnection, localConnection);
|
||||
if (isImportCancelled) throw new Error("Import cancelled");
|
||||
completedSteps++;
|
||||
console.log('Categories import result:', results.categories);
|
||||
totalRecordsAdded += results.categories?.recordsAdded || 0;
|
||||
totalRecordsUpdated += results.categories?.recordsUpdated || 0;
|
||||
}
|
||||
|
||||
if (IMPORT_PRODUCTS) {
|
||||
results.products = await importProducts(prodConnection, localConnection, INCREMENTAL_UPDATE);
|
||||
if (isImportCancelled) throw new Error("Import cancelled");
|
||||
completedSteps++;
|
||||
console.log('Products import result:', results.products);
|
||||
totalRecordsAdded += results.products?.recordsAdded || 0;
|
||||
totalRecordsUpdated += results.products?.recordsUpdated || 0;
|
||||
}
|
||||
|
||||
if (IMPORT_ORDERS) {
|
||||
results.orders = await importOrders(prodConnection, localConnection, INCREMENTAL_UPDATE);
|
||||
if (isImportCancelled) throw new Error("Import cancelled");
|
||||
completedSteps++;
|
||||
console.log('Orders import result:', results.orders);
|
||||
totalRecordsAdded += results.orders?.recordsAdded || 0;
|
||||
totalRecordsUpdated += results.orders?.recordsUpdated || 0;
|
||||
}
|
||||
|
||||
if (IMPORT_PURCHASE_ORDERS) {
|
||||
results.purchaseOrders = await importPurchaseOrders(prodConnection, localConnection, INCREMENTAL_UPDATE);
|
||||
if (isImportCancelled) throw new Error("Import cancelled");
|
||||
completedSteps++;
|
||||
console.log('Purchase orders import result:', results.purchaseOrders);
|
||||
totalRecordsAdded += results.purchaseOrders?.recordsAdded || 0;
|
||||
totalRecordsUpdated += results.purchaseOrders?.recordsUpdated || 0;
|
||||
}
|
||||
|
||||
const endTime = Date.now();
|
||||
const totalElapsedSeconds = Math.round((endTime - startTime) / 1000);
|
||||
|
||||
// Update import history with final stats
|
||||
await localConnection.query(`
|
||||
UPDATE import_history
|
||||
SET
|
||||
end_time = NOW(),
|
||||
duration_seconds = ?,
|
||||
records_added = ?,
|
||||
records_updated = ?,
|
||||
status = 'completed',
|
||||
additional_info = JSON_OBJECT(
|
||||
'categories_enabled', ?,
|
||||
'products_enabled', ?,
|
||||
'orders_enabled', ?,
|
||||
'purchase_orders_enabled', ?,
|
||||
'categories_result', CAST(? AS JSON),
|
||||
'products_result', CAST(? AS JSON),
|
||||
'orders_result', CAST(? AS JSON),
|
||||
'purchase_orders_result', CAST(? AS JSON)
|
||||
)
|
||||
WHERE id = ?
|
||||
`, [
|
||||
totalElapsedSeconds,
|
||||
totalRecordsAdded,
|
||||
totalRecordsUpdated,
|
||||
IMPORT_CATEGORIES,
|
||||
IMPORT_PRODUCTS,
|
||||
IMPORT_ORDERS,
|
||||
IMPORT_PURCHASE_ORDERS,
|
||||
JSON.stringify(results.categories),
|
||||
JSON.stringify(results.products),
|
||||
JSON.stringify(results.orders),
|
||||
JSON.stringify(results.purchaseOrders),
|
||||
importHistoryId
|
||||
]);
|
||||
|
||||
outputProgress({
|
||||
status: "complete",
|
||||
operation: "Import process",
|
||||
message: `${INCREMENTAL_UPDATE ? 'Incremental' : 'Full'} import completed successfully in ${formatElapsedTime(totalElapsedSeconds)}`,
|
||||
current: completedSteps,
|
||||
total: totalSteps,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
timing: {
|
||||
start_time: new Date(startTime).toISOString(),
|
||||
end_time: new Date(endTime).toISOString(),
|
||||
elapsed_time: formatElapsedTime(startTime),
|
||||
elapsed_seconds: totalElapsedSeconds,
|
||||
total_duration: formatElapsedTime(totalElapsedSeconds)
|
||||
},
|
||||
results
|
||||
});
|
||||
|
||||
return results;
|
||||
} catch (error) {
|
||||
const endTime = Date.now();
|
||||
const totalElapsedSeconds = Math.round((endTime - startTime) / 1000);
|
||||
|
||||
// Update import history with error
|
||||
if (importHistoryId && connections?.localConnection) {
|
||||
await connections.localConnection.query(`
|
||||
UPDATE import_history
|
||||
SET
|
||||
end_time = NOW(),
|
||||
duration_seconds = ?,
|
||||
status = ?,
|
||||
error_message = ?
|
||||
WHERE id = ?
|
||||
`, [totalElapsedSeconds, error.message === "Import cancelled" ? 'cancelled' : 'failed', error.message, importHistoryId]);
|
||||
}
|
||||
|
||||
console.error("Error during import process:", error);
|
||||
outputProgress({
|
||||
status: error.message === "Import cancelled" ? "cancelled" : "error",
|
||||
operation: "Import process",
|
||||
message: error.message === "Import cancelled"
|
||||
? `${INCREMENTAL_UPDATE ? 'Incremental' : 'Full'} import cancelled by user after ${formatElapsedTime(totalElapsedSeconds)}`
|
||||
: `${INCREMENTAL_UPDATE ? 'Incremental' : 'Full'} import failed after ${formatElapsedTime(totalElapsedSeconds)}`,
|
||||
error: error.message,
|
||||
current: completedSteps,
|
||||
total: totalSteps,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
timing: {
|
||||
start_time: new Date(startTime).toISOString(),
|
||||
end_time: new Date(endTime).toISOString(),
|
||||
elapsed_time: formatElapsedTime(startTime),
|
||||
elapsed_seconds: totalElapsedSeconds,
|
||||
total_duration: formatElapsedTime(totalElapsedSeconds)
|
||||
}
|
||||
});
|
||||
throw error;
|
||||
} finally {
|
||||
if (connections) {
|
||||
await closeConnections(connections);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Run the import only if this is the main module
|
||||
if (require.main === module) {
|
||||
main().catch((error) => {
|
||||
console.error("Unhandled error in main process:", error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
// Export the functions needed by the route
|
||||
module.exports = {
|
||||
main,
|
||||
cancelImport,
|
||||
};
|
||||
@@ -1,182 +0,0 @@
|
||||
const { outputProgress, formatElapsedTime } = require('../metrics/utils/progress');
|
||||
|
||||
async function importCategories(prodConnection, localConnection) {
|
||||
outputProgress({
|
||||
operation: "Starting categories import",
|
||||
status: "running",
|
||||
});
|
||||
|
||||
const startTime = Date.now();
|
||||
const typeOrder = [10, 20, 11, 21, 12, 13];
|
||||
let totalInserted = 0;
|
||||
let skippedCategories = [];
|
||||
|
||||
try {
|
||||
// Process each type in order with its own query
|
||||
for (const type of typeOrder) {
|
||||
const [categories] = await prodConnection.query(
|
||||
`
|
||||
SELECT
|
||||
pc.cat_id,
|
||||
pc.name,
|
||||
pc.type,
|
||||
CASE
|
||||
WHEN pc.type IN (10, 20) THEN NULL -- Top level categories should have no parent
|
||||
WHEN pc.master_cat_id IS NULL THEN NULL
|
||||
ELSE pc.master_cat_id
|
||||
END as parent_id,
|
||||
pc.combined_name as description
|
||||
FROM product_categories pc
|
||||
WHERE pc.type = ?
|
||||
ORDER BY pc.cat_id
|
||||
`,
|
||||
[type]
|
||||
);
|
||||
|
||||
if (categories.length === 0) continue;
|
||||
|
||||
console.log(`\nProcessing ${categories.length} type ${type} categories`);
|
||||
if (type === 10) {
|
||||
console.log("Type 10 categories:", JSON.stringify(categories, null, 2));
|
||||
}
|
||||
|
||||
// For types that can have parents (11, 21, 12, 13), verify parent existence
|
||||
let categoriesToInsert = categories;
|
||||
if (![10, 20].includes(type)) {
|
||||
// Get all parent IDs
|
||||
const parentIds = [
|
||||
...new Set(
|
||||
categories.map((c) => c.parent_id).filter((id) => id !== null)
|
||||
),
|
||||
];
|
||||
|
||||
// Check which parents exist
|
||||
const [existingParents] = await localConnection.query(
|
||||
"SELECT cat_id FROM categories WHERE cat_id IN (?)",
|
||||
[parentIds]
|
||||
);
|
||||
const existingParentIds = new Set(existingParents.map((p) => p.cat_id));
|
||||
|
||||
// Filter categories and track skipped ones
|
||||
categoriesToInsert = categories.filter(
|
||||
(cat) =>
|
||||
cat.parent_id === null || existingParentIds.has(cat.parent_id)
|
||||
);
|
||||
const invalidCategories = categories.filter(
|
||||
(cat) =>
|
||||
cat.parent_id !== null && !existingParentIds.has(cat.parent_id)
|
||||
);
|
||||
|
||||
if (invalidCategories.length > 0) {
|
||||
const skippedInfo = invalidCategories.map((c) => ({
|
||||
id: c.cat_id,
|
||||
name: c.name,
|
||||
type: c.type,
|
||||
missing_parent: c.parent_id,
|
||||
}));
|
||||
skippedCategories.push(...skippedInfo);
|
||||
|
||||
console.log(
|
||||
"\nSkipping categories with missing parents:",
|
||||
invalidCategories
|
||||
.map(
|
||||
(c) =>
|
||||
`${c.cat_id} - ${c.name} (missing parent: ${c.parent_id})`
|
||||
)
|
||||
.join("\n")
|
||||
);
|
||||
}
|
||||
|
||||
if (categoriesToInsert.length === 0) {
|
||||
console.log(
|
||||
`No valid categories of type ${type} to insert - all had missing parents`
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(
|
||||
`Inserting ${categoriesToInsert.length} type ${type} categories`
|
||||
);
|
||||
|
||||
const placeholders = categoriesToInsert
|
||||
.map(() => "(?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)")
|
||||
.join(",");
|
||||
|
||||
const values = categoriesToInsert.flatMap((cat) => [
|
||||
cat.cat_id,
|
||||
cat.name,
|
||||
cat.type,
|
||||
cat.parent_id,
|
||||
cat.description,
|
||||
"active",
|
||||
]);
|
||||
|
||||
// Insert categories and create relationships in one query to avoid race conditions
|
||||
await localConnection.query(
|
||||
`
|
||||
INSERT INTO categories (cat_id, name, type, parent_id, description, status, created_at, updated_at)
|
||||
VALUES ${placeholders}
|
||||
ON DUPLICATE KEY UPDATE
|
||||
name = VALUES(name),
|
||||
type = VALUES(type),
|
||||
parent_id = VALUES(parent_id),
|
||||
description = VALUES(description),
|
||||
status = VALUES(status),
|
||||
updated_at = CURRENT_TIMESTAMP
|
||||
`,
|
||||
values
|
||||
);
|
||||
|
||||
totalInserted += categoriesToInsert.length;
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Categories import",
|
||||
current: totalInserted,
|
||||
total: totalInserted,
|
||||
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
|
||||
});
|
||||
}
|
||||
|
||||
// After all imports, if we skipped any categories, throw an error
|
||||
if (skippedCategories.length > 0) {
|
||||
const error = new Error(
|
||||
"Categories import completed with errors - some categories were skipped due to missing parents"
|
||||
);
|
||||
error.skippedCategories = skippedCategories;
|
||||
throw error;
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
status: "complete",
|
||||
operation: "Categories import completed",
|
||||
current: totalInserted,
|
||||
total: totalInserted,
|
||||
duration: formatElapsedTime((Date.now() - startTime) / 1000),
|
||||
});
|
||||
|
||||
return {
|
||||
status: "complete",
|
||||
totalImported: totalInserted
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("Error importing categories:", error);
|
||||
if (error.skippedCategories) {
|
||||
console.error(
|
||||
"Skipped categories:",
|
||||
JSON.stringify(error.skippedCategories, null, 2)
|
||||
);
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
status: "error",
|
||||
operation: "Categories import failed",
|
||||
error: error.message,
|
||||
skippedCategories: error.skippedCategories
|
||||
});
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = importCategories;
|
||||
@@ -1,568 +0,0 @@
|
||||
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics/utils/progress');
|
||||
const { importMissingProducts, setupTemporaryTables, cleanupTemporaryTables, materializeCalculations } = require('./products');
|
||||
|
||||
/**
|
||||
* Imports orders from a production MySQL database to a local MySQL database.
|
||||
* It can run in two modes:
|
||||
* 1. Incremental update mode (default): Only fetch orders that have changed since the last sync time.
|
||||
* 2. Full update mode: Fetch all eligible orders within the last 5 years regardless of timestamp.
|
||||
*
|
||||
* @param {object} prodConnection - A MySQL connection to production DB (MySQL 5.7).
|
||||
* @param {object} localConnection - A MySQL connection to local DB (MySQL 8.0).
|
||||
* @param {boolean} incrementalUpdate - Set to false for a full sync; true for incremental.
|
||||
*
|
||||
* @returns {object} Information about the sync operation.
|
||||
*/
|
||||
async function importOrders(prodConnection, localConnection, incrementalUpdate = true) {
|
||||
const startTime = Date.now();
|
||||
const skippedOrders = new Set();
|
||||
const missingProducts = new Set();
|
||||
let recordsAdded = 0;
|
||||
let recordsUpdated = 0;
|
||||
let processedCount = 0;
|
||||
let importedCount = 0;
|
||||
let totalOrderItems = 0;
|
||||
let totalUniqueOrders = 0;
|
||||
|
||||
// Add a cumulative counter for processed orders before the loop
|
||||
let cumulativeProcessedOrders = 0;
|
||||
|
||||
try {
|
||||
// Insert temporary table creation queries
|
||||
await localConnection.query(`
|
||||
CREATE TABLE IF NOT EXISTS temp_order_items (
|
||||
order_id INT UNSIGNED NOT NULL,
|
||||
pid INT UNSIGNED NOT NULL,
|
||||
SKU VARCHAR(50) NOT NULL,
|
||||
price DECIMAL(10,2) NOT NULL,
|
||||
quantity INT NOT NULL,
|
||||
base_discount DECIMAL(10,2) DEFAULT 0,
|
||||
PRIMARY KEY (order_id, pid)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
||||
`);
|
||||
await localConnection.query(`
|
||||
CREATE TABLE IF NOT EXISTS temp_order_meta (
|
||||
order_id INT UNSIGNED NOT NULL,
|
||||
date DATE NOT NULL,
|
||||
customer VARCHAR(100) NOT NULL,
|
||||
customer_name VARCHAR(150) NOT NULL,
|
||||
status INT,
|
||||
canceled TINYINT(1),
|
||||
PRIMARY KEY (order_id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
||||
`);
|
||||
await localConnection.query(`
|
||||
CREATE TABLE IF NOT EXISTS temp_order_discounts (
|
||||
order_id INT UNSIGNED NOT NULL,
|
||||
pid INT UNSIGNED NOT NULL,
|
||||
discount DECIMAL(10,2) NOT NULL,
|
||||
PRIMARY KEY (order_id, pid)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
||||
`);
|
||||
await localConnection.query(`
|
||||
CREATE TABLE IF NOT EXISTS temp_order_taxes (
|
||||
order_id INT UNSIGNED NOT NULL,
|
||||
pid INT UNSIGNED NOT NULL,
|
||||
tax DECIMAL(10,2) NOT NULL,
|
||||
PRIMARY KEY (order_id, pid)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
||||
`);
|
||||
await localConnection.query(`
|
||||
CREATE TABLE IF NOT EXISTS temp_order_costs (
|
||||
order_id INT UNSIGNED NOT NULL,
|
||||
pid INT UNSIGNED NOT NULL,
|
||||
costeach DECIMAL(10,3) DEFAULT 0.000,
|
||||
PRIMARY KEY (order_id, pid)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
||||
`);
|
||||
|
||||
// Get column names from the local table
|
||||
const [columns] = await localConnection.query(`
|
||||
SELECT COLUMN_NAME
|
||||
FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_NAME = 'orders'
|
||||
ORDER BY ORDINAL_POSITION
|
||||
`);
|
||||
const columnNames = columns.map(col => col.COLUMN_NAME);
|
||||
|
||||
// Get last sync info
|
||||
const [syncInfo] = await localConnection.query(
|
||||
"SELECT last_sync_timestamp FROM sync_status WHERE table_name = 'orders'"
|
||||
);
|
||||
const lastSyncTime = syncInfo?.[0]?.last_sync_timestamp || '1970-01-01';
|
||||
|
||||
console.log('Orders: Using last sync time:', lastSyncTime);
|
||||
|
||||
// First get count of order items
|
||||
const [[{ total }]] = await prodConnection.query(`
|
||||
SELECT COUNT(*) as total
|
||||
FROM order_items oi
|
||||
USE INDEX (PRIMARY)
|
||||
JOIN _order o ON oi.order_id = o.order_id
|
||||
WHERE o.order_status >= 15
|
||||
AND o.date_placed_onlydate >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
|
||||
AND o.date_placed_onlydate IS NOT NULL
|
||||
${incrementalUpdate ? `
|
||||
AND (
|
||||
o.stamp > ?
|
||||
OR oi.stamp > ?
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM order_discount_items odi
|
||||
WHERE odi.order_id = o.order_id
|
||||
AND odi.pid = oi.prod_pid
|
||||
)
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM order_tax_info oti
|
||||
JOIN order_tax_info_products otip ON oti.taxinfo_id = otip.taxinfo_id
|
||||
WHERE oti.order_id = o.order_id
|
||||
AND otip.pid = oi.prod_pid
|
||||
AND oti.stamp > ?
|
||||
)
|
||||
)
|
||||
` : ''}
|
||||
`, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime] : []);
|
||||
|
||||
totalOrderItems = total;
|
||||
console.log('Orders: Found changes:', totalOrderItems);
|
||||
|
||||
// Get order items in batches
|
||||
const [orderItems] = await prodConnection.query(`
|
||||
SELECT
|
||||
oi.order_id,
|
||||
oi.prod_pid as pid,
|
||||
oi.prod_itemnumber as SKU,
|
||||
oi.prod_price as price,
|
||||
oi.qty_ordered as quantity,
|
||||
COALESCE(oi.prod_price_reg - oi.prod_price, 0) as base_discount,
|
||||
oi.stamp as last_modified
|
||||
FROM order_items oi
|
||||
USE INDEX (PRIMARY)
|
||||
JOIN _order o ON oi.order_id = o.order_id
|
||||
WHERE o.order_status >= 15
|
||||
AND o.date_placed_onlydate >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
|
||||
AND o.date_placed_onlydate IS NOT NULL
|
||||
${incrementalUpdate ? `
|
||||
AND (
|
||||
o.stamp > ?
|
||||
OR oi.stamp > ?
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM order_discount_items odi
|
||||
WHERE odi.order_id = o.order_id
|
||||
AND odi.pid = oi.prod_pid
|
||||
)
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM order_tax_info oti
|
||||
JOIN order_tax_info_products otip ON oti.taxinfo_id = otip.taxinfo_id
|
||||
WHERE oti.order_id = o.order_id
|
||||
AND otip.pid = oi.prod_pid
|
||||
AND oti.stamp > ?
|
||||
)
|
||||
)
|
||||
` : ''}
|
||||
`, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime] : []);
|
||||
|
||||
console.log('Orders: Processing', orderItems.length, 'order items');
|
||||
|
||||
// Insert order items in batches
|
||||
for (let i = 0; i < orderItems.length; i += 5000) {
|
||||
const batch = orderItems.slice(i, Math.min(i + 5000, orderItems.length));
|
||||
const placeholders = batch.map(() => "(?, ?, ?, ?, ?, ?)").join(",");
|
||||
const values = batch.flatMap(item => [
|
||||
item.order_id, item.pid, item.SKU, item.price, item.quantity, item.base_discount
|
||||
]);
|
||||
|
||||
await localConnection.query(`
|
||||
INSERT INTO temp_order_items (order_id, pid, SKU, price, quantity, base_discount)
|
||||
VALUES ${placeholders}
|
||||
ON DUPLICATE KEY UPDATE
|
||||
SKU = VALUES(SKU),
|
||||
price = VALUES(price),
|
||||
quantity = VALUES(quantity),
|
||||
base_discount = VALUES(base_discount)
|
||||
`, values);
|
||||
|
||||
processedCount = i + batch.length;
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Orders import",
|
||||
message: `Loading order items: ${processedCount} of ${totalOrderItems}`,
|
||||
current: processedCount,
|
||||
total: totalOrderItems
|
||||
});
|
||||
}
|
||||
|
||||
// Get unique order IDs
|
||||
const orderIds = [...new Set(orderItems.map(item => item.order_id))];
|
||||
totalUniqueOrders = orderIds.length;
|
||||
console.log('Total unique order IDs:', totalUniqueOrders);
|
||||
|
||||
// Reset processed count for order processing phase
|
||||
processedCount = 0;
|
||||
|
||||
// Get order metadata in batches
|
||||
for (let i = 0; i < orderIds.length; i += 5000) {
|
||||
const batchIds = orderIds.slice(i, i + 5000);
|
||||
console.log(`Processing batch ${i/5000 + 1}, size: ${batchIds.length}`);
|
||||
console.log('Sample of batch IDs:', batchIds.slice(0, 5));
|
||||
|
||||
const [orders] = await prodConnection.query(`
|
||||
SELECT
|
||||
o.order_id,
|
||||
o.date_placed_onlydate as date,
|
||||
o.order_cid as customer,
|
||||
CONCAT(COALESCE(u.firstname, ''), ' ', COALESCE(u.lastname, '')) as customer_name,
|
||||
o.order_status as status,
|
||||
CASE WHEN o.date_cancelled != '0000-00-00 00:00:00' THEN 1 ELSE 0 END as canceled
|
||||
FROM _order o
|
||||
LEFT JOIN users u ON o.order_cid = u.cid
|
||||
WHERE o.order_id IN (?)
|
||||
`, [batchIds]);
|
||||
|
||||
console.log(`Retrieved ${orders.length} orders for ${batchIds.length} IDs`);
|
||||
const duplicates = orders.filter((order, index, self) =>
|
||||
self.findIndex(o => o.order_id === order.order_id) !== index
|
||||
);
|
||||
if (duplicates.length > 0) {
|
||||
console.log('Found duplicates:', duplicates);
|
||||
}
|
||||
|
||||
const placeholders = orders.map(() => "(?, ?, ?, ?, ?, ?)").join(",");
|
||||
const values = orders.flatMap(order => [
|
||||
order.order_id, order.date, order.customer, order.customer_name, order.status, order.canceled
|
||||
]);
|
||||
|
||||
await localConnection.query(`
|
||||
INSERT INTO temp_order_meta VALUES ${placeholders}
|
||||
ON DUPLICATE KEY UPDATE
|
||||
date = VALUES(date),
|
||||
customer = VALUES(customer),
|
||||
customer_name = VALUES(customer_name),
|
||||
status = VALUES(status),
|
||||
canceled = VALUES(canceled)
|
||||
`, values);
|
||||
|
||||
processedCount = i + orders.length;
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Orders import",
|
||||
message: `Loading order metadata: ${processedCount} of ${totalUniqueOrders}`,
|
||||
current: processedCount,
|
||||
total: totalUniqueOrders
|
||||
});
|
||||
}
|
||||
|
||||
// Reset processed count for final phase
|
||||
processedCount = 0;
|
||||
|
||||
// Get promotional discounts in batches
|
||||
for (let i = 0; i < orderIds.length; i += 5000) {
|
||||
const batchIds = orderIds.slice(i, i + 5000);
|
||||
const [discounts] = await prodConnection.query(`
|
||||
SELECT order_id, pid, SUM(amount) as discount
|
||||
FROM order_discount_items
|
||||
WHERE order_id IN (?)
|
||||
GROUP BY order_id, pid
|
||||
`, [batchIds]);
|
||||
|
||||
if (discounts.length > 0) {
|
||||
const placeholders = discounts.map(() => "(?, ?, ?)").join(",");
|
||||
const values = discounts.flatMap(d => [d.order_id, d.pid, d.discount]);
|
||||
|
||||
await localConnection.query(`
|
||||
INSERT INTO temp_order_discounts VALUES ${placeholders}
|
||||
ON DUPLICATE KEY UPDATE
|
||||
discount = VALUES(discount)
|
||||
`, values);
|
||||
}
|
||||
}
|
||||
|
||||
// Get tax information in batches
|
||||
for (let i = 0; i < orderIds.length; i += 5000) {
|
||||
const batchIds = orderIds.slice(i, i + 5000);
|
||||
const [taxes] = await prodConnection.query(`
|
||||
SELECT DISTINCT
|
||||
oti.order_id,
|
||||
otip.pid,
|
||||
otip.item_taxes_to_collect as tax
|
||||
FROM order_tax_info oti
|
||||
JOIN (
|
||||
SELECT order_id, MAX(stamp) as max_stamp
|
||||
FROM order_tax_info
|
||||
WHERE order_id IN (?)
|
||||
GROUP BY order_id
|
||||
) latest ON oti.order_id = latest.order_id AND oti.stamp = latest.max_stamp
|
||||
JOIN order_tax_info_products otip ON oti.taxinfo_id = otip.taxinfo_id
|
||||
`, [batchIds]);
|
||||
|
||||
if (taxes.length > 0) {
|
||||
// Remove any duplicates before inserting
|
||||
const uniqueTaxes = new Map();
|
||||
taxes.forEach(t => {
|
||||
const key = `${t.order_id}-${t.pid}`;
|
||||
uniqueTaxes.set(key, t);
|
||||
});
|
||||
|
||||
const values = Array.from(uniqueTaxes.values()).flatMap(t => [t.order_id, t.pid, t.tax]);
|
||||
if (values.length > 0) {
|
||||
const placeholders = Array(uniqueTaxes.size).fill("(?, ?, ?)").join(",");
|
||||
await localConnection.query(`
|
||||
INSERT INTO temp_order_taxes VALUES ${placeholders}
|
||||
ON DUPLICATE KEY UPDATE tax = VALUES(tax)
|
||||
`, values);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get costeach values in batches
|
||||
for (let i = 0; i < orderIds.length; i += 5000) {
|
||||
const batchIds = orderIds.slice(i, i + 5000);
|
||||
const [costs] = await prodConnection.query(`
|
||||
SELECT orderid as order_id, pid, costeach
|
||||
FROM order_costs
|
||||
WHERE orderid IN (?)
|
||||
`, [batchIds]);
|
||||
|
||||
if (costs.length > 0) {
|
||||
const placeholders = costs.map(() => '(?, ?, ?)').join(",");
|
||||
const values = costs.flatMap(c => [c.order_id, c.pid, c.costeach]);
|
||||
await localConnection.query(`
|
||||
INSERT INTO temp_order_costs (order_id, pid, costeach)
|
||||
VALUES ${placeholders}
|
||||
ON DUPLICATE KEY UPDATE costeach = VALUES(costeach)
|
||||
`, values);
|
||||
}
|
||||
}
|
||||
|
||||
// Now combine all the data and insert into orders table
|
||||
// Pre-check all products at once instead of per batch
|
||||
const allOrderPids = [...new Set(orderItems.map(item => item.pid))];
|
||||
const [existingProducts] = allOrderPids.length > 0 ? await localConnection.query(
|
||||
"SELECT pid FROM products WHERE pid IN (?)",
|
||||
[allOrderPids]
|
||||
) : [[]];
|
||||
const existingPids = new Set(existingProducts.map(p => p.pid));
|
||||
|
||||
// Process in larger batches
|
||||
for (let i = 0; i < orderIds.length; i += 5000) {
|
||||
const batchIds = orderIds.slice(i, i + 5000);
|
||||
|
||||
// Get combined data for this batch
|
||||
const [orders] = await localConnection.query(`
|
||||
SELECT
|
||||
oi.order_id as order_number,
|
||||
oi.pid,
|
||||
oi.SKU,
|
||||
om.date,
|
||||
oi.price,
|
||||
oi.quantity,
|
||||
oi.base_discount + COALESCE(od.discount, 0) as discount,
|
||||
COALESCE(ot.tax, 0) as tax,
|
||||
0 as tax_included,
|
||||
0 as shipping,
|
||||
om.customer,
|
||||
om.customer_name,
|
||||
om.status,
|
||||
om.canceled,
|
||||
COALESCE(tc.costeach, 0) as costeach
|
||||
FROM temp_order_items oi
|
||||
JOIN temp_order_meta om ON oi.order_id = om.order_id
|
||||
LEFT JOIN temp_order_discounts od ON oi.order_id = od.order_id AND oi.pid = od.pid
|
||||
LEFT JOIN temp_order_taxes ot ON oi.order_id = ot.order_id AND oi.pid = ot.pid
|
||||
LEFT JOIN temp_order_costs tc ON oi.order_id = tc.order_id AND oi.pid = tc.pid
|
||||
WHERE oi.order_id IN (?)
|
||||
`, [batchIds]);
|
||||
|
||||
// Filter orders and track missing products - do this in a single pass
|
||||
const validOrders = [];
|
||||
const values = [];
|
||||
const processedOrderItems = new Set(); // Track unique order items
|
||||
const processedOrders = new Set(); // Track unique orders
|
||||
|
||||
for (const order of orders) {
|
||||
if (!existingPids.has(order.pid)) {
|
||||
missingProducts.add(order.pid);
|
||||
skippedOrders.add(order.order_number);
|
||||
continue;
|
||||
}
|
||||
validOrders.push(order);
|
||||
values.push(...columnNames.map(col => order[col] ?? null));
|
||||
processedOrderItems.add(`${order.order_number}-${order.pid}`);
|
||||
processedOrders.add(order.order_number);
|
||||
}
|
||||
|
||||
if (validOrders.length > 0) {
|
||||
// Pre-compute the placeholders string once
|
||||
const singlePlaceholder = `(${columnNames.map(() => "?").join(",")})`;
|
||||
const placeholders = Array(validOrders.length).fill(singlePlaceholder).join(",");
|
||||
|
||||
const result = await localConnection.query(`
|
||||
INSERT INTO orders (${columnNames.join(",")})
|
||||
VALUES ${placeholders}
|
||||
ON DUPLICATE KEY UPDATE
|
||||
SKU = VALUES(SKU),
|
||||
date = VALUES(date),
|
||||
price = VALUES(price),
|
||||
quantity = VALUES(quantity),
|
||||
discount = VALUES(discount),
|
||||
tax = VALUES(tax),
|
||||
tax_included = VALUES(tax_included),
|
||||
shipping = VALUES(shipping),
|
||||
customer = VALUES(customer),
|
||||
customer_name = VALUES(customer_name),
|
||||
status = VALUES(status),
|
||||
canceled = VALUES(canceled),
|
||||
costeach = VALUES(costeach)
|
||||
`, validOrders.map(o => columnNames.map(col => o[col] ?? null)).flat());
|
||||
|
||||
const affectedRows = result[0].affectedRows;
|
||||
const updates = Math.floor(affectedRows / 2);
|
||||
const inserts = affectedRows - (updates * 2);
|
||||
|
||||
recordsAdded += inserts;
|
||||
recordsUpdated += updates;
|
||||
importedCount += processedOrderItems.size; // Count unique order items processed
|
||||
}
|
||||
|
||||
// Update progress based on unique orders processed
|
||||
cumulativeProcessedOrders += processedOrders.size;
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Orders import",
|
||||
message: `Imported ${importedCount} order items (${cumulativeProcessedOrders} of ${totalUniqueOrders} orders processed)`,
|
||||
current: cumulativeProcessedOrders,
|
||||
total: totalUniqueOrders,
|
||||
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
|
||||
remaining: estimateRemaining(startTime, cumulativeProcessedOrders, totalUniqueOrders),
|
||||
rate: calculateRate(startTime, cumulativeProcessedOrders)
|
||||
});
|
||||
}
|
||||
|
||||
// Now try to import any orders that were skipped due to missing products
|
||||
if (skippedOrders.size > 0) {
|
||||
try {
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Orders import",
|
||||
message: `Retrying import of ${skippedOrders.size} orders with previously missing products`,
|
||||
});
|
||||
|
||||
// Get the orders that were skipped
|
||||
const [skippedProdOrders] = await localConnection.query(`
|
||||
SELECT DISTINCT
|
||||
oi.order_id as order_number,
|
||||
oi.pid,
|
||||
oi.SKU,
|
||||
om.date,
|
||||
oi.price,
|
||||
oi.quantity,
|
||||
oi.base_discount + COALESCE(od.discount, 0) as discount,
|
||||
COALESCE(ot.tax, 0) as tax,
|
||||
0 as tax_included,
|
||||
0 as shipping,
|
||||
om.customer,
|
||||
om.customer_name,
|
||||
om.status,
|
||||
om.canceled,
|
||||
COALESCE(tc.costeach, 0) as costeach
|
||||
FROM temp_order_items oi
|
||||
JOIN temp_order_meta om ON oi.order_id = om.order_id
|
||||
LEFT JOIN temp_order_discounts od ON oi.order_id = od.order_id AND oi.pid = od.pid
|
||||
LEFT JOIN temp_order_taxes ot ON oi.order_id = ot.order_id AND oi.pid = ot.pid
|
||||
LEFT JOIN temp_order_costs tc ON oi.order_id = tc.order_id AND oi.pid = tc.pid
|
||||
WHERE oi.order_id IN (?)
|
||||
`, [Array.from(skippedOrders)]);
|
||||
|
||||
// Check which products exist now
|
||||
const skippedPids = [...new Set(skippedProdOrders.map(o => o.pid))];
|
||||
const [existingProducts] = skippedPids.length > 0 ? await localConnection.query(
|
||||
"SELECT pid FROM products WHERE pid IN (?)",
|
||||
[skippedPids]
|
||||
) : [[]];
|
||||
const existingPids = new Set(existingProducts.map(p => p.pid));
|
||||
|
||||
// Filter orders that can now be imported
|
||||
const validOrders = skippedProdOrders.filter(order => existingPids.has(order.pid));
|
||||
const retryOrderItems = new Set(); // Track unique order items in retry
|
||||
|
||||
if (validOrders.length > 0) {
|
||||
const placeholders = validOrders.map(() => `(${columnNames.map(() => "?").join(", ")})`).join(",");
|
||||
const values = validOrders.map(o => columnNames.map(col => o[col] ?? null)).flat();
|
||||
|
||||
const result = await localConnection.query(`
|
||||
INSERT INTO orders (${columnNames.join(", ")})
|
||||
VALUES ${placeholders}
|
||||
ON DUPLICATE KEY UPDATE
|
||||
SKU = VALUES(SKU),
|
||||
date = VALUES(date),
|
||||
price = VALUES(price),
|
||||
quantity = VALUES(quantity),
|
||||
discount = VALUES(discount),
|
||||
tax = VALUES(tax),
|
||||
tax_included = VALUES(tax_included),
|
||||
shipping = VALUES(shipping),
|
||||
customer = VALUES(customer),
|
||||
customer_name = VALUES(customer_name),
|
||||
status = VALUES(status),
|
||||
canceled = VALUES(canceled),
|
||||
costeach = VALUES(costeach)
|
||||
`, values);
|
||||
|
||||
const affectedRows = result[0].affectedRows;
|
||||
const updates = Math.floor(affectedRows / 2);
|
||||
const inserts = affectedRows - (updates * 2);
|
||||
|
||||
// Track unique order items
|
||||
validOrders.forEach(order => {
|
||||
retryOrderItems.add(`${order.order_number}-${order.pid}`);
|
||||
});
|
||||
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Orders import",
|
||||
message: `Successfully imported ${retryOrderItems.size} previously skipped order items`,
|
||||
});
|
||||
|
||||
// Update the main counters
|
||||
recordsAdded += inserts;
|
||||
recordsUpdated += updates;
|
||||
importedCount += retryOrderItems.size;
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('Warning: Failed to retry skipped orders:', error.message);
|
||||
console.warn(`Skipped ${skippedOrders.size} orders due to ${missingProducts.size} missing products`);
|
||||
}
|
||||
}
|
||||
|
||||
// Clean up temporary tables after ALL processing is complete
|
||||
await localConnection.query(`
|
||||
DROP TEMPORARY TABLE IF EXISTS temp_order_items;
|
||||
DROP TEMPORARY TABLE IF EXISTS temp_order_meta;
|
||||
DROP TEMPORARY TABLE IF EXISTS temp_order_discounts;
|
||||
DROP TEMPORARY TABLE IF EXISTS temp_order_taxes;
|
||||
DROP TEMPORARY TABLE IF EXISTS temp_order_costs;
|
||||
`);
|
||||
|
||||
// Only update sync status if we get here (no errors thrown)
|
||||
await localConnection.query(`
|
||||
INSERT INTO sync_status (table_name, last_sync_timestamp)
|
||||
VALUES ('orders', NOW())
|
||||
ON DUPLICATE KEY UPDATE last_sync_timestamp = NOW()
|
||||
`);
|
||||
|
||||
return {
|
||||
status: "complete",
|
||||
totalImported: Math.floor(importedCount),
|
||||
recordsAdded: recordsAdded || 0,
|
||||
recordsUpdated: Math.floor(recordsUpdated),
|
||||
totalSkipped: skippedOrders.size,
|
||||
missingProducts: missingProducts.size,
|
||||
incrementalUpdate,
|
||||
lastSyncTime
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("Error during orders import:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = importOrders;
|
||||
@@ -1,739 +0,0 @@
|
||||
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics/utils/progress');
|
||||
|
||||
// Utility functions
|
||||
const imageUrlBase = 'https://sbing.com/i/products/0000/';
|
||||
const getImageUrls = (pid, iid = 1) => {
|
||||
const paddedPid = pid.toString().padStart(6, '0');
|
||||
// Use padded PID only for the first 3 digits
|
||||
const prefix = paddedPid.slice(0, 3);
|
||||
// Use the actual pid for the rest of the URL
|
||||
const basePath = `${imageUrlBase}${prefix}/${pid}`;
|
||||
return {
|
||||
image: `${basePath}-t-${iid}.jpg`,
|
||||
image_175: `${basePath}-175x175-${iid}.jpg`,
|
||||
image_full: `${basePath}-o-${iid}.jpg`
|
||||
};
|
||||
};
|
||||
|
||||
async function setupAndCleanupTempTables(connection, operation = 'setup') {
|
||||
if (operation === 'setup') {
|
||||
await connection.query(`
|
||||
CREATE TEMPORARY TABLE IF NOT EXISTS temp_products (
|
||||
pid BIGINT NOT NULL,
|
||||
title VARCHAR(255),
|
||||
description TEXT,
|
||||
SKU VARCHAR(50),
|
||||
stock_quantity INT DEFAULT 0,
|
||||
pending_qty INT DEFAULT 0,
|
||||
preorder_count INT DEFAULT 0,
|
||||
notions_inv_count INT DEFAULT 0,
|
||||
price DECIMAL(10,3) NOT NULL DEFAULT 0,
|
||||
regular_price DECIMAL(10,3) NOT NULL DEFAULT 0,
|
||||
cost_price DECIMAL(10,3),
|
||||
vendor VARCHAR(100),
|
||||
vendor_reference VARCHAR(100),
|
||||
notions_reference VARCHAR(100),
|
||||
brand VARCHAR(100),
|
||||
line VARCHAR(100),
|
||||
subline VARCHAR(100),
|
||||
artist VARCHAR(100),
|
||||
category_ids TEXT,
|
||||
created_at DATETIME,
|
||||
first_received DATETIME,
|
||||
landing_cost_price DECIMAL(10,3),
|
||||
barcode VARCHAR(50),
|
||||
harmonized_tariff_code VARCHAR(50),
|
||||
updated_at DATETIME,
|
||||
visible BOOLEAN,
|
||||
replenishable BOOLEAN,
|
||||
permalink VARCHAR(255),
|
||||
moq DECIMAL(10,3),
|
||||
rating DECIMAL(10,2),
|
||||
reviews INT,
|
||||
weight DECIMAL(10,3),
|
||||
length DECIMAL(10,3),
|
||||
width DECIMAL(10,3),
|
||||
height DECIMAL(10,3),
|
||||
country_of_origin VARCHAR(100),
|
||||
location VARCHAR(100),
|
||||
total_sold INT,
|
||||
baskets INT,
|
||||
notifies INT,
|
||||
date_last_sold DATETIME,
|
||||
needs_update BOOLEAN DEFAULT TRUE,
|
||||
PRIMARY KEY (pid),
|
||||
INDEX idx_needs_update (needs_update)
|
||||
) ENGINE=InnoDB;
|
||||
`);
|
||||
} else {
|
||||
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_products;');
|
||||
}
|
||||
}
|
||||
|
||||
async function materializeCalculations(prodConnection, localConnection, incrementalUpdate = true, lastSyncTime = '1970-01-01') {
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Products import",
|
||||
message: "Fetching product data from production"
|
||||
});
|
||||
|
||||
// Get all product data in a single optimized query
|
||||
const [prodData] = await prodConnection.query(`
|
||||
SELECT
|
||||
p.pid,
|
||||
p.description AS title,
|
||||
p.notes AS description,
|
||||
p.itemnumber AS SKU,
|
||||
p.date_created,
|
||||
p.datein AS first_received,
|
||||
p.location,
|
||||
p.upc AS barcode,
|
||||
p.harmonized_tariff_code,
|
||||
p.stamp AS updated_at,
|
||||
CASE WHEN si.show + si.buyable > 0 THEN 1 ELSE 0 END AS visible,
|
||||
CASE
|
||||
WHEN p.reorder < 0 THEN 0
|
||||
WHEN (
|
||||
(IFNULL(pls.date_sold, '0000-00-00') = '0000-00-00' OR pls.date_sold <= DATE_SUB(CURDATE(), INTERVAL 5 YEAR))
|
||||
OR (p.datein = '0000-00-00 00:00:00' OR p.datein <= DATE_SUB(NOW(), INTERVAL 5 YEAR))
|
||||
OR (p.date_refill = '0000-00-00 00:00:00' OR p.date_refill <= DATE_SUB(NOW(), INTERVAL 5 YEAR))
|
||||
) THEN 0
|
||||
ELSE 1
|
||||
END AS replenishable,
|
||||
COALESCE(si.available_local, 0) - COALESCE(
|
||||
(SELECT SUM(oi.qty_ordered - oi.qty_placed)
|
||||
FROM order_items oi
|
||||
JOIN _order o ON oi.order_id = o.order_id
|
||||
WHERE oi.prod_pid = p.pid
|
||||
AND o.date_placed != '0000-00-00 00:00:00'
|
||||
AND o.date_shipped = '0000-00-00 00:00:00'
|
||||
AND oi.pick_finished = 0
|
||||
AND oi.qty_back = 0
|
||||
AND o.order_status != 15
|
||||
AND o.order_status < 90
|
||||
AND oi.qty_ordered >= oi.qty_placed
|
||||
AND oi.qty_ordered > 0
|
||||
), 0
|
||||
) as stock_quantity,
|
||||
COALESCE(
|
||||
(SELECT SUM(oi.qty_ordered - oi.qty_placed)
|
||||
FROM order_items oi
|
||||
JOIN _order o ON oi.order_id = o.order_id
|
||||
WHERE oi.prod_pid = p.pid
|
||||
AND o.date_placed != '0000-00-00 00:00:00'
|
||||
AND o.date_shipped = '0000-00-00 00:00:00'
|
||||
AND oi.pick_finished = 0
|
||||
AND oi.qty_back = 0
|
||||
AND o.order_status != 15
|
||||
AND o.order_status < 90
|
||||
AND oi.qty_ordered >= oi.qty_placed
|
||||
AND oi.qty_ordered > 0
|
||||
), 0
|
||||
) as pending_qty,
|
||||
COALESCE(ci.onpreorder, 0) as preorder_count,
|
||||
COALESCE(pnb.inventory, 0) as notions_inv_count,
|
||||
COALESCE(pcp.price_each, 0) as price,
|
||||
COALESCE(p.sellingprice, 0) AS regular_price,
|
||||
CASE
|
||||
WHEN EXISTS (SELECT 1 FROM product_inventory WHERE pid = p.pid AND count > 0)
|
||||
THEN (SELECT ROUND(AVG(costeach), 5) FROM product_inventory WHERE pid = p.pid AND count > 0)
|
||||
ELSE (SELECT costeach FROM product_inventory WHERE pid = p.pid ORDER BY daterec DESC LIMIT 1)
|
||||
END AS cost_price,
|
||||
NULL as landing_cost_price,
|
||||
s.companyname AS vendor,
|
||||
CASE
|
||||
WHEN s.companyname = 'Notions' THEN sid.notions_itemnumber
|
||||
ELSE sid.supplier_itemnumber
|
||||
END AS vendor_reference,
|
||||
sid.notions_itemnumber AS notions_reference,
|
||||
CONCAT('https://www.acherryontop.com/shop/product/', p.pid) AS permalink,
|
||||
pc1.name AS brand,
|
||||
pc2.name AS line,
|
||||
pc3.name AS subline,
|
||||
pc4.name AS artist,
|
||||
COALESCE(CASE
|
||||
WHEN sid.supplier_id = 92 THEN sid.notions_qty_per_unit
|
||||
ELSE sid.supplier_qty_per_unit
|
||||
END, sid.notions_qty_per_unit) AS moq,
|
||||
p.rating,
|
||||
p.rating_votes AS reviews,
|
||||
p.weight,
|
||||
p.length,
|
||||
p.width,
|
||||
p.height,
|
||||
p.country_of_origin,
|
||||
(SELECT COUNT(*) FROM mybasket mb WHERE mb.item = p.pid AND mb.qty > 0) AS baskets,
|
||||
(SELECT COUNT(*) FROM product_notify pn WHERE pn.pid = p.pid) AS notifies,
|
||||
p.totalsold AS total_sold,
|
||||
pls.date_sold as date_last_sold,
|
||||
GROUP_CONCAT(DISTINCT CASE
|
||||
WHEN pc.cat_id IS NOT NULL
|
||||
AND pc.type IN (10, 20, 11, 21, 12, 13)
|
||||
AND pci.cat_id NOT IN (16, 17)
|
||||
THEN pci.cat_id
|
||||
END) as category_ids
|
||||
FROM products p
|
||||
LEFT JOIN shop_inventory si ON p.pid = si.pid AND si.store = 0
|
||||
LEFT JOIN current_inventory ci ON p.pid = ci.pid
|
||||
LEFT JOIN product_notions_b2b pnb ON p.pid = pnb.pid
|
||||
LEFT JOIN product_current_prices pcp ON p.pid = pcp.pid AND pcp.active = 1
|
||||
LEFT JOIN supplier_item_data sid ON p.pid = sid.pid
|
||||
LEFT JOIN suppliers s ON sid.supplier_id = s.supplierid
|
||||
LEFT JOIN product_category_index pci ON p.pid = pci.pid
|
||||
LEFT JOIN product_categories pc ON pci.cat_id = pc.cat_id
|
||||
LEFT JOIN product_categories pc1 ON p.company = pc1.cat_id
|
||||
LEFT JOIN product_categories pc2 ON p.line = pc2.cat_id
|
||||
LEFT JOIN product_categories pc3 ON p.subline = pc3.cat_id
|
||||
LEFT JOIN product_categories pc4 ON p.artist = pc4.cat_id
|
||||
LEFT JOIN product_last_sold pls ON p.pid = pls.pid
|
||||
WHERE ${incrementalUpdate ? `
|
||||
p.stamp > ? OR
|
||||
ci.stamp > ? OR
|
||||
pcp.date_deactive > ? OR
|
||||
pcp.date_active > ? OR
|
||||
pnb.date_updated > ?
|
||||
` : 'TRUE'}
|
||||
GROUP BY p.pid
|
||||
`, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime] : []);
|
||||
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Products import",
|
||||
message: `Processing ${prodData.length} product records`
|
||||
});
|
||||
|
||||
// Insert all product data into temp table in batches
|
||||
for (let i = 0; i < prodData.length; i += 1000) {
|
||||
const batch = prodData.slice(i, i + 1000);
|
||||
const values = batch.map(row => [
|
||||
row.pid,
|
||||
row.title,
|
||||
row.description,
|
||||
row.SKU,
|
||||
// Set stock quantity to 0 if it's over 5000
|
||||
row.stock_quantity > 5000 ? 0 : Math.max(0, row.stock_quantity),
|
||||
row.pending_qty,
|
||||
row.preorder_count,
|
||||
row.notions_inv_count,
|
||||
row.price,
|
||||
row.regular_price,
|
||||
row.cost_price,
|
||||
row.vendor,
|
||||
row.vendor_reference,
|
||||
row.notions_reference,
|
||||
row.brand,
|
||||
row.line,
|
||||
row.subline,
|
||||
row.artist,
|
||||
row.category_ids,
|
||||
row.date_created, // map to created_at
|
||||
row.first_received,
|
||||
row.landing_cost_price,
|
||||
row.barcode,
|
||||
row.harmonized_tariff_code,
|
||||
row.updated_at,
|
||||
row.visible,
|
||||
row.replenishable,
|
||||
row.permalink,
|
||||
row.moq,
|
||||
row.rating ? Number(row.rating).toFixed(2) : null,
|
||||
row.reviews,
|
||||
row.weight,
|
||||
row.length,
|
||||
row.width,
|
||||
row.height,
|
||||
row.country_of_origin,
|
||||
row.location,
|
||||
row.total_sold,
|
||||
row.baskets,
|
||||
row.notifies,
|
||||
row.date_last_sold,
|
||||
true // Mark as needing update
|
||||
]);
|
||||
|
||||
if (values.length > 0) {
|
||||
await localConnection.query(`
|
||||
INSERT INTO temp_products (
|
||||
pid, title, description, SKU,
|
||||
stock_quantity, pending_qty, preorder_count, notions_inv_count,
|
||||
price, regular_price, cost_price,
|
||||
vendor, vendor_reference, notions_reference,
|
||||
brand, line, subline, artist,
|
||||
category_ids, created_at, first_received,
|
||||
landing_cost_price, barcode, harmonized_tariff_code,
|
||||
updated_at, visible, replenishable, permalink,
|
||||
moq, rating, reviews, weight, length, width,
|
||||
height, country_of_origin, location, total_sold,
|
||||
baskets, notifies, date_last_sold, needs_update
|
||||
)
|
||||
VALUES ?
|
||||
ON DUPLICATE KEY UPDATE
|
||||
title = VALUES(title),
|
||||
description = VALUES(description),
|
||||
SKU = VALUES(SKU),
|
||||
stock_quantity = VALUES(stock_quantity),
|
||||
pending_qty = VALUES(pending_qty),
|
||||
preorder_count = VALUES(preorder_count),
|
||||
notions_inv_count = VALUES(notions_inv_count),
|
||||
price = VALUES(price),
|
||||
regular_price = VALUES(regular_price),
|
||||
cost_price = VALUES(cost_price),
|
||||
vendor = VALUES(vendor),
|
||||
vendor_reference = VALUES(vendor_reference),
|
||||
notions_reference = VALUES(notions_reference),
|
||||
brand = VALUES(brand),
|
||||
line = VALUES(line),
|
||||
subline = VALUES(subline),
|
||||
artist = VALUES(artist),
|
||||
category_ids = VALUES(category_ids),
|
||||
created_at = VALUES(created_at),
|
||||
first_received = VALUES(first_received),
|
||||
landing_cost_price = VALUES(landing_cost_price),
|
||||
barcode = VALUES(barcode),
|
||||
harmonized_tariff_code = VALUES(harmonized_tariff_code),
|
||||
updated_at = VALUES(updated_at),
|
||||
visible = VALUES(visible),
|
||||
replenishable = VALUES(replenishable),
|
||||
permalink = VALUES(permalink),
|
||||
moq = VALUES(moq),
|
||||
rating = VALUES(rating),
|
||||
reviews = VALUES(reviews),
|
||||
weight = VALUES(weight),
|
||||
length = VALUES(length),
|
||||
width = VALUES(width),
|
||||
height = VALUES(height),
|
||||
country_of_origin = VALUES(country_of_origin),
|
||||
location = VALUES(location),
|
||||
total_sold = VALUES(total_sold),
|
||||
baskets = VALUES(baskets),
|
||||
notifies = VALUES(notifies),
|
||||
date_last_sold = VALUES(date_last_sold),
|
||||
needs_update = TRUE
|
||||
`, [values]);
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Products import",
|
||||
message: `Processed ${Math.min(i + 1000, prodData.length)} of ${prodData.length} product records`,
|
||||
current: i + batch.length,
|
||||
total: prodData.length
|
||||
});
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Products import",
|
||||
message: "Finished materializing calculations"
|
||||
});
|
||||
}
|
||||
|
||||
async function importProducts(prodConnection, localConnection, incrementalUpdate = true) {
|
||||
const startTime = Date.now();
|
||||
let recordsAdded = 0;
|
||||
let recordsUpdated = 0;
|
||||
|
||||
try {
|
||||
// Get column names first
|
||||
const [columns] = await localConnection.query(`
|
||||
SELECT COLUMN_NAME
|
||||
FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_NAME = 'products'
|
||||
ORDER BY ORDINAL_POSITION
|
||||
`);
|
||||
const columnNames = columns.map(col => col.COLUMN_NAME);
|
||||
|
||||
// Get last sync info
|
||||
const [syncInfo] = await localConnection.query(
|
||||
"SELECT last_sync_timestamp FROM sync_status WHERE table_name = 'products'"
|
||||
);
|
||||
const lastSyncTime = syncInfo?.[0]?.last_sync_timestamp || '1970-01-01';
|
||||
|
||||
console.log('Products: Using last sync time:', lastSyncTime);
|
||||
|
||||
// Setup temporary tables
|
||||
await setupAndCleanupTempTables(localConnection, 'setup');
|
||||
|
||||
// Materialize calculations - this will populate temp_products
|
||||
await materializeCalculations(prodConnection, localConnection, incrementalUpdate, lastSyncTime);
|
||||
|
||||
// Get actual count from temp table - only count products that need updates
|
||||
const [[{ actualTotal }]] = await localConnection.query(`
|
||||
SELECT COUNT(DISTINCT pid) as actualTotal
|
||||
FROM temp_products
|
||||
WHERE needs_update = 1
|
||||
`);
|
||||
|
||||
console.log('Products: Found changes:', actualTotal);
|
||||
|
||||
// Process in batches
|
||||
const BATCH_SIZE = 5000;
|
||||
let processed = 0;
|
||||
|
||||
while (processed < actualTotal) {
|
||||
const [batch] = await localConnection.query(`
|
||||
SELECT * FROM temp_products
|
||||
WHERE needs_update = 1
|
||||
LIMIT ? OFFSET ?
|
||||
`, [BATCH_SIZE, processed]);
|
||||
|
||||
if (!batch || batch.length === 0) break;
|
||||
|
||||
// Add image URLs
|
||||
batch.forEach(row => {
|
||||
const urls = getImageUrls(row.pid);
|
||||
row.image = urls.image;
|
||||
row.image_175 = urls.image_175;
|
||||
row.image_full = urls.image_full;
|
||||
});
|
||||
|
||||
if (batch.length > 0) {
|
||||
// Get existing products in one query
|
||||
const [existingProducts] = await localConnection.query(
|
||||
`SELECT ${columnNames.join(',')} FROM products WHERE pid IN (?)`,
|
||||
[batch.map(p => p.pid)]
|
||||
);
|
||||
const existingPidsMap = new Map(existingProducts.map(p => [p.pid, p]));
|
||||
|
||||
// Split into inserts and updates
|
||||
const insertsAndUpdates = batch.reduce((acc, product) => {
|
||||
if (existingPidsMap.has(product.pid)) {
|
||||
const existing = existingPidsMap.get(product.pid);
|
||||
// Check if any values are different
|
||||
const hasChanges = columnNames.some(col => {
|
||||
const newVal = product[col] ?? null;
|
||||
const oldVal = existing[col] ?? null;
|
||||
if (col === "managing_stock") return false; // Skip this as it's always 1
|
||||
if (typeof newVal === 'number' && typeof oldVal === 'number') {
|
||||
return Math.abs(newVal - oldVal) > 0.00001;
|
||||
}
|
||||
return newVal !== oldVal;
|
||||
});
|
||||
|
||||
if (hasChanges) {
|
||||
acc.updates.push(product);
|
||||
}
|
||||
} else {
|
||||
acc.inserts.push(product);
|
||||
}
|
||||
return acc;
|
||||
}, { inserts: [], updates: [] });
|
||||
|
||||
// Process inserts
|
||||
if (insertsAndUpdates.inserts.length > 0) {
|
||||
const insertValues = insertsAndUpdates.inserts.map(product =>
|
||||
columnNames.map(col => {
|
||||
const val = product[col] ?? null;
|
||||
if (col === "managing_stock") return 1;
|
||||
return val;
|
||||
})
|
||||
);
|
||||
|
||||
const insertPlaceholders = insertsAndUpdates.inserts
|
||||
.map(() => `(${Array(columnNames.length).fill('?').join(',')})`)
|
||||
.join(',');
|
||||
|
||||
const insertResult = await localConnection.query(`
|
||||
INSERT INTO products (${columnNames.join(',')})
|
||||
VALUES ${insertPlaceholders}
|
||||
`, insertValues.flat());
|
||||
|
||||
recordsAdded += insertResult[0].affectedRows;
|
||||
}
|
||||
|
||||
// Process updates
|
||||
if (insertsAndUpdates.updates.length > 0) {
|
||||
const updateValues = insertsAndUpdates.updates.map(product =>
|
||||
columnNames.map(col => {
|
||||
const val = product[col] ?? null;
|
||||
if (col === "managing_stock") return 1;
|
||||
return val;
|
||||
})
|
||||
);
|
||||
|
||||
const updatePlaceholders = insertsAndUpdates.updates
|
||||
.map(() => `(${Array(columnNames.length).fill('?').join(',')})`)
|
||||
.join(',');
|
||||
|
||||
const updateResult = await localConnection.query(`
|
||||
INSERT INTO products (${columnNames.join(',')})
|
||||
VALUES ${updatePlaceholders}
|
||||
ON DUPLICATE KEY UPDATE
|
||||
${columnNames
|
||||
.filter(col => col !== 'pid')
|
||||
.map(col => `${col} = VALUES(${col})`)
|
||||
.join(',')};
|
||||
`, updateValues.flat());
|
||||
|
||||
recordsUpdated += insertsAndUpdates.updates.length;
|
||||
}
|
||||
|
||||
// Process category relationships
|
||||
if (batch.some(p => p.category_ids)) {
|
||||
const categoryRelationships = batch
|
||||
.filter(p => p.category_ids)
|
||||
.flatMap(product =>
|
||||
product.category_ids
|
||||
.split(',')
|
||||
.map(id => id.trim())
|
||||
.filter(id => id)
|
||||
.map(Number)
|
||||
.filter(id => !isNaN(id))
|
||||
.map(catId => [catId, product.pid])
|
||||
);
|
||||
|
||||
if (categoryRelationships.length > 0) {
|
||||
// Verify categories exist before inserting relationships
|
||||
const uniqueCatIds = [...new Set(categoryRelationships.map(([catId]) => catId))];
|
||||
const [existingCats] = await localConnection.query(
|
||||
"SELECT cat_id FROM categories WHERE cat_id IN (?)",
|
||||
[uniqueCatIds]
|
||||
);
|
||||
const existingCatIds = new Set(existingCats.map(c => c.cat_id));
|
||||
|
||||
// Filter relationships to only include existing categories
|
||||
const validRelationships = categoryRelationships.filter(([catId]) =>
|
||||
existingCatIds.has(catId)
|
||||
);
|
||||
|
||||
if (validRelationships.length > 0) {
|
||||
const catPlaceholders = validRelationships
|
||||
.map(() => "(?, ?)")
|
||||
.join(",");
|
||||
await localConnection.query(
|
||||
`INSERT IGNORE INTO product_categories (cat_id, pid)
|
||||
VALUES ${catPlaceholders}`,
|
||||
validRelationships.flat()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
processed += batch.length;
|
||||
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Products import",
|
||||
message: `Processed ${processed} of ${actualTotal} products`,
|
||||
current: processed,
|
||||
total: actualTotal,
|
||||
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
|
||||
remaining: estimateRemaining(startTime, processed, actualTotal),
|
||||
rate: calculateRate(startTime, processed)
|
||||
});
|
||||
}
|
||||
|
||||
// Drop temporary tables
|
||||
await setupAndCleanupTempTables(localConnection, 'cleanup');
|
||||
|
||||
// Only update sync status if we get here (no errors thrown)
|
||||
await localConnection.query(`
|
||||
INSERT INTO sync_status (table_name, last_sync_timestamp)
|
||||
VALUES ('products', NOW())
|
||||
ON DUPLICATE KEY UPDATE last_sync_timestamp = NOW()
|
||||
`);
|
||||
|
||||
return {
|
||||
status: "complete",
|
||||
totalImported: actualTotal,
|
||||
recordsAdded: recordsAdded || 0,
|
||||
recordsUpdated: recordsUpdated || 0,
|
||||
incrementalUpdate,
|
||||
lastSyncTime
|
||||
};
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function importMissingProducts(prodConnection, localConnection, missingPids) {
|
||||
try {
|
||||
// Get column names first
|
||||
const [columns] = await localConnection.query(`
|
||||
SELECT COLUMN_NAME
|
||||
FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_NAME = 'products'
|
||||
ORDER BY ORDINAL_POSITION
|
||||
`);
|
||||
const columnNames = columns.map((col) => col.COLUMN_NAME);
|
||||
|
||||
// Get the missing products with all their data in one optimized query
|
||||
const [products] = await prodConnection.query(`
|
||||
SELECT
|
||||
p.pid,
|
||||
p.description AS title,
|
||||
p.notes AS description,
|
||||
p.itemnumber AS SKU,
|
||||
p.date_created,
|
||||
p.datein AS first_received,
|
||||
p.location,
|
||||
p.upc AS barcode,
|
||||
p.harmonized_tariff_code,
|
||||
p.stamp AS updated_at,
|
||||
CASE WHEN si.show + si.buyable > 0 THEN 1 ELSE 0 END AS visible,
|
||||
CASE
|
||||
WHEN p.reorder < 0 THEN 0
|
||||
WHEN (
|
||||
(IFNULL(pls.date_sold, '0000-00-00') = '0000-00-00' OR pls.date_sold <= DATE_SUB(CURDATE(), INTERVAL 5 YEAR))
|
||||
OR (p.datein = '0000-00-00 00:00:00' OR p.datein <= DATE_SUB(NOW(), INTERVAL 5 YEAR))
|
||||
OR (p.date_refill = '0000-00-00 00:00:00' OR p.date_refill <= DATE_SUB(NOW(), INTERVAL 5 YEAR))
|
||||
) THEN 0
|
||||
ELSE 1
|
||||
END AS replenishable,
|
||||
COALESCE(si.available_local, 0) as stock_quantity,
|
||||
COALESCE(
|
||||
(SELECT SUM(oi.qty_ordered - oi.qty_placed)
|
||||
FROM order_items oi
|
||||
JOIN _order o ON oi.order_id = o.order_id
|
||||
WHERE oi.prod_pid = p.pid
|
||||
AND o.date_placed != '0000-00-00 00:00:00'
|
||||
AND o.date_shipped = '0000-00-00 00:00:00'
|
||||
AND oi.pick_finished = 0
|
||||
AND oi.qty_back = 0
|
||||
AND o.order_status != 15
|
||||
AND o.order_status < 90
|
||||
AND oi.qty_ordered >= oi.qty_placed
|
||||
AND oi.qty_ordered > 0
|
||||
), 0
|
||||
) as pending_qty,
|
||||
COALESCE(ci.onpreorder, 0) as preorder_count,
|
||||
COALESCE(pnb.inventory, 0) as notions_inv_count,
|
||||
COALESCE(pcp.price_each, 0) as price,
|
||||
COALESCE(p.sellingprice, 0) AS regular_price,
|
||||
CASE
|
||||
WHEN EXISTS (SELECT 1 FROM product_inventory WHERE pid = p.pid AND count > 0)
|
||||
THEN (SELECT ROUND(AVG(costeach), 5) FROM product_inventory WHERE pid = p.pid AND count > 0)
|
||||
ELSE (SELECT costeach FROM product_inventory WHERE pid = p.pid ORDER BY daterec DESC LIMIT 1)
|
||||
END AS cost_price,
|
||||
NULL AS landing_cost_price,
|
||||
p.rating,
|
||||
p.rating_votes AS reviews,
|
||||
p.weight,
|
||||
p.length,
|
||||
p.width,
|
||||
p.height,
|
||||
(SELECT COUNT(*) FROM mybasket mb WHERE mb.item = p.pid AND mb.qty > 0) AS baskets,
|
||||
(SELECT COUNT(*) FROM product_notify pn WHERE pn.pid = p.pid) AS notifies,
|
||||
p.totalsold AS total_sold,
|
||||
p.country_of_origin,
|
||||
pls.date_sold as date_last_sold,
|
||||
GROUP_CONCAT(DISTINCT CASE WHEN pc.cat_id IS NOT NULL THEN pci.cat_id END) as category_ids
|
||||
FROM products p
|
||||
LEFT JOIN shop_inventory si ON p.pid = si.pid AND si.store = 0
|
||||
LEFT JOIN supplier_item_data sid ON p.pid = sid.pid
|
||||
LEFT JOIN suppliers s ON sid.supplier_id = s.supplierid
|
||||
LEFT JOIN product_category_index pci ON p.pid = pci.pid
|
||||
LEFT JOIN product_categories pc ON pci.cat_id = pc.cat_id
|
||||
AND pc.type IN (10, 20, 11, 21, 12, 13)
|
||||
AND pci.cat_id NOT IN (16, 17)
|
||||
LEFT JOIN product_categories pc1 ON p.company = pc1.cat_id
|
||||
LEFT JOIN product_categories pc2 ON p.line = pc2.cat_id
|
||||
LEFT JOIN product_categories pc3 ON p.subline = pc3.cat_id
|
||||
LEFT JOIN product_categories pc4 ON p.artist = pc4.cat_id
|
||||
LEFT JOIN product_last_sold pls ON p.pid = pls.pid
|
||||
LEFT JOIN current_inventory ci ON p.pid = ci.pid
|
||||
LEFT JOIN product_current_prices pcp ON p.pid = pcp.pid AND pcp.active = 1
|
||||
LEFT JOIN product_notions_b2b pnb ON p.pid = pnb.pid
|
||||
WHERE p.pid IN (?)
|
||||
GROUP BY p.pid
|
||||
`, [missingPids]);
|
||||
|
||||
// Add image URLs
|
||||
products.forEach(product => {
|
||||
const urls = getImageUrls(product.pid);
|
||||
product.image = urls.image;
|
||||
product.image_175 = urls.image_175;
|
||||
product.image_full = urls.image_full;
|
||||
});
|
||||
|
||||
let recordsAdded = 0;
|
||||
let recordsUpdated = 0;
|
||||
|
||||
if (products.length > 0) {
|
||||
// Map values in the same order as columns
|
||||
const productValues = products.flatMap(product =>
|
||||
columnNames.map(col => {
|
||||
const val = product[col] ?? null;
|
||||
if (col === "managing_stock") return 1;
|
||||
if (typeof val === "number") return val || 0;
|
||||
return val;
|
||||
})
|
||||
);
|
||||
|
||||
// Generate placeholders for all products
|
||||
const placeholders = products
|
||||
.map(() => `(${Array(columnNames.length).fill("?").join(",")})`)
|
||||
.join(",");
|
||||
|
||||
// Build and execute the query
|
||||
const query = `
|
||||
INSERT INTO products (${columnNames.join(",")})
|
||||
VALUES ${placeholders}
|
||||
ON DUPLICATE KEY UPDATE ${columnNames
|
||||
.filter((col) => col !== "pid")
|
||||
.map((col) => `${col} = VALUES(${col})`)
|
||||
.join(",")};
|
||||
`;
|
||||
|
||||
const result = await localConnection.query(query, productValues);
|
||||
recordsAdded = result.affectedRows - result.changedRows;
|
||||
recordsUpdated = result.changedRows;
|
||||
|
||||
// Handle category relationships if any
|
||||
const categoryRelationships = [];
|
||||
products.forEach(product => {
|
||||
if (product.category_ids) {
|
||||
const catIds = product.category_ids
|
||||
.split(",")
|
||||
.map(id => id.trim())
|
||||
.filter(id => id)
|
||||
.map(Number);
|
||||
catIds.forEach(catId => {
|
||||
if (catId) categoryRelationships.push([catId, product.pid]);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
if (categoryRelationships.length > 0) {
|
||||
// Verify categories exist before inserting relationships
|
||||
const uniqueCatIds = [...new Set(categoryRelationships.map(([catId]) => catId))];
|
||||
const [existingCats] = await localConnection.query(
|
||||
"SELECT cat_id FROM categories WHERE cat_id IN (?)",
|
||||
[uniqueCatIds]
|
||||
);
|
||||
const existingCatIds = new Set(existingCats.map(c => c.cat_id));
|
||||
|
||||
// Filter relationships to only include existing categories
|
||||
const validRelationships = categoryRelationships.filter(([catId]) =>
|
||||
existingCatIds.has(catId)
|
||||
);
|
||||
|
||||
if (validRelationships.length > 0) {
|
||||
const catPlaceholders = validRelationships
|
||||
.map(() => "(?, ?)")
|
||||
.join(",");
|
||||
await localConnection.query(
|
||||
`INSERT IGNORE INTO product_categories (cat_id, pid)
|
||||
VALUES ${catPlaceholders}`,
|
||||
validRelationships.flat()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
status: "complete",
|
||||
totalImported: products.length,
|
||||
recordsAdded,
|
||||
recordsUpdated
|
||||
};
|
||||
} catch (error) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
importProducts,
|
||||
importMissingProducts
|
||||
};
|
||||
@@ -1,543 +0,0 @@
|
||||
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics/utils/progress');
|
||||
|
||||
async function importPurchaseOrders(prodConnection, localConnection, incrementalUpdate = true) {
|
||||
const startTime = Date.now();
|
||||
let recordsAdded = 0;
|
||||
let recordsUpdated = 0;
|
||||
|
||||
try {
|
||||
// Get last sync info
|
||||
const [syncInfo] = await localConnection.query(
|
||||
"SELECT last_sync_timestamp FROM sync_status WHERE table_name = 'purchase_orders'"
|
||||
);
|
||||
const lastSyncTime = syncInfo?.[0]?.last_sync_timestamp || '1970-01-01';
|
||||
|
||||
console.log('Purchase Orders: Using last sync time:', lastSyncTime);
|
||||
|
||||
// Insert temporary table creation query for purchase orders
|
||||
await localConnection.query(`
|
||||
CREATE TABLE IF NOT EXISTS temp_purchase_orders (
|
||||
po_id INT UNSIGNED NOT NULL,
|
||||
pid INT UNSIGNED NOT NULL,
|
||||
vendor VARCHAR(255),
|
||||
date DATE,
|
||||
expected_date DATE,
|
||||
status INT,
|
||||
notes TEXT,
|
||||
PRIMARY KEY (po_id, pid)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
|
||||
`);
|
||||
|
||||
outputProgress({
|
||||
operation: `Starting ${incrementalUpdate ? 'incremental' : 'full'} purchase orders import`,
|
||||
status: "running",
|
||||
});
|
||||
|
||||
// Get column names for the insert
|
||||
const [columns] = await localConnection.query(`
|
||||
SELECT COLUMN_NAME
|
||||
FROM INFORMATION_SCHEMA.COLUMNS
|
||||
WHERE TABLE_NAME = 'purchase_orders'
|
||||
ORDER BY ORDINAL_POSITION
|
||||
`);
|
||||
const columnNames = columns
|
||||
.map((col) => col.COLUMN_NAME)
|
||||
.filter((name) => name !== "id");
|
||||
|
||||
// Build incremental conditions
|
||||
const incrementalWhereClause = incrementalUpdate
|
||||
? `AND (
|
||||
p.date_updated > ?
|
||||
OR p.date_ordered > ?
|
||||
OR p.date_estin > ?
|
||||
OR r.date_updated > ?
|
||||
OR r.date_created > ?
|
||||
OR r.date_checked > ?
|
||||
OR rp.stamp > ?
|
||||
OR rp.received_date > ?
|
||||
)`
|
||||
: "";
|
||||
const incrementalParams = incrementalUpdate
|
||||
? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime]
|
||||
: [];
|
||||
|
||||
// First get all relevant PO IDs with basic info
|
||||
const [[{ total }]] = await prodConnection.query(`
|
||||
SELECT COUNT(*) as total
|
||||
FROM (
|
||||
SELECT DISTINCT pop.po_id, pop.pid
|
||||
FROM po p
|
||||
USE INDEX (idx_date_created)
|
||||
JOIN po_products pop ON p.po_id = pop.po_id
|
||||
JOIN suppliers s ON p.supplier_id = s.supplierid
|
||||
WHERE p.date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
|
||||
${incrementalUpdate ? `
|
||||
AND (
|
||||
p.date_updated > ?
|
||||
OR p.date_ordered > ?
|
||||
OR p.date_estin > ?
|
||||
)
|
||||
` : ''}
|
||||
UNION
|
||||
SELECT DISTINCT r.receiving_id as po_id, rp.pid
|
||||
FROM receivings_products rp
|
||||
USE INDEX (received_date)
|
||||
LEFT JOIN receivings r ON r.receiving_id = rp.receiving_id
|
||||
WHERE rp.received_date >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
|
||||
${incrementalUpdate ? `
|
||||
AND (
|
||||
r.date_created > ?
|
||||
OR r.date_checked > ?
|
||||
OR rp.stamp > ?
|
||||
OR rp.received_date > ?
|
||||
)
|
||||
` : ''}
|
||||
) all_items
|
||||
`, incrementalUpdate ? [
|
||||
lastSyncTime, lastSyncTime, lastSyncTime, // PO conditions
|
||||
lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime // Receiving conditions
|
||||
] : []);
|
||||
|
||||
console.log('Purchase Orders: Found changes:', total);
|
||||
|
||||
const [poList] = await prodConnection.query(`
|
||||
SELECT DISTINCT
|
||||
COALESCE(p.po_id, r.receiving_id) as po_id,
|
||||
COALESCE(
|
||||
NULLIF(s1.companyname, ''),
|
||||
NULLIF(s2.companyname, ''),
|
||||
'Unknown Vendor'
|
||||
) as vendor,
|
||||
CASE
|
||||
WHEN p.po_id IS NOT NULL THEN
|
||||
DATE(COALESCE(
|
||||
NULLIF(p.date_ordered, '0000-00-00 00:00:00'),
|
||||
p.date_created
|
||||
))
|
||||
WHEN r.receiving_id IS NOT NULL THEN
|
||||
DATE(r.date_created)
|
||||
END as date,
|
||||
CASE
|
||||
WHEN p.date_estin = '0000-00-00' THEN NULL
|
||||
WHEN p.date_estin IS NULL THEN NULL
|
||||
WHEN p.date_estin NOT REGEXP '^[0-9]{4}-[0-9]{2}-[0-9]{2}$' THEN NULL
|
||||
ELSE p.date_estin
|
||||
END as expected_date,
|
||||
COALESCE(p.status, 50) as status,
|
||||
p.short_note as notes,
|
||||
p.notes as long_note
|
||||
FROM (
|
||||
SELECT po_id FROM po
|
||||
USE INDEX (idx_date_created)
|
||||
WHERE date_ordered >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
|
||||
${incrementalUpdate ? `
|
||||
AND (
|
||||
date_ordered > ?
|
||||
OR date_updated > ?
|
||||
OR date_estin > ?
|
||||
)
|
||||
` : ''}
|
||||
UNION
|
||||
SELECT DISTINCT r.receiving_id as po_id
|
||||
FROM receivings r
|
||||
JOIN receivings_products rp USE INDEX (received_date) ON r.receiving_id = rp.receiving_id
|
||||
WHERE rp.received_date >= DATE_SUB(CURRENT_DATE, INTERVAL ${incrementalUpdate ? '1' : '5'} YEAR)
|
||||
${incrementalUpdate ? `
|
||||
AND (
|
||||
r.date_created > ?
|
||||
OR r.date_checked > ?
|
||||
OR rp.stamp > ?
|
||||
OR rp.received_date > ?
|
||||
)
|
||||
` : ''}
|
||||
) ids
|
||||
LEFT JOIN po p ON ids.po_id = p.po_id
|
||||
LEFT JOIN suppliers s1 ON p.supplier_id = s1.supplierid
|
||||
LEFT JOIN receivings r ON ids.po_id = r.receiving_id
|
||||
LEFT JOIN suppliers s2 ON r.supplier_id = s2.supplierid
|
||||
ORDER BY po_id
|
||||
`, incrementalUpdate ? [
|
||||
lastSyncTime, lastSyncTime, lastSyncTime, // PO conditions
|
||||
lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime // Receiving conditions
|
||||
] : []);
|
||||
|
||||
console.log('Sample PO dates:', poList.slice(0, 5).map(po => ({
|
||||
po_id: po.po_id,
|
||||
raw_date_ordered: po.raw_date_ordered,
|
||||
raw_date_created: po.raw_date_created,
|
||||
raw_date_estin: po.raw_date_estin,
|
||||
computed_date: po.date,
|
||||
expected_date: po.expected_date
|
||||
})));
|
||||
|
||||
const totalItems = total;
|
||||
let processed = 0;
|
||||
|
||||
const BATCH_SIZE = 5000;
|
||||
const PROGRESS_INTERVAL = 500;
|
||||
let lastProgressUpdate = Date.now();
|
||||
|
||||
outputProgress({
|
||||
operation: `Starting purchase orders import - Processing ${totalItems} purchase order items`,
|
||||
status: "running",
|
||||
});
|
||||
|
||||
for (let i = 0; i < poList.length; i += BATCH_SIZE) {
|
||||
const batch = poList.slice(i, Math.min(i + BATCH_SIZE, poList.length));
|
||||
const poIds = batch.map(po => po.po_id);
|
||||
|
||||
// Get all products for these POs in one query
|
||||
const [poProducts] = await prodConnection.query(`
|
||||
SELECT
|
||||
pop.po_id,
|
||||
pop.pid,
|
||||
pr.itemnumber as sku,
|
||||
pr.description as name,
|
||||
pop.cost_each,
|
||||
pop.qty_each as ordered
|
||||
FROM po_products pop
|
||||
USE INDEX (PRIMARY)
|
||||
JOIN products pr ON pop.pid = pr.pid
|
||||
WHERE pop.po_id IN (?)
|
||||
`, [poIds]);
|
||||
|
||||
// Process PO products in smaller sub-batches to avoid packet size issues
|
||||
const SUB_BATCH_SIZE = 5000;
|
||||
for (let j = 0; j < poProducts.length; j += SUB_BATCH_SIZE) {
|
||||
const productBatch = poProducts.slice(j, j + SUB_BATCH_SIZE);
|
||||
const productPids = [...new Set(productBatch.map(p => p.pid))];
|
||||
const batchPoIds = [...new Set(productBatch.map(p => p.po_id))];
|
||||
|
||||
// Get receivings for this batch with employee names
|
||||
const [receivings] = await prodConnection.query(`
|
||||
SELECT
|
||||
r.po_id,
|
||||
rp.pid,
|
||||
rp.receiving_id,
|
||||
rp.qty_each,
|
||||
rp.cost_each,
|
||||
COALESCE(rp.received_date, r.date_created) as received_date,
|
||||
rp.received_by,
|
||||
CONCAT(e.firstname, ' ', e.lastname) as received_by_name,
|
||||
CASE
|
||||
WHEN r.po_id IS NULL THEN 2 -- No PO
|
||||
WHEN r.po_id IN (?) THEN 0 -- Original PO
|
||||
ELSE 1 -- Different PO
|
||||
END as is_alt_po
|
||||
FROM receivings_products rp
|
||||
USE INDEX (received_date)
|
||||
LEFT JOIN receivings r ON r.receiving_id = rp.receiving_id
|
||||
LEFT JOIN employees e ON rp.received_by = e.employeeid
|
||||
WHERE rp.pid IN (?)
|
||||
AND rp.received_date >= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR)
|
||||
ORDER BY r.po_id, rp.pid, rp.received_date
|
||||
`, [batchPoIds, productPids]);
|
||||
|
||||
// Create maps for this sub-batch
|
||||
const poProductMap = new Map();
|
||||
productBatch.forEach(product => {
|
||||
const key = `${product.po_id}-${product.pid}`;
|
||||
poProductMap.set(key, product);
|
||||
});
|
||||
|
||||
const receivingMap = new Map();
|
||||
const altReceivingMap = new Map();
|
||||
const noPOReceivingMap = new Map();
|
||||
|
||||
receivings.forEach(receiving => {
|
||||
const key = `${receiving.po_id}-${receiving.pid}`;
|
||||
if (receiving.is_alt_po === 2) {
|
||||
// No PO
|
||||
if (!noPOReceivingMap.has(receiving.pid)) {
|
||||
noPOReceivingMap.set(receiving.pid, []);
|
||||
}
|
||||
noPOReceivingMap.get(receiving.pid).push(receiving);
|
||||
} else if (receiving.is_alt_po === 1) {
|
||||
// Different PO
|
||||
if (!altReceivingMap.has(receiving.pid)) {
|
||||
altReceivingMap.set(receiving.pid, []);
|
||||
}
|
||||
altReceivingMap.get(receiving.pid).push(receiving);
|
||||
} else {
|
||||
// Original PO
|
||||
if (!receivingMap.has(key)) {
|
||||
receivingMap.set(key, []);
|
||||
}
|
||||
receivingMap.get(key).push(receiving);
|
||||
}
|
||||
});
|
||||
|
||||
// Verify PIDs exist
|
||||
const [existingPids] = await localConnection.query(
|
||||
'SELECT pid FROM products WHERE pid IN (?)',
|
||||
[productPids]
|
||||
);
|
||||
const validPids = new Set(existingPids.map(p => p.pid));
|
||||
|
||||
// First check which PO lines already exist and get their current values
|
||||
const poLines = Array.from(poProductMap.values())
|
||||
.filter(p => validPids.has(p.pid))
|
||||
.map(p => [p.po_id, p.pid]);
|
||||
|
||||
const [existingPOs] = await localConnection.query(
|
||||
`SELECT ${columnNames.join(',')} FROM purchase_orders WHERE (po_id, pid) IN (${poLines.map(() => "(?,?)").join(",")})`,
|
||||
poLines.flat()
|
||||
);
|
||||
const existingPOMap = new Map(
|
||||
existingPOs.map(po => [`${po.po_id}-${po.pid}`, po])
|
||||
);
|
||||
|
||||
// Split into inserts and updates
|
||||
const insertsAndUpdates = { inserts: [], updates: [] };
|
||||
let batchProcessed = 0;
|
||||
|
||||
for (const po of batch) {
|
||||
const poProducts = Array.from(poProductMap.values())
|
||||
.filter(p => p.po_id === po.po_id && validPids.has(p.pid));
|
||||
|
||||
for (const product of poProducts) {
|
||||
const key = `${po.po_id}-${product.pid}`;
|
||||
const receivingHistory = receivingMap.get(key) || [];
|
||||
const altReceivingHistory = altReceivingMap.get(product.pid) || [];
|
||||
const noPOReceivingHistory = noPOReceivingMap.get(product.pid) || [];
|
||||
|
||||
// Combine all receivings and sort by date
|
||||
const allReceivings = [
|
||||
...receivingHistory.map(r => ({ ...r, type: 'original' })),
|
||||
...altReceivingHistory.map(r => ({ ...r, type: 'alternate' })),
|
||||
...noPOReceivingHistory.map(r => ({ ...r, type: 'no_po' }))
|
||||
].sort((a, b) => new Date(a.received_date || '9999-12-31') - new Date(b.received_date || '9999-12-31'));
|
||||
|
||||
// Split receivings into original PO and others
|
||||
const originalPOReceivings = allReceivings.filter(r => r.type === 'original');
|
||||
const otherReceivings = allReceivings.filter(r => r.type !== 'original');
|
||||
|
||||
// Track FIFO fulfillment
|
||||
let remainingToFulfill = product.ordered;
|
||||
const fulfillmentTracking = [];
|
||||
let totalReceived = 0;
|
||||
let actualCost = null; // Will store the cost of the first receiving that fulfills this PO
|
||||
let firstFulfillmentReceiving = null;
|
||||
let lastFulfillmentReceiving = null;
|
||||
|
||||
for (const receiving of allReceivings) {
|
||||
const qtyToApply = Math.min(remainingToFulfill, receiving.qty_each);
|
||||
if (qtyToApply > 0) {
|
||||
// If this is the first receiving being applied, use its cost
|
||||
if (actualCost === null) {
|
||||
actualCost = receiving.cost_each;
|
||||
firstFulfillmentReceiving = receiving;
|
||||
}
|
||||
lastFulfillmentReceiving = receiving;
|
||||
fulfillmentTracking.push({
|
||||
receiving_id: receiving.receiving_id,
|
||||
qty_applied: qtyToApply,
|
||||
qty_total: receiving.qty_each,
|
||||
cost: receiving.cost_each,
|
||||
date: receiving.received_date,
|
||||
received_by: receiving.received_by,
|
||||
received_by_name: receiving.received_by_name || 'Unknown',
|
||||
type: receiving.type,
|
||||
remaining_qty: receiving.qty_each - qtyToApply
|
||||
});
|
||||
remainingToFulfill -= qtyToApply;
|
||||
} else {
|
||||
// Track excess receivings
|
||||
fulfillmentTracking.push({
|
||||
receiving_id: receiving.receiving_id,
|
||||
qty_applied: 0,
|
||||
qty_total: receiving.qty_each,
|
||||
cost: receiving.cost_each,
|
||||
date: receiving.received_date,
|
||||
received_by: receiving.received_by,
|
||||
received_by_name: receiving.received_by_name || 'Unknown',
|
||||
type: receiving.type,
|
||||
is_excess: true
|
||||
});
|
||||
}
|
||||
totalReceived += receiving.qty_each;
|
||||
}
|
||||
|
||||
const receiving_status = !totalReceived ? 1 : // created
|
||||
remainingToFulfill > 0 ? 30 : // partial
|
||||
40; // full
|
||||
|
||||
function formatDate(dateStr) {
|
||||
if (!dateStr) return null;
|
||||
if (dateStr === '0000-00-00' || dateStr === '0000-00-00 00:00:00') return null;
|
||||
if (typeof dateStr === 'string' && !dateStr.match(/^\d{4}-\d{2}-\d{2}/)) return null;
|
||||
try {
|
||||
const date = new Date(dateStr);
|
||||
if (isNaN(date.getTime())) return null;
|
||||
if (date.getFullYear() < 1900 || date.getFullYear() > 2100) return null;
|
||||
return date.toISOString().split('T')[0];
|
||||
} catch (e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const rowValues = columnNames.map(col => {
|
||||
switch (col) {
|
||||
case 'po_id': return po.po_id;
|
||||
case 'vendor': return po.vendor;
|
||||
case 'date': return formatDate(po.date);
|
||||
case 'expected_date': return formatDate(po.expected_date);
|
||||
case 'pid': return product.pid;
|
||||
case 'sku': return product.sku;
|
||||
case 'name': return product.name;
|
||||
case 'cost_price': return actualCost || product.cost_each;
|
||||
case 'po_cost_price': return product.cost_each;
|
||||
case 'status': return po.status;
|
||||
case 'notes': return po.notes;
|
||||
case 'long_note': return po.long_note;
|
||||
case 'ordered': return product.ordered;
|
||||
case 'received': return totalReceived;
|
||||
case 'unfulfilled': return remainingToFulfill;
|
||||
case 'excess_received': return Math.max(0, totalReceived - product.ordered);
|
||||
case 'received_date': return formatDate(firstFulfillmentReceiving?.received_date);
|
||||
case 'last_received_date': return formatDate(lastFulfillmentReceiving?.received_date);
|
||||
case 'received_by': return firstFulfillmentReceiving?.received_by_name || null;
|
||||
case 'receiving_status': return receiving_status;
|
||||
case 'receiving_history': return JSON.stringify({
|
||||
fulfillment: fulfillmentTracking,
|
||||
ordered_qty: product.ordered,
|
||||
total_received: totalReceived,
|
||||
remaining_unfulfilled: remainingToFulfill,
|
||||
excess_received: Math.max(0, totalReceived - product.ordered),
|
||||
po_cost: product.cost_each,
|
||||
actual_cost: actualCost || product.cost_each
|
||||
});
|
||||
default: return null;
|
||||
}
|
||||
});
|
||||
|
||||
if (existingPOMap.has(key)) {
|
||||
const existing = existingPOMap.get(key);
|
||||
// Check if any values are different
|
||||
const hasChanges = columnNames.some(col => {
|
||||
const newVal = rowValues[columnNames.indexOf(col)];
|
||||
const oldVal = existing[col] ?? null;
|
||||
// Special handling for numbers to avoid type coercion issues
|
||||
if (typeof newVal === 'number' && typeof oldVal === 'number') {
|
||||
return Math.abs(newVal - oldVal) > 0.00001; // Allow for tiny floating point differences
|
||||
}
|
||||
// Special handling for receiving_history - parse and compare
|
||||
if (col === 'receiving_history') {
|
||||
const newHistory = JSON.parse(newVal || '{}');
|
||||
const oldHistory = JSON.parse(oldVal || '{}');
|
||||
return JSON.stringify(newHistory) !== JSON.stringify(oldHistory);
|
||||
}
|
||||
return newVal !== oldVal;
|
||||
});
|
||||
|
||||
if (hasChanges) {
|
||||
insertsAndUpdates.updates.push({
|
||||
po_id: po.po_id,
|
||||
pid: product.pid,
|
||||
values: rowValues
|
||||
});
|
||||
}
|
||||
} else {
|
||||
insertsAndUpdates.inserts.push({
|
||||
po_id: po.po_id,
|
||||
pid: product.pid,
|
||||
values: rowValues
|
||||
});
|
||||
}
|
||||
batchProcessed++;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle inserts
|
||||
if (insertsAndUpdates.inserts.length > 0) {
|
||||
const insertPlaceholders = insertsAndUpdates.inserts
|
||||
.map(() => `(${Array(columnNames.length).fill("?").join(",")})`)
|
||||
.join(",");
|
||||
|
||||
const insertResult = await localConnection.query(`
|
||||
INSERT INTO purchase_orders (${columnNames.join(",")})
|
||||
VALUES ${insertPlaceholders}
|
||||
`, insertsAndUpdates.inserts.map(i => i.values).flat());
|
||||
|
||||
const affectedRows = insertResult[0].affectedRows;
|
||||
// For an upsert, MySQL counts rows twice for updates
|
||||
// So if affectedRows is odd, we have (updates * 2 + inserts)
|
||||
const updates = Math.floor(affectedRows / 2);
|
||||
const inserts = affectedRows - (updates * 2);
|
||||
|
||||
recordsAdded += inserts;
|
||||
recordsUpdated += Math.floor(updates); // Ensure we never have fractional updates
|
||||
processed += batchProcessed;
|
||||
}
|
||||
|
||||
// Handle updates - now we know these actually have changes
|
||||
if (insertsAndUpdates.updates.length > 0) {
|
||||
const updatePlaceholders = insertsAndUpdates.updates
|
||||
.map(() => `(${Array(columnNames.length).fill("?").join(",")})`)
|
||||
.join(",");
|
||||
|
||||
const updateResult = await localConnection.query(`
|
||||
INSERT INTO purchase_orders (${columnNames.join(",")})
|
||||
VALUES ${updatePlaceholders}
|
||||
ON DUPLICATE KEY UPDATE ${columnNames
|
||||
.filter((col) => col !== "po_id" && col !== "pid")
|
||||
.map((col) => `${col} = VALUES(${col})`)
|
||||
.join(",")};
|
||||
`, insertsAndUpdates.updates.map(u => u.values).flat());
|
||||
|
||||
const affectedRows = updateResult[0].affectedRows;
|
||||
// For an upsert, MySQL counts rows twice for updates
|
||||
// So if affectedRows is odd, we have (updates * 2 + inserts)
|
||||
const updates = Math.floor(affectedRows / 2);
|
||||
const inserts = affectedRows - (updates * 2);
|
||||
|
||||
recordsUpdated += Math.floor(updates); // Ensure we never have fractional updates
|
||||
processed += batchProcessed;
|
||||
}
|
||||
|
||||
// Update progress based on time interval
|
||||
const now = Date.now();
|
||||
if (now - lastProgressUpdate >= PROGRESS_INTERVAL || processed === totalItems) {
|
||||
outputProgress({
|
||||
status: "running",
|
||||
operation: "Purchase orders import",
|
||||
current: processed,
|
||||
total: totalItems,
|
||||
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
|
||||
remaining: estimateRemaining(startTime, processed, totalItems),
|
||||
rate: calculateRate(startTime, processed)
|
||||
});
|
||||
lastProgressUpdate = now;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Only update sync status if we get here (no errors thrown)
|
||||
await localConnection.query(`
|
||||
INSERT INTO sync_status (table_name, last_sync_timestamp)
|
||||
VALUES ('purchase_orders', NOW())
|
||||
ON DUPLICATE KEY UPDATE
|
||||
last_sync_timestamp = NOW(),
|
||||
last_sync_id = LAST_INSERT_ID(last_sync_id)
|
||||
`);
|
||||
|
||||
return {
|
||||
status: "complete",
|
||||
totalImported: totalItems,
|
||||
recordsAdded: recordsAdded || 0,
|
||||
recordsUpdated: recordsUpdated || 0,
|
||||
incrementalUpdate,
|
||||
lastSyncTime
|
||||
};
|
||||
|
||||
} catch (error) {
|
||||
outputProgress({
|
||||
operation: `${incrementalUpdate ? 'Incremental' : 'Full'} purchase orders import failed`,
|
||||
status: "error",
|
||||
error: error.message,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = importPurchaseOrders;
|
||||
@@ -1,82 +0,0 @@
|
||||
// Split into inserts and updates
|
||||
const insertsAndUpdates = batch.reduce((acc, po) => {
|
||||
const key = `${po.po_id}-${po.pid}`;
|
||||
if (existingPOMap.has(key)) {
|
||||
const existing = existingPOMap.get(key);
|
||||
// Check if any values are different
|
||||
const hasChanges = columnNames.some(col => {
|
||||
const newVal = po[col] ?? null;
|
||||
const oldVal = existing[col] ?? null;
|
||||
// Special handling for numbers to avoid type coercion issues
|
||||
if (typeof newVal === 'number' && typeof oldVal === 'number') {
|
||||
return Math.abs(newVal - oldVal) > 0.00001; // Allow for tiny floating point differences
|
||||
}
|
||||
// Special handling for receiving_history JSON
|
||||
if (col === 'receiving_history') {
|
||||
return JSON.stringify(newVal) !== JSON.stringify(oldVal);
|
||||
}
|
||||
return newVal !== oldVal;
|
||||
});
|
||||
|
||||
if (hasChanges) {
|
||||
console.log(`PO line changed: ${key}`, {
|
||||
po_id: po.po_id,
|
||||
pid: po.pid,
|
||||
changes: columnNames.filter(col => {
|
||||
const newVal = po[col] ?? null;
|
||||
const oldVal = existing[col] ?? null;
|
||||
if (typeof newVal === 'number' && typeof oldVal === 'number') {
|
||||
return Math.abs(newVal - oldVal) > 0.00001;
|
||||
}
|
||||
if (col === 'receiving_history') {
|
||||
return JSON.stringify(newVal) !== JSON.stringify(oldVal);
|
||||
}
|
||||
return newVal !== oldVal;
|
||||
})
|
||||
});
|
||||
acc.updates.push({
|
||||
po_id: po.po_id,
|
||||
pid: po.pid,
|
||||
values: columnNames.map(col => po[col] ?? null)
|
||||
});
|
||||
}
|
||||
} else {
|
||||
console.log(`New PO line: ${key}`);
|
||||
acc.inserts.push({
|
||||
po_id: po.po_id,
|
||||
pid: po.pid,
|
||||
values: columnNames.map(col => po[col] ?? null)
|
||||
});
|
||||
}
|
||||
return acc;
|
||||
}, { inserts: [], updates: [] });
|
||||
|
||||
// Handle inserts
|
||||
if (insertsAndUpdates.inserts.length > 0) {
|
||||
const insertPlaceholders = Array(insertsAndUpdates.inserts.length).fill(placeholderGroup).join(",");
|
||||
|
||||
const insertResult = await localConnection.query(`
|
||||
INSERT INTO purchase_orders (${columnNames.join(",")})
|
||||
VALUES ${insertPlaceholders}
|
||||
`, insertsAndUpdates.inserts.map(i => i.values).flat());
|
||||
|
||||
recordsAdded += insertResult[0].affectedRows;
|
||||
}
|
||||
|
||||
// Handle updates
|
||||
if (insertsAndUpdates.updates.length > 0) {
|
||||
const updatePlaceholders = Array(insertsAndUpdates.updates.length).fill(placeholderGroup).join(",");
|
||||
|
||||
const updateResult = await localConnection.query(`
|
||||
INSERT INTO purchase_orders (${columnNames.join(",")})
|
||||
VALUES ${updatePlaceholders}
|
||||
ON DUPLICATE KEY UPDATE
|
||||
${columnNames
|
||||
.filter(col => col !== "po_id" && col !== "pid")
|
||||
.map(col => `${col} = VALUES(${col})`)
|
||||
.join(",")};
|
||||
`, insertsAndUpdates.updates.map(u => u.values).flat());
|
||||
|
||||
// Each update affects 2 rows in affectedRows, so we divide by 2 to get actual count
|
||||
recordsUpdated += insertsAndUpdates.updates.length;
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
const mysql = require("mysql2/promise");
|
||||
const { Client } = require("ssh2");
|
||||
const dotenv = require("dotenv");
|
||||
const path = require("path");
|
||||
|
||||
// Helper function to setup SSH tunnel
|
||||
async function setupSshTunnel(sshConfig) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const ssh = new Client();
|
||||
|
||||
ssh.on('error', (err) => {
|
||||
console.error('SSH connection error:', err);
|
||||
});
|
||||
|
||||
ssh.on('end', () => {
|
||||
console.log('SSH connection ended normally');
|
||||
});
|
||||
|
||||
ssh.on('close', () => {
|
||||
console.log('SSH connection closed');
|
||||
});
|
||||
|
||||
ssh
|
||||
.on("ready", () => {
|
||||
ssh.forwardOut(
|
||||
"127.0.0.1",
|
||||
0,
|
||||
sshConfig.prodDbConfig.host,
|
||||
sshConfig.prodDbConfig.port,
|
||||
async (err, stream) => {
|
||||
if (err) reject(err);
|
||||
resolve({ ssh, stream });
|
||||
}
|
||||
);
|
||||
})
|
||||
.connect(sshConfig.ssh);
|
||||
});
|
||||
}
|
||||
|
||||
// Helper function to setup database connections
|
||||
async function setupConnections(sshConfig) {
|
||||
const tunnel = await setupSshTunnel(sshConfig);
|
||||
|
||||
const prodConnection = await mysql.createConnection({
|
||||
...sshConfig.prodDbConfig,
|
||||
stream: tunnel.stream,
|
||||
});
|
||||
|
||||
const localConnection = await mysql.createPool({
|
||||
...sshConfig.localDbConfig,
|
||||
waitForConnections: true,
|
||||
connectionLimit: 10,
|
||||
queueLimit: 0
|
||||
});
|
||||
|
||||
return {
|
||||
ssh: tunnel.ssh,
|
||||
prodConnection,
|
||||
localConnection
|
||||
};
|
||||
}
|
||||
|
||||
// Helper function to close connections
|
||||
async function closeConnections(connections) {
|
||||
const { ssh, prodConnection, localConnection } = connections;
|
||||
|
||||
try {
|
||||
if (prodConnection) await prodConnection.end();
|
||||
if (localConnection) await localConnection.end();
|
||||
|
||||
// Wait a bit for any pending data to be written before closing SSH
|
||||
await new Promise(resolve => setTimeout(resolve, 100));
|
||||
|
||||
if (ssh) {
|
||||
ssh.on('close', () => {
|
||||
console.log('SSH connection closed cleanly');
|
||||
});
|
||||
ssh.end();
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Error during cleanup:', err);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
setupConnections,
|
||||
closeConnections
|
||||
};
|
||||
135
inventory-server/scripts/metrics-new/calculate_brand_metrics.sql
Normal file
135
inventory-server/scripts/metrics-new/calculate_brand_metrics.sql
Normal file
@@ -0,0 +1,135 @@
|
||||
-- Description: Calculates and updates aggregated metrics per brand.
|
||||
-- Dependencies: product_metrics, products, calculate_status table.
|
||||
-- Frequency: Daily (after product_metrics update).
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
_module_name VARCHAR := 'brand_metrics';
|
||||
_start_time TIMESTAMPTZ := clock_timestamp();
|
||||
_min_revenue NUMERIC := 50.00; -- Minimum revenue threshold for margin calculation
|
||||
BEGIN
|
||||
RAISE NOTICE 'Running % calculation...', _module_name;
|
||||
|
||||
WITH BrandAggregates AS (
|
||||
-- Aggregate metrics from product_metrics table per brand
|
||||
SELECT
|
||||
COALESCE(p.brand, 'Unbranded') AS brand_group, -- Group NULL/empty brands together
|
||||
COUNT(DISTINCT pm.pid) AS product_count,
|
||||
COUNT(DISTINCT CASE WHEN pm.is_visible THEN pm.pid END) AS active_product_count,
|
||||
COUNT(DISTINCT CASE WHEN pm.is_replenishable THEN pm.pid END) AS replenishable_product_count,
|
||||
SUM(pm.current_stock) AS current_stock_units,
|
||||
SUM(pm.current_stock_cost) AS current_stock_cost,
|
||||
SUM(pm.current_stock_retail) AS current_stock_retail,
|
||||
-- Only include products with valid sales data in each time period
|
||||
COUNT(DISTINCT CASE WHEN pm.sales_7d > 0 THEN pm.pid END) AS products_with_sales_7d,
|
||||
SUM(CASE WHEN pm.sales_7d > 0 THEN pm.sales_7d ELSE 0 END) AS sales_7d,
|
||||
SUM(CASE WHEN pm.revenue_7d > 0 THEN pm.revenue_7d ELSE 0 END) AS revenue_7d,
|
||||
|
||||
COUNT(DISTINCT CASE WHEN pm.sales_30d > 0 THEN pm.pid END) AS products_with_sales_30d,
|
||||
SUM(CASE WHEN pm.sales_30d > 0 THEN pm.sales_30d ELSE 0 END) AS sales_30d,
|
||||
SUM(CASE WHEN pm.revenue_30d > 0 THEN pm.revenue_30d ELSE 0 END) AS revenue_30d,
|
||||
SUM(CASE WHEN pm.cogs_30d > 0 THEN pm.cogs_30d ELSE 0 END) AS cogs_30d,
|
||||
SUM(CASE WHEN pm.profit_30d != 0 THEN pm.profit_30d ELSE 0 END) AS profit_30d,
|
||||
|
||||
COUNT(DISTINCT CASE WHEN pm.sales_365d > 0 THEN pm.pid END) AS products_with_sales_365d,
|
||||
SUM(CASE WHEN pm.sales_365d > 0 THEN pm.sales_365d ELSE 0 END) AS sales_365d,
|
||||
SUM(CASE WHEN pm.revenue_365d > 0 THEN pm.revenue_365d ELSE 0 END) AS revenue_365d,
|
||||
|
||||
COUNT(DISTINCT CASE WHEN pm.lifetime_sales > 0 THEN pm.pid END) AS products_with_lifetime_sales,
|
||||
SUM(CASE WHEN pm.lifetime_sales > 0 THEN pm.lifetime_sales ELSE 0 END) AS lifetime_sales,
|
||||
SUM(CASE WHEN pm.lifetime_revenue > 0 THEN pm.lifetime_revenue ELSE 0 END) AS lifetime_revenue
|
||||
FROM public.product_metrics pm
|
||||
JOIN public.products p ON pm.pid = p.pid
|
||||
GROUP BY brand_group
|
||||
),
|
||||
AllBrands AS (
|
||||
-- Ensure all brands from products table are included, mapping NULL/empty to 'Unbranded'
|
||||
SELECT DISTINCT COALESCE(brand, 'Unbranded') as brand_group
|
||||
FROM public.products
|
||||
)
|
||||
INSERT INTO public.brand_metrics (
|
||||
brand_name, last_calculated,
|
||||
product_count, active_product_count, replenishable_product_count,
|
||||
current_stock_units, current_stock_cost, current_stock_retail,
|
||||
sales_7d, revenue_7d, sales_30d, revenue_30d, profit_30d, cogs_30d,
|
||||
sales_365d, revenue_365d, lifetime_sales, lifetime_revenue,
|
||||
avg_margin_30d
|
||||
)
|
||||
SELECT
|
||||
b.brand_group,
|
||||
_start_time,
|
||||
-- Base Aggregates
|
||||
COALESCE(ba.product_count, 0),
|
||||
COALESCE(ba.active_product_count, 0),
|
||||
COALESCE(ba.replenishable_product_count, 0),
|
||||
COALESCE(ba.current_stock_units, 0),
|
||||
COALESCE(ba.current_stock_cost, 0.00),
|
||||
COALESCE(ba.current_stock_retail, 0.00),
|
||||
-- Sales Aggregates
|
||||
COALESCE(ba.sales_7d, 0), COALESCE(ba.revenue_7d, 0.00),
|
||||
COALESCE(ba.sales_30d, 0), COALESCE(ba.revenue_30d, 0.00),
|
||||
COALESCE(ba.profit_30d, 0.00), COALESCE(ba.cogs_30d, 0.00),
|
||||
COALESCE(ba.sales_365d, 0), COALESCE(ba.revenue_365d, 0.00),
|
||||
COALESCE(ba.lifetime_sales, 0), COALESCE(ba.lifetime_revenue, 0.00),
|
||||
-- KPIs - Calculate margin only for brands with significant revenue
|
||||
CASE
|
||||
WHEN COALESCE(ba.revenue_30d, 0) >= _min_revenue THEN
|
||||
-- Directly calculate margin from revenue and cogs for consistency
|
||||
-- This is mathematically equivalent to profit/revenue but more explicit
|
||||
((COALESCE(ba.revenue_30d, 0) - COALESCE(ba.cogs_30d, 0)) / COALESCE(ba.revenue_30d, 1)) * 100.0
|
||||
ELSE NULL -- No margin for low/no revenue brands
|
||||
END
|
||||
FROM AllBrands b
|
||||
LEFT JOIN BrandAggregates ba ON b.brand_group = ba.brand_group
|
||||
|
||||
ON CONFLICT (brand_name) DO UPDATE SET
|
||||
last_calculated = EXCLUDED.last_calculated,
|
||||
product_count = EXCLUDED.product_count,
|
||||
active_product_count = EXCLUDED.active_product_count,
|
||||
replenishable_product_count = EXCLUDED.replenishable_product_count,
|
||||
current_stock_units = EXCLUDED.current_stock_units,
|
||||
current_stock_cost = EXCLUDED.current_stock_cost,
|
||||
current_stock_retail = EXCLUDED.current_stock_retail,
|
||||
sales_7d = EXCLUDED.sales_7d, revenue_7d = EXCLUDED.revenue_7d,
|
||||
sales_30d = EXCLUDED.sales_30d, revenue_30d = EXCLUDED.revenue_30d,
|
||||
profit_30d = EXCLUDED.profit_30d, cogs_30d = EXCLUDED.cogs_30d,
|
||||
sales_365d = EXCLUDED.sales_365d, revenue_365d = EXCLUDED.revenue_365d,
|
||||
lifetime_sales = EXCLUDED.lifetime_sales, lifetime_revenue = EXCLUDED.lifetime_revenue,
|
||||
avg_margin_30d = EXCLUDED.avg_margin_30d
|
||||
WHERE -- Only update if at least one value has changed
|
||||
brand_metrics.product_count IS DISTINCT FROM EXCLUDED.product_count OR
|
||||
brand_metrics.active_product_count IS DISTINCT FROM EXCLUDED.active_product_count OR
|
||||
brand_metrics.current_stock_units IS DISTINCT FROM EXCLUDED.current_stock_units OR
|
||||
brand_metrics.sales_30d IS DISTINCT FROM EXCLUDED.sales_30d OR
|
||||
brand_metrics.revenue_30d IS DISTINCT FROM EXCLUDED.revenue_30d OR
|
||||
brand_metrics.lifetime_sales IS DISTINCT FROM EXCLUDED.lifetime_sales;
|
||||
|
||||
-- Update calculate_status
|
||||
INSERT INTO public.calculate_status (module_name, last_calculation_timestamp)
|
||||
VALUES (_module_name, _start_time)
|
||||
ON CONFLICT (module_name) DO UPDATE SET last_calculation_timestamp = _start_time;
|
||||
|
||||
RAISE NOTICE 'Finished % calculation. Duration: %', _module_name, clock_timestamp() - _start_time;
|
||||
END $$;
|
||||
|
||||
-- Return metrics about the update operation for tracking
|
||||
WITH update_stats AS (
|
||||
SELECT
|
||||
COUNT(*) as total_brands,
|
||||
COUNT(*) FILTER (WHERE last_calculated >= NOW() - INTERVAL '5 minutes') as rows_processed,
|
||||
SUM(product_count) as total_products,
|
||||
SUM(active_product_count) as total_active_products,
|
||||
SUM(sales_30d) as total_sales_30d,
|
||||
SUM(revenue_30d) as total_revenue_30d,
|
||||
AVG(avg_margin_30d) as overall_avg_margin_30d
|
||||
FROM public.brand_metrics
|
||||
)
|
||||
SELECT
|
||||
rows_processed,
|
||||
total_brands,
|
||||
total_products::int,
|
||||
total_active_products::int,
|
||||
total_sales_30d::int,
|
||||
ROUND(total_revenue_30d, 2) as total_revenue_30d,
|
||||
ROUND(overall_avg_margin_30d, 2) as overall_avg_margin_30d
|
||||
FROM update_stats;
|
||||
@@ -0,0 +1,309 @@
|
||||
-- Description: Calculates and updates aggregated metrics per category.
|
||||
-- Dependencies: product_metrics, products, categories, product_categories, calculate_status table.
|
||||
-- Frequency: Daily (after product_metrics update).
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
_module_name VARCHAR := 'category_metrics';
|
||||
_start_time TIMESTAMPTZ := clock_timestamp();
|
||||
_min_revenue NUMERIC := 50.00; -- Minimum revenue threshold for margin calculation
|
||||
BEGIN
|
||||
RAISE NOTICE 'Running % calculation...', _module_name;
|
||||
|
||||
WITH
|
||||
-- Identify the hierarchy depth for each category
|
||||
CategoryDepth AS (
|
||||
WITH RECURSIVE CategoryTree AS (
|
||||
-- Base case: Start with categories without parents (root categories)
|
||||
SELECT cat_id, name, parent_id, 0 AS depth
|
||||
FROM public.categories
|
||||
WHERE parent_id IS NULL
|
||||
|
||||
UNION ALL
|
||||
|
||||
-- Recursive step: Add child categories with incremented depth
|
||||
SELECT c.cat_id, c.name, c.parent_id, ct.depth + 1
|
||||
FROM public.categories c
|
||||
JOIN CategoryTree ct ON c.parent_id = ct.cat_id
|
||||
)
|
||||
SELECT cat_id, depth
|
||||
FROM CategoryTree
|
||||
),
|
||||
-- For each product, find the most specific (deepest) category it belongs to
|
||||
ProductDeepestCategory AS (
|
||||
SELECT
|
||||
pc.pid,
|
||||
pc.cat_id
|
||||
FROM public.product_categories pc
|
||||
JOIN CategoryDepth cd ON pc.cat_id = cd.cat_id
|
||||
-- This is the key part: for each product, select only the category with maximum depth
|
||||
WHERE (pc.pid, cd.depth) IN (
|
||||
SELECT pc2.pid, MAX(cd2.depth)
|
||||
FROM public.product_categories pc2
|
||||
JOIN CategoryDepth cd2 ON pc2.cat_id = cd2.cat_id
|
||||
GROUP BY pc2.pid
|
||||
)
|
||||
),
|
||||
-- Calculate metrics only at the most specific category level for each product
|
||||
-- These are the direct metrics (only products directly in this category)
|
||||
DirectCategoryMetrics AS (
|
||||
SELECT
|
||||
pdc.cat_id,
|
||||
-- Counts
|
||||
COUNT(DISTINCT pm.pid) AS product_count,
|
||||
COUNT(DISTINCT CASE WHEN pm.is_visible THEN pm.pid END) AS active_product_count,
|
||||
COUNT(DISTINCT CASE WHEN pm.is_replenishable THEN pm.pid END) AS replenishable_product_count,
|
||||
-- Current Stock
|
||||
SUM(pm.current_stock) AS current_stock_units,
|
||||
SUM(pm.current_stock_cost) AS current_stock_cost,
|
||||
SUM(pm.current_stock_retail) AS current_stock_retail,
|
||||
-- Rolling Periods - Only include products with actual sales in each period
|
||||
SUM(CASE WHEN pm.sales_7d > 0 THEN pm.sales_7d ELSE 0 END) AS sales_7d,
|
||||
SUM(CASE WHEN pm.revenue_7d > 0 THEN pm.revenue_7d ELSE 0 END) AS revenue_7d,
|
||||
SUM(CASE WHEN pm.sales_30d > 0 THEN pm.sales_30d ELSE 0 END) AS sales_30d,
|
||||
SUM(CASE WHEN pm.revenue_30d > 0 THEN pm.revenue_30d ELSE 0 END) AS revenue_30d,
|
||||
SUM(CASE WHEN pm.cogs_30d > 0 THEN pm.cogs_30d ELSE 0 END) AS cogs_30d,
|
||||
SUM(CASE WHEN pm.profit_30d != 0 THEN pm.profit_30d ELSE 0 END) AS profit_30d,
|
||||
SUM(CASE WHEN pm.sales_365d > 0 THEN pm.sales_365d ELSE 0 END) AS sales_365d,
|
||||
SUM(CASE WHEN pm.revenue_365d > 0 THEN pm.revenue_365d ELSE 0 END) AS revenue_365d,
|
||||
SUM(CASE WHEN pm.lifetime_sales > 0 THEN pm.lifetime_sales ELSE 0 END) AS lifetime_sales,
|
||||
SUM(CASE WHEN pm.lifetime_revenue > 0 THEN pm.lifetime_revenue ELSE 0 END) AS lifetime_revenue,
|
||||
-- Data for KPIs - Only average stock for products with stock
|
||||
SUM(CASE WHEN pm.avg_stock_units_30d > 0 THEN pm.avg_stock_units_30d ELSE 0 END) AS total_avg_stock_units_30d
|
||||
FROM public.product_metrics pm
|
||||
JOIN ProductDeepestCategory pdc ON pm.pid = pdc.pid
|
||||
GROUP BY pdc.cat_id
|
||||
),
|
||||
-- Build a category lookup table for parent relationships
|
||||
CategoryHierarchyPaths AS (
|
||||
WITH RECURSIVE ParentPaths AS (
|
||||
-- Base case: All categories with their immediate parents
|
||||
SELECT
|
||||
cat_id,
|
||||
cat_id as leaf_id, -- Every category is its own leaf initially
|
||||
ARRAY[cat_id] as path
|
||||
FROM public.categories
|
||||
|
||||
UNION ALL
|
||||
|
||||
-- Recursive step: Walk up the parent chain
|
||||
SELECT
|
||||
c.parent_id as cat_id,
|
||||
pp.leaf_id, -- Keep the original leaf_id
|
||||
c.parent_id || pp.path as path
|
||||
FROM ParentPaths pp
|
||||
JOIN public.categories c ON pp.cat_id = c.cat_id
|
||||
WHERE c.parent_id IS NOT NULL -- Stop at root categories
|
||||
)
|
||||
-- Select distinct paths to avoid duplication
|
||||
SELECT DISTINCT cat_id, leaf_id
|
||||
FROM ParentPaths
|
||||
),
|
||||
-- Aggregate metrics from leaf categories to their ancestors without duplication
|
||||
-- These are the rolled-up metrics (including all child categories)
|
||||
RollupMetrics AS (
|
||||
SELECT
|
||||
chp.cat_id,
|
||||
-- For each parent category, count distinct products to avoid duplication
|
||||
COUNT(DISTINCT dcm.cat_id) AS child_categories_count,
|
||||
SUM(dcm.product_count) AS rollup_product_count,
|
||||
SUM(dcm.active_product_count) AS rollup_active_product_count,
|
||||
SUM(dcm.replenishable_product_count) AS rollup_replenishable_product_count,
|
||||
SUM(dcm.current_stock_units) AS rollup_current_stock_units,
|
||||
SUM(dcm.current_stock_cost) AS rollup_current_stock_cost,
|
||||
SUM(dcm.current_stock_retail) AS rollup_current_stock_retail,
|
||||
SUM(dcm.sales_7d) AS rollup_sales_7d,
|
||||
SUM(dcm.revenue_7d) AS rollup_revenue_7d,
|
||||
SUM(dcm.sales_30d) AS rollup_sales_30d,
|
||||
SUM(dcm.revenue_30d) AS rollup_revenue_30d,
|
||||
SUM(dcm.cogs_30d) AS rollup_cogs_30d,
|
||||
SUM(dcm.profit_30d) AS rollup_profit_30d,
|
||||
SUM(dcm.sales_365d) AS rollup_sales_365d,
|
||||
SUM(dcm.revenue_365d) AS rollup_revenue_365d,
|
||||
SUM(dcm.lifetime_sales) AS rollup_lifetime_sales,
|
||||
SUM(dcm.lifetime_revenue) AS rollup_lifetime_revenue,
|
||||
SUM(dcm.total_avg_stock_units_30d) AS rollup_total_avg_stock_units_30d
|
||||
FROM CategoryHierarchyPaths chp
|
||||
JOIN DirectCategoryMetrics dcm ON chp.leaf_id = dcm.cat_id
|
||||
GROUP BY chp.cat_id
|
||||
),
|
||||
-- Combine direct and rollup metrics
|
||||
CombinedMetrics AS (
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
c.type,
|
||||
c.parent_id,
|
||||
-- Direct metrics (just this category)
|
||||
COALESCE(dcm.product_count, 0) AS direct_product_count,
|
||||
COALESCE(dcm.active_product_count, 0) AS direct_active_product_count,
|
||||
COALESCE(dcm.replenishable_product_count, 0) AS direct_replenishable_product_count,
|
||||
COALESCE(dcm.current_stock_units, 0) AS direct_current_stock_units,
|
||||
COALESCE(dcm.current_stock_cost, 0) AS direct_current_stock_cost,
|
||||
COALESCE(dcm.current_stock_retail, 0) AS direct_current_stock_retail,
|
||||
COALESCE(dcm.sales_7d, 0) AS direct_sales_7d,
|
||||
COALESCE(dcm.revenue_7d, 0) AS direct_revenue_7d,
|
||||
COALESCE(dcm.sales_30d, 0) AS direct_sales_30d,
|
||||
COALESCE(dcm.revenue_30d, 0) AS direct_revenue_30d,
|
||||
COALESCE(dcm.cogs_30d, 0) AS direct_cogs_30d,
|
||||
COALESCE(dcm.profit_30d, 0) AS direct_profit_30d,
|
||||
COALESCE(dcm.sales_365d, 0) AS direct_sales_365d,
|
||||
COALESCE(dcm.revenue_365d, 0) AS direct_revenue_365d,
|
||||
COALESCE(dcm.lifetime_sales, 0) AS direct_lifetime_sales,
|
||||
COALESCE(dcm.lifetime_revenue, 0) AS direct_lifetime_revenue,
|
||||
COALESCE(dcm.total_avg_stock_units_30d, 0) AS direct_avg_stock_units_30d,
|
||||
|
||||
-- Rolled up metrics (this category + all children)
|
||||
COALESCE(rm.rollup_product_count, 0) AS product_count,
|
||||
COALESCE(rm.rollup_active_product_count, 0) AS active_product_count,
|
||||
COALESCE(rm.rollup_replenishable_product_count, 0) AS replenishable_product_count,
|
||||
COALESCE(rm.rollup_current_stock_units, 0) AS current_stock_units,
|
||||
COALESCE(rm.rollup_current_stock_cost, 0) AS current_stock_cost,
|
||||
COALESCE(rm.rollup_current_stock_retail, 0) AS current_stock_retail,
|
||||
COALESCE(rm.rollup_sales_7d, 0) AS sales_7d,
|
||||
COALESCE(rm.rollup_revenue_7d, 0) AS revenue_7d,
|
||||
COALESCE(rm.rollup_sales_30d, 0) AS sales_30d,
|
||||
COALESCE(rm.rollup_revenue_30d, 0) AS revenue_30d,
|
||||
COALESCE(rm.rollup_cogs_30d, 0) AS cogs_30d,
|
||||
COALESCE(rm.rollup_profit_30d, 0) AS profit_30d,
|
||||
COALESCE(rm.rollup_sales_365d, 0) AS sales_365d,
|
||||
COALESCE(rm.rollup_revenue_365d, 0) AS revenue_365d,
|
||||
COALESCE(rm.rollup_lifetime_sales, 0) AS lifetime_sales,
|
||||
COALESCE(rm.rollup_lifetime_revenue, 0) AS lifetime_revenue,
|
||||
COALESCE(rm.rollup_total_avg_stock_units_30d, 0) AS total_avg_stock_units_30d
|
||||
FROM public.categories c
|
||||
LEFT JOIN DirectCategoryMetrics dcm ON c.cat_id = dcm.cat_id
|
||||
LEFT JOIN RollupMetrics rm ON c.cat_id = rm.cat_id
|
||||
)
|
||||
INSERT INTO public.category_metrics (
|
||||
category_id, category_name, category_type, parent_id, last_calculated,
|
||||
-- Store all direct and rolled up metrics
|
||||
product_count, active_product_count, replenishable_product_count,
|
||||
current_stock_units, current_stock_cost, current_stock_retail,
|
||||
sales_7d, revenue_7d, sales_30d, revenue_30d, profit_30d, cogs_30d,
|
||||
sales_365d, revenue_365d, lifetime_sales, lifetime_revenue,
|
||||
-- Also store direct metrics with direct_ prefix
|
||||
direct_product_count, direct_active_product_count, direct_replenishable_product_count,
|
||||
direct_current_stock_units, direct_stock_cost, direct_stock_retail,
|
||||
direct_sales_7d, direct_revenue_7d, direct_sales_30d, direct_revenue_30d,
|
||||
direct_profit_30d, direct_cogs_30d, direct_sales_365d, direct_revenue_365d,
|
||||
direct_lifetime_sales, direct_lifetime_revenue,
|
||||
-- KPIs
|
||||
avg_margin_30d, stock_turn_30d
|
||||
)
|
||||
SELECT
|
||||
cm.cat_id,
|
||||
cm.name,
|
||||
cm.type,
|
||||
cm.parent_id,
|
||||
_start_time,
|
||||
-- Rolled-up metrics (total including children)
|
||||
cm.product_count,
|
||||
cm.active_product_count,
|
||||
cm.replenishable_product_count,
|
||||
cm.current_stock_units,
|
||||
cm.current_stock_cost,
|
||||
cm.current_stock_retail,
|
||||
cm.sales_7d, cm.revenue_7d,
|
||||
cm.sales_30d, cm.revenue_30d, cm.profit_30d, cm.cogs_30d,
|
||||
cm.sales_365d, cm.revenue_365d,
|
||||
cm.lifetime_sales, cm.lifetime_revenue,
|
||||
-- Direct metrics (just this category)
|
||||
cm.direct_product_count,
|
||||
cm.direct_active_product_count,
|
||||
cm.direct_replenishable_product_count,
|
||||
cm.direct_current_stock_units,
|
||||
cm.direct_current_stock_cost,
|
||||
cm.direct_current_stock_retail,
|
||||
cm.direct_sales_7d, cm.direct_revenue_7d,
|
||||
cm.direct_sales_30d, cm.direct_revenue_30d, cm.direct_profit_30d, cm.direct_cogs_30d,
|
||||
cm.direct_sales_365d, cm.direct_revenue_365d,
|
||||
cm.direct_lifetime_sales, cm.direct_lifetime_revenue,
|
||||
-- KPIs - Calculate margin only for categories with significant revenue
|
||||
CASE
|
||||
WHEN cm.revenue_30d >= _min_revenue THEN
|
||||
((cm.revenue_30d - cm.cogs_30d) / cm.revenue_30d) * 100.0
|
||||
ELSE NULL -- No margin for low/no revenue categories
|
||||
END,
|
||||
-- Stock Turn calculation
|
||||
CASE
|
||||
WHEN cm.total_avg_stock_units_30d > 0 THEN
|
||||
cm.sales_30d / cm.total_avg_stock_units_30d
|
||||
ELSE NULL -- No stock turn if no average stock
|
||||
END
|
||||
FROM CombinedMetrics cm
|
||||
|
||||
ON CONFLICT (category_id) DO UPDATE SET
|
||||
category_name = EXCLUDED.category_name,
|
||||
category_type = EXCLUDED.category_type,
|
||||
parent_id = EXCLUDED.parent_id,
|
||||
last_calculated = EXCLUDED.last_calculated,
|
||||
|
||||
-- ROLLED-UP METRICS (includes this category + all descendants)
|
||||
product_count = EXCLUDED.product_count,
|
||||
active_product_count = EXCLUDED.active_product_count,
|
||||
replenishable_product_count = EXCLUDED.replenishable_product_count,
|
||||
current_stock_units = EXCLUDED.current_stock_units,
|
||||
current_stock_cost = EXCLUDED.current_stock_cost,
|
||||
current_stock_retail = EXCLUDED.current_stock_retail,
|
||||
sales_7d = EXCLUDED.sales_7d, revenue_7d = EXCLUDED.revenue_7d,
|
||||
sales_30d = EXCLUDED.sales_30d, revenue_30d = EXCLUDED.revenue_30d,
|
||||
profit_30d = EXCLUDED.profit_30d, cogs_30d = EXCLUDED.cogs_30d,
|
||||
sales_365d = EXCLUDED.sales_365d, revenue_365d = EXCLUDED.revenue_365d,
|
||||
lifetime_sales = EXCLUDED.lifetime_sales, lifetime_revenue = EXCLUDED.lifetime_revenue,
|
||||
|
||||
-- DIRECT METRICS (only products directly in this category)
|
||||
direct_product_count = EXCLUDED.direct_product_count,
|
||||
direct_active_product_count = EXCLUDED.direct_active_product_count,
|
||||
direct_replenishable_product_count = EXCLUDED.direct_replenishable_product_count,
|
||||
direct_current_stock_units = EXCLUDED.direct_current_stock_units,
|
||||
direct_stock_cost = EXCLUDED.direct_stock_cost,
|
||||
direct_stock_retail = EXCLUDED.direct_stock_retail,
|
||||
direct_sales_7d = EXCLUDED.direct_sales_7d, direct_revenue_7d = EXCLUDED.direct_revenue_7d,
|
||||
direct_sales_30d = EXCLUDED.direct_sales_30d, direct_revenue_30d = EXCLUDED.direct_revenue_30d,
|
||||
direct_profit_30d = EXCLUDED.direct_profit_30d, direct_cogs_30d = EXCLUDED.direct_cogs_30d,
|
||||
direct_sales_365d = EXCLUDED.direct_sales_365d, direct_revenue_365d = EXCLUDED.direct_revenue_365d,
|
||||
direct_lifetime_sales = EXCLUDED.direct_lifetime_sales, direct_lifetime_revenue = EXCLUDED.direct_lifetime_revenue,
|
||||
|
||||
-- Calculated KPIs
|
||||
avg_margin_30d = EXCLUDED.avg_margin_30d,
|
||||
stock_turn_30d = EXCLUDED.stock_turn_30d
|
||||
WHERE -- Only update if at least one value has changed
|
||||
category_metrics.product_count IS DISTINCT FROM EXCLUDED.product_count OR
|
||||
category_metrics.active_product_count IS DISTINCT FROM EXCLUDED.active_product_count OR
|
||||
category_metrics.current_stock_units IS DISTINCT FROM EXCLUDED.current_stock_units OR
|
||||
category_metrics.sales_30d IS DISTINCT FROM EXCLUDED.sales_30d OR
|
||||
category_metrics.revenue_30d IS DISTINCT FROM EXCLUDED.revenue_30d OR
|
||||
category_metrics.lifetime_sales IS DISTINCT FROM EXCLUDED.lifetime_sales OR
|
||||
category_metrics.direct_product_count IS DISTINCT FROM EXCLUDED.direct_product_count OR
|
||||
category_metrics.direct_sales_30d IS DISTINCT FROM EXCLUDED.direct_sales_30d;
|
||||
|
||||
-- Update calculate_status
|
||||
INSERT INTO public.calculate_status (module_name, last_calculation_timestamp)
|
||||
VALUES (_module_name, _start_time)
|
||||
ON CONFLICT (module_name) DO UPDATE SET last_calculation_timestamp = _start_time;
|
||||
|
||||
RAISE NOTICE 'Finished % calculation. Duration: %', _module_name, clock_timestamp() - _start_time;
|
||||
END $$;
|
||||
|
||||
-- Return metrics about the update operation for tracking
|
||||
WITH update_stats AS (
|
||||
SELECT
|
||||
COUNT(*) as total_categories,
|
||||
COUNT(*) FILTER (WHERE last_calculated >= NOW() - INTERVAL '5 minutes') as rows_processed,
|
||||
COUNT(*) FILTER (WHERE category_type = 11) as main_categories, -- 11 = category
|
||||
COUNT(*) FILTER (WHERE category_type = 12) as subcategories, -- 12 = subcategory
|
||||
SUM(product_count) as total_products,
|
||||
SUM(active_product_count) as total_active_products,
|
||||
SUM(current_stock_units) as total_stock_units
|
||||
FROM public.category_metrics
|
||||
)
|
||||
SELECT
|
||||
rows_processed,
|
||||
total_categories,
|
||||
main_categories,
|
||||
subcategories,
|
||||
total_products::int,
|
||||
total_active_products::int,
|
||||
total_stock_units::int
|
||||
FROM update_stats;
|
||||
@@ -1,224 +0,0 @@
|
||||
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress');
|
||||
const { getConnection } = require('./utils/db');
|
||||
|
||||
async function calculateBrandMetrics(startTime, totalProducts, processedCount, isCancelled = false) {
|
||||
const connection = await getConnection();
|
||||
try {
|
||||
if (isCancelled) {
|
||||
outputProgress({
|
||||
status: 'cancelled',
|
||||
operation: 'Brand metrics calculation cancelled',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: null,
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
return processedCount;
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Starting brand metrics calculation',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
// Calculate brand metrics with optimized queries
|
||||
await connection.query(`
|
||||
INSERT INTO brand_metrics (
|
||||
brand,
|
||||
product_count,
|
||||
active_products,
|
||||
total_stock_units,
|
||||
total_stock_cost,
|
||||
total_stock_retail,
|
||||
total_revenue,
|
||||
avg_margin,
|
||||
growth_rate
|
||||
)
|
||||
WITH filtered_products AS (
|
||||
SELECT
|
||||
p.*,
|
||||
CASE WHEN p.stock_quantity <= 5000 THEN p.pid END as valid_pid,
|
||||
CASE WHEN p.visible = true AND p.stock_quantity <= 5000 THEN p.pid END as active_pid,
|
||||
CASE
|
||||
WHEN p.stock_quantity IS NULL OR p.stock_quantity < 0 OR p.stock_quantity > 5000 THEN 0
|
||||
ELSE p.stock_quantity
|
||||
END as valid_stock
|
||||
FROM products p
|
||||
WHERE p.brand IS NOT NULL
|
||||
),
|
||||
sales_periods AS (
|
||||
SELECT
|
||||
p.brand,
|
||||
SUM(o.quantity * o.price) as period_revenue,
|
||||
CASE
|
||||
WHEN o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 3 MONTH) THEN 'current'
|
||||
WHEN o.date BETWEEN DATE_SUB(CURRENT_DATE, INTERVAL 15 MONTH) AND DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH) THEN 'previous'
|
||||
END as period_type
|
||||
FROM filtered_products p
|
||||
JOIN orders o ON p.pid = o.pid
|
||||
WHERE o.canceled = false
|
||||
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 15 MONTH)
|
||||
GROUP BY p.brand, period_type
|
||||
),
|
||||
brand_data AS (
|
||||
SELECT
|
||||
p.brand,
|
||||
COUNT(DISTINCT p.valid_pid) as product_count,
|
||||
COUNT(DISTINCT p.active_pid) as active_products,
|
||||
SUM(p.valid_stock) as total_stock_units,
|
||||
SUM(p.valid_stock * p.cost_price) as total_stock_cost,
|
||||
SUM(p.valid_stock * p.price) as total_stock_retail,
|
||||
COALESCE(SUM(o.quantity * o.price), 0) as total_revenue,
|
||||
CASE
|
||||
WHEN SUM(o.quantity * o.price) > 0 THEN
|
||||
(SUM((o.price - p.cost_price) * o.quantity) * 100.0) / SUM(o.price * o.quantity)
|
||||
ELSE 0
|
||||
END as avg_margin
|
||||
FROM filtered_products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid AND o.canceled = false
|
||||
GROUP BY p.brand
|
||||
)
|
||||
SELECT
|
||||
bd.brand,
|
||||
bd.product_count,
|
||||
bd.active_products,
|
||||
bd.total_stock_units,
|
||||
bd.total_stock_cost,
|
||||
bd.total_stock_retail,
|
||||
bd.total_revenue,
|
||||
bd.avg_margin,
|
||||
CASE
|
||||
WHEN MAX(CASE WHEN sp.period_type = 'previous' THEN sp.period_revenue END) = 0
|
||||
AND MAX(CASE WHEN sp.period_type = 'current' THEN sp.period_revenue END) > 0 THEN 100.0
|
||||
WHEN MAX(CASE WHEN sp.period_type = 'previous' THEN sp.period_revenue END) = 0 THEN 0.0
|
||||
ELSE LEAST(
|
||||
GREATEST(
|
||||
((MAX(CASE WHEN sp.period_type = 'current' THEN sp.period_revenue END) -
|
||||
MAX(CASE WHEN sp.period_type = 'previous' THEN sp.period_revenue END)) /
|
||||
NULLIF(MAX(CASE WHEN sp.period_type = 'previous' THEN sp.period_revenue END), 0)) * 100.0,
|
||||
-100.0
|
||||
),
|
||||
999.99
|
||||
)
|
||||
END as growth_rate
|
||||
FROM brand_data bd
|
||||
LEFT JOIN sales_periods sp ON bd.brand = sp.brand
|
||||
GROUP BY bd.brand, bd.product_count, bd.active_products, bd.total_stock_units,
|
||||
bd.total_stock_cost, bd.total_stock_retail, bd.total_revenue, bd.avg_margin
|
||||
ON DUPLICATE KEY UPDATE
|
||||
product_count = VALUES(product_count),
|
||||
active_products = VALUES(active_products),
|
||||
total_stock_units = VALUES(total_stock_units),
|
||||
total_stock_cost = VALUES(total_stock_cost),
|
||||
total_stock_retail = VALUES(total_stock_retail),
|
||||
total_revenue = VALUES(total_revenue),
|
||||
avg_margin = VALUES(avg_margin),
|
||||
growth_rate = VALUES(growth_rate),
|
||||
last_calculated_at = CURRENT_TIMESTAMP
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.97);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Brand metrics calculated, starting time-based metrics',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
if (isCancelled) return processedCount;
|
||||
|
||||
// Calculate brand time-based metrics with optimized query
|
||||
await connection.query(`
|
||||
INSERT INTO brand_time_metrics (
|
||||
brand,
|
||||
year,
|
||||
month,
|
||||
product_count,
|
||||
active_products,
|
||||
total_stock_units,
|
||||
total_stock_cost,
|
||||
total_stock_retail,
|
||||
total_revenue,
|
||||
avg_margin
|
||||
)
|
||||
WITH filtered_products AS (
|
||||
SELECT
|
||||
p.*,
|
||||
CASE WHEN p.stock_quantity <= 5000 THEN p.pid END as valid_pid,
|
||||
CASE WHEN p.visible = true AND p.stock_quantity <= 5000 THEN p.pid END as active_pid,
|
||||
CASE
|
||||
WHEN p.stock_quantity IS NULL OR p.stock_quantity < 0 OR p.stock_quantity > 5000 THEN 0
|
||||
ELSE p.stock_quantity
|
||||
END as valid_stock
|
||||
FROM products p
|
||||
WHERE p.brand IS NOT NULL
|
||||
),
|
||||
monthly_metrics AS (
|
||||
SELECT
|
||||
p.brand,
|
||||
YEAR(o.date) as year,
|
||||
MONTH(o.date) as month,
|
||||
COUNT(DISTINCT p.valid_pid) as product_count,
|
||||
COUNT(DISTINCT p.active_pid) as active_products,
|
||||
SUM(p.valid_stock) as total_stock_units,
|
||||
SUM(p.valid_stock * p.cost_price) as total_stock_cost,
|
||||
SUM(p.valid_stock * p.price) as total_stock_retail,
|
||||
SUM(o.quantity * o.price) as total_revenue,
|
||||
CASE
|
||||
WHEN SUM(o.quantity * o.price) > 0 THEN
|
||||
(SUM((o.price - p.cost_price) * o.quantity) * 100.0) / SUM(o.price * o.quantity)
|
||||
ELSE 0
|
||||
END as avg_margin
|
||||
FROM filtered_products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid AND o.canceled = false
|
||||
WHERE o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
|
||||
GROUP BY p.brand, YEAR(o.date), MONTH(o.date)
|
||||
)
|
||||
SELECT *
|
||||
FROM monthly_metrics
|
||||
ON DUPLICATE KEY UPDATE
|
||||
product_count = VALUES(product_count),
|
||||
active_products = VALUES(active_products),
|
||||
total_stock_units = VALUES(total_stock_units),
|
||||
total_stock_cost = VALUES(total_stock_cost),
|
||||
total_stock_retail = VALUES(total_stock_retail),
|
||||
total_revenue = VALUES(total_revenue),
|
||||
avg_margin = VALUES(avg_margin)
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.99);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Brand time-based metrics calculated',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
return processedCount;
|
||||
} catch (error) {
|
||||
logError(error, 'Error calculating brand metrics');
|
||||
throw error;
|
||||
} finally {
|
||||
if (connection) {
|
||||
connection.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = calculateBrandMetrics;
|
||||
@@ -1,239 +0,0 @@
|
||||
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress');
|
||||
const { getConnection } = require('./utils/db');
|
||||
|
||||
async function calculateCategoryMetrics(startTime, totalProducts, processedCount, isCancelled = false) {
|
||||
const connection = await getConnection();
|
||||
try {
|
||||
if (isCancelled) {
|
||||
outputProgress({
|
||||
status: 'cancelled',
|
||||
operation: 'Category metrics calculation cancelled',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: null,
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
return processedCount;
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Starting category metrics calculation',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
// First, calculate base category metrics
|
||||
await connection.query(`
|
||||
INSERT INTO category_metrics (
|
||||
category_id,
|
||||
product_count,
|
||||
active_products,
|
||||
total_value,
|
||||
status,
|
||||
last_calculated_at
|
||||
)
|
||||
SELECT
|
||||
c.cat_id,
|
||||
COUNT(DISTINCT p.pid) as product_count,
|
||||
COUNT(DISTINCT CASE WHEN p.visible = true THEN p.pid END) as active_products,
|
||||
COALESCE(SUM(p.stock_quantity * p.cost_price), 0) as total_value,
|
||||
c.status,
|
||||
NOW() as last_calculated_at
|
||||
FROM categories c
|
||||
LEFT JOIN product_categories pc ON c.cat_id = pc.cat_id
|
||||
LEFT JOIN products p ON pc.pid = p.pid
|
||||
GROUP BY c.cat_id, c.status
|
||||
ON DUPLICATE KEY UPDATE
|
||||
product_count = VALUES(product_count),
|
||||
active_products = VALUES(active_products),
|
||||
total_value = VALUES(total_value),
|
||||
status = VALUES(status),
|
||||
last_calculated_at = VALUES(last_calculated_at)
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.90);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Base category metrics calculated, updating with margin data',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
if (isCancelled) return processedCount;
|
||||
|
||||
// Then update with margin and turnover data
|
||||
await connection.query(`
|
||||
WITH category_sales AS (
|
||||
SELECT
|
||||
pc.cat_id,
|
||||
SUM(o.quantity * o.price) as total_sales,
|
||||
SUM(o.quantity * (o.price - p.cost_price)) as total_margin,
|
||||
SUM(o.quantity) as units_sold,
|
||||
AVG(GREATEST(p.stock_quantity, 0)) as avg_stock
|
||||
FROM product_categories pc
|
||||
JOIN products p ON pc.pid = p.pid
|
||||
JOIN orders o ON p.pid = o.pid
|
||||
WHERE o.canceled = false
|
||||
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 1 YEAR)
|
||||
GROUP BY pc.cat_id
|
||||
)
|
||||
UPDATE category_metrics cm
|
||||
JOIN category_sales cs ON cm.category_id = cs.cat_id
|
||||
SET
|
||||
cm.avg_margin = COALESCE(cs.total_margin * 100.0 / NULLIF(cs.total_sales, 0), 0),
|
||||
cm.turnover_rate = LEAST(COALESCE(cs.units_sold / NULLIF(cs.avg_stock, 0), 0), 999.99),
|
||||
cm.last_calculated_at = NOW()
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.95);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Margin data updated, calculating growth rates',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
if (isCancelled) return processedCount;
|
||||
|
||||
// Finally update growth rates
|
||||
await connection.query(`
|
||||
WITH current_period AS (
|
||||
SELECT
|
||||
pc.cat_id,
|
||||
SUM(o.quantity * o.price) as revenue
|
||||
FROM product_categories pc
|
||||
JOIN products p ON pc.pid = p.pid
|
||||
JOIN orders o ON p.pid = o.pid
|
||||
WHERE o.canceled = false
|
||||
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 3 MONTH)
|
||||
GROUP BY pc.cat_id
|
||||
),
|
||||
previous_period AS (
|
||||
SELECT
|
||||
pc.cat_id,
|
||||
SUM(o.quantity * o.price) as revenue
|
||||
FROM product_categories pc
|
||||
JOIN products p ON pc.pid = p.pid
|
||||
JOIN orders o ON p.pid = o.pid
|
||||
WHERE o.canceled = false
|
||||
AND o.date BETWEEN DATE_SUB(CURRENT_DATE, INTERVAL 15 MONTH)
|
||||
AND DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
|
||||
GROUP BY pc.cat_id
|
||||
)
|
||||
UPDATE category_metrics cm
|
||||
LEFT JOIN current_period cp ON cm.category_id = cp.cat_id
|
||||
LEFT JOIN previous_period pp ON cm.category_id = pp.cat_id
|
||||
SET
|
||||
cm.growth_rate = CASE
|
||||
WHEN pp.revenue = 0 AND COALESCE(cp.revenue, 0) > 0 THEN 100.0
|
||||
WHEN pp.revenue = 0 THEN 0.0
|
||||
ELSE LEAST(
|
||||
GREATEST(
|
||||
((COALESCE(cp.revenue, 0) - pp.revenue) / pp.revenue) * 100.0,
|
||||
-100.0
|
||||
),
|
||||
999.99
|
||||
)
|
||||
END,
|
||||
cm.last_calculated_at = NOW()
|
||||
WHERE cp.cat_id IS NOT NULL OR pp.cat_id IS NOT NULL
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.97);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Growth rates calculated, updating time-based metrics',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
if (isCancelled) return processedCount;
|
||||
|
||||
// Calculate time-based metrics
|
||||
await connection.query(`
|
||||
INSERT INTO category_time_metrics (
|
||||
category_id,
|
||||
year,
|
||||
month,
|
||||
product_count,
|
||||
active_products,
|
||||
total_value,
|
||||
total_revenue,
|
||||
avg_margin,
|
||||
turnover_rate
|
||||
)
|
||||
SELECT
|
||||
pc.cat_id,
|
||||
YEAR(o.date) as year,
|
||||
MONTH(o.date) as month,
|
||||
COUNT(DISTINCT p.pid) as product_count,
|
||||
COUNT(DISTINCT CASE WHEN p.visible = true THEN p.pid END) as active_products,
|
||||
SUM(p.stock_quantity * p.cost_price) as total_value,
|
||||
SUM(o.quantity * o.price) as total_revenue,
|
||||
COALESCE(
|
||||
SUM(o.quantity * (o.price - p.cost_price)) * 100.0 /
|
||||
NULLIF(SUM(o.quantity * o.price), 0),
|
||||
0
|
||||
) as avg_margin,
|
||||
COALESCE(
|
||||
SUM(o.quantity) / NULLIF(AVG(GREATEST(p.stock_quantity, 0)), 0),
|
||||
0
|
||||
) as turnover_rate
|
||||
FROM product_categories pc
|
||||
JOIN products p ON pc.pid = p.pid
|
||||
JOIN orders o ON p.pid = o.pid
|
||||
WHERE o.canceled = false
|
||||
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
|
||||
GROUP BY pc.cat_id, YEAR(o.date), MONTH(o.date)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
product_count = VALUES(product_count),
|
||||
active_products = VALUES(active_products),
|
||||
total_value = VALUES(total_value),
|
||||
total_revenue = VALUES(total_revenue),
|
||||
avg_margin = VALUES(avg_margin),
|
||||
turnover_rate = VALUES(turnover_rate)
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.99);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Time-based metrics calculated',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
return processedCount;
|
||||
} catch (error) {
|
||||
logError(error, 'Error calculating category metrics');
|
||||
throw error;
|
||||
} finally {
|
||||
if (connection) {
|
||||
connection.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = calculateCategoryMetrics;
|
||||
@@ -1,132 +0,0 @@
|
||||
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress');
|
||||
const { getConnection } = require('./utils/db');
|
||||
|
||||
async function calculateFinancialMetrics(startTime, totalProducts, processedCount, isCancelled = false) {
|
||||
const connection = await getConnection();
|
||||
try {
|
||||
if (isCancelled) {
|
||||
outputProgress({
|
||||
status: 'cancelled',
|
||||
operation: 'Financial metrics calculation cancelled',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: null,
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
return processedCount;
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Starting financial metrics calculation',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
// Calculate financial metrics with optimized query
|
||||
await connection.query(`
|
||||
WITH product_financials AS (
|
||||
SELECT
|
||||
p.pid,
|
||||
p.cost_price * p.stock_quantity as inventory_value,
|
||||
SUM(o.quantity * o.price) as total_revenue,
|
||||
SUM(o.quantity * p.cost_price) as cost_of_goods_sold,
|
||||
SUM(o.quantity * (o.price - p.cost_price)) as gross_profit,
|
||||
MIN(o.date) as first_sale_date,
|
||||
MAX(o.date) as last_sale_date,
|
||||
DATEDIFF(MAX(o.date), MIN(o.date)) + 1 as calculation_period_days,
|
||||
COUNT(DISTINCT DATE(o.date)) as active_days
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
WHERE o.canceled = false
|
||||
AND DATE(o.date) >= DATE_SUB(CURDATE(), INTERVAL 12 MONTH)
|
||||
GROUP BY p.pid
|
||||
)
|
||||
UPDATE product_metrics pm
|
||||
JOIN product_financials pf ON pm.pid = pf.pid
|
||||
SET
|
||||
pm.inventory_value = COALESCE(pf.inventory_value, 0),
|
||||
pm.total_revenue = COALESCE(pf.total_revenue, 0),
|
||||
pm.cost_of_goods_sold = COALESCE(pf.cost_of_goods_sold, 0),
|
||||
pm.gross_profit = COALESCE(pf.gross_profit, 0),
|
||||
pm.gmroi = CASE
|
||||
WHEN COALESCE(pf.inventory_value, 0) > 0 AND pf.active_days > 0 THEN
|
||||
(COALESCE(pf.gross_profit, 0) * (365.0 / pf.active_days)) / COALESCE(pf.inventory_value, 0)
|
||||
ELSE 0
|
||||
END
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.65);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Base financial metrics calculated, updating time aggregates',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
if (isCancelled) return processedCount;
|
||||
|
||||
// Update time-based aggregates with optimized query
|
||||
await connection.query(`
|
||||
WITH monthly_financials AS (
|
||||
SELECT
|
||||
p.pid,
|
||||
YEAR(o.date) as year,
|
||||
MONTH(o.date) as month,
|
||||
p.cost_price * p.stock_quantity as inventory_value,
|
||||
SUM(o.quantity * (o.price - p.cost_price)) as gross_profit,
|
||||
COUNT(DISTINCT DATE(o.date)) as active_days,
|
||||
MIN(o.date) as period_start,
|
||||
MAX(o.date) as period_end
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
WHERE o.canceled = false
|
||||
GROUP BY p.pid, YEAR(o.date), MONTH(o.date)
|
||||
)
|
||||
UPDATE product_time_aggregates pta
|
||||
JOIN monthly_financials mf ON pta.pid = mf.pid
|
||||
AND pta.year = mf.year
|
||||
AND pta.month = mf.month
|
||||
SET
|
||||
pta.inventory_value = COALESCE(mf.inventory_value, 0),
|
||||
pta.gmroi = CASE
|
||||
WHEN COALESCE(mf.inventory_value, 0) > 0 AND mf.active_days > 0 THEN
|
||||
(COALESCE(mf.gross_profit, 0) * (365.0 / mf.active_days)) / COALESCE(mf.inventory_value, 0)
|
||||
ELSE 0
|
||||
END
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.70);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Time-based aggregates updated',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
return processedCount;
|
||||
} catch (error) {
|
||||
logError(error, 'Error calculating financial metrics');
|
||||
throw error;
|
||||
} finally {
|
||||
if (connection) {
|
||||
connection.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = calculateFinancialMetrics;
|
||||
@@ -1,281 +0,0 @@
|
||||
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress');
|
||||
const { getConnection } = require('./utils/db');
|
||||
|
||||
// Helper function to handle NaN and undefined values
|
||||
function sanitizeValue(value) {
|
||||
if (value === undefined || value === null || Number.isNaN(value)) {
|
||||
return null;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
async function calculateProductMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) {
|
||||
const connection = await getConnection();
|
||||
try {
|
||||
// Skip flags are inherited from the parent scope
|
||||
const SKIP_PRODUCT_BASE_METRICS = 0;
|
||||
const SKIP_PRODUCT_TIME_AGGREGATES = 0;
|
||||
|
||||
if (isCancelled) {
|
||||
outputProgress({
|
||||
status: 'cancelled',
|
||||
operation: 'Product metrics calculation cancelled',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: null,
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
return processedCount;
|
||||
}
|
||||
|
||||
// Calculate base product metrics
|
||||
if (!SKIP_PRODUCT_BASE_METRICS) {
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Starting base product metrics calculation',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
// Calculate base metrics
|
||||
await connection.query(`
|
||||
UPDATE product_metrics pm
|
||||
JOIN (
|
||||
SELECT
|
||||
p.pid,
|
||||
p.cost_price * p.stock_quantity as inventory_value,
|
||||
SUM(o.quantity) as total_quantity,
|
||||
COUNT(DISTINCT o.order_number) as number_of_orders,
|
||||
SUM(o.quantity * o.price) as total_revenue,
|
||||
SUM(o.quantity * p.cost_price) as cost_of_goods_sold,
|
||||
AVG(o.price) as avg_price,
|
||||
STDDEV(o.price) as price_std,
|
||||
MIN(o.date) as first_sale_date,
|
||||
MAX(o.date) as last_sale_date,
|
||||
COUNT(DISTINCT DATE(o.date)) as active_days
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid AND o.canceled = false
|
||||
GROUP BY p.pid
|
||||
) stats ON pm.pid = stats.pid
|
||||
SET
|
||||
pm.inventory_value = COALESCE(stats.inventory_value, 0),
|
||||
pm.avg_quantity_per_order = COALESCE(stats.total_quantity / NULLIF(stats.number_of_orders, 0), 0),
|
||||
pm.number_of_orders = COALESCE(stats.number_of_orders, 0),
|
||||
pm.total_revenue = COALESCE(stats.total_revenue, 0),
|
||||
pm.cost_of_goods_sold = COALESCE(stats.cost_of_goods_sold, 0),
|
||||
pm.gross_profit = COALESCE(stats.total_revenue - stats.cost_of_goods_sold, 0),
|
||||
pm.avg_margin_percent = CASE
|
||||
WHEN COALESCE(stats.total_revenue, 0) > 0
|
||||
THEN ((stats.total_revenue - stats.cost_of_goods_sold) / stats.total_revenue) * 100
|
||||
ELSE 0
|
||||
END,
|
||||
pm.first_sale_date = stats.first_sale_date,
|
||||
pm.last_sale_date = stats.last_sale_date,
|
||||
pm.gmroi = CASE
|
||||
WHEN COALESCE(stats.inventory_value, 0) > 0
|
||||
THEN (stats.total_revenue - stats.cost_of_goods_sold) / stats.inventory_value
|
||||
ELSE 0
|
||||
END,
|
||||
pm.last_calculated_at = NOW()
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.4);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Base product metrics calculated',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
} else {
|
||||
processedCount = Math.floor(totalProducts * 0.4);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Skipping base product metrics calculation',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
}
|
||||
|
||||
if (isCancelled) return processedCount;
|
||||
|
||||
// Calculate product time aggregates
|
||||
if (!SKIP_PRODUCT_TIME_AGGREGATES) {
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Starting product time aggregates calculation',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
// Calculate time-based aggregates
|
||||
await connection.query(`
|
||||
INSERT INTO product_time_aggregates (
|
||||
pid,
|
||||
year,
|
||||
month,
|
||||
total_quantity_sold,
|
||||
total_revenue,
|
||||
total_cost,
|
||||
order_count,
|
||||
avg_price,
|
||||
profit_margin,
|
||||
inventory_value,
|
||||
gmroi
|
||||
)
|
||||
SELECT
|
||||
p.pid,
|
||||
YEAR(o.date) as year,
|
||||
MONTH(o.date) as month,
|
||||
SUM(o.quantity) as total_quantity_sold,
|
||||
SUM(o.quantity * o.price) as total_revenue,
|
||||
SUM(o.quantity * p.cost_price) as total_cost,
|
||||
COUNT(DISTINCT o.order_number) as order_count,
|
||||
AVG(o.price) as avg_price,
|
||||
CASE
|
||||
WHEN SUM(o.quantity * o.price) > 0
|
||||
THEN ((SUM(o.quantity * o.price) - SUM(o.quantity * p.cost_price)) / SUM(o.quantity * o.price)) * 100
|
||||
ELSE 0
|
||||
END as profit_margin,
|
||||
p.cost_price * p.stock_quantity as inventory_value,
|
||||
CASE
|
||||
WHEN p.cost_price * p.stock_quantity > 0
|
||||
THEN (SUM(o.quantity * (o.price - p.cost_price))) / (p.cost_price * p.stock_quantity)
|
||||
ELSE 0
|
||||
END as gmroi
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid AND o.canceled = false
|
||||
WHERE o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
|
||||
GROUP BY p.pid, YEAR(o.date), MONTH(o.date)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
total_quantity_sold = VALUES(total_quantity_sold),
|
||||
total_revenue = VALUES(total_revenue),
|
||||
total_cost = VALUES(total_cost),
|
||||
order_count = VALUES(order_count),
|
||||
avg_price = VALUES(avg_price),
|
||||
profit_margin = VALUES(profit_margin),
|
||||
inventory_value = VALUES(inventory_value),
|
||||
gmroi = VALUES(gmroi)
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.6);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Product time aggregates calculated',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
} else {
|
||||
processedCount = Math.floor(totalProducts * 0.6);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Skipping product time aggregates calculation',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
}
|
||||
|
||||
return processedCount;
|
||||
} catch (error) {
|
||||
logError(error, 'Error calculating product metrics');
|
||||
throw error;
|
||||
} finally {
|
||||
if (connection) {
|
||||
connection.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function calculateStockStatus(stock, config, daily_sales_avg, weekly_sales_avg, monthly_sales_avg) {
|
||||
if (stock <= 0) {
|
||||
return 'Out of Stock';
|
||||
}
|
||||
|
||||
// Use the most appropriate sales average based on data quality
|
||||
let sales_avg = daily_sales_avg;
|
||||
if (sales_avg === 0) {
|
||||
sales_avg = weekly_sales_avg / 7;
|
||||
}
|
||||
if (sales_avg === 0) {
|
||||
sales_avg = monthly_sales_avg / 30;
|
||||
}
|
||||
|
||||
if (sales_avg === 0) {
|
||||
return stock <= config.low_stock_threshold ? 'Low Stock' : 'In Stock';
|
||||
}
|
||||
|
||||
const days_of_stock = stock / sales_avg;
|
||||
|
||||
if (days_of_stock <= config.critical_days) {
|
||||
return 'Critical';
|
||||
} else if (days_of_stock <= config.reorder_days) {
|
||||
return 'Reorder';
|
||||
} else if (days_of_stock > config.overstock_days) {
|
||||
return 'Overstocked';
|
||||
}
|
||||
|
||||
return 'Healthy';
|
||||
}
|
||||
|
||||
function calculateReorderQuantities(stock, stock_status, daily_sales_avg, avg_lead_time, config) {
|
||||
// Calculate safety stock based on service level and lead time
|
||||
const z_score = 1.96; // 95% service level
|
||||
const lead_time = avg_lead_time || config.target_days;
|
||||
const safety_stock = Math.ceil(daily_sales_avg * Math.sqrt(lead_time) * z_score);
|
||||
|
||||
// Calculate reorder point
|
||||
const lead_time_demand = daily_sales_avg * lead_time;
|
||||
const reorder_point = Math.ceil(lead_time_demand + safety_stock);
|
||||
|
||||
// Calculate reorder quantity using EOQ formula if we have the necessary data
|
||||
let reorder_qty = 0;
|
||||
if (daily_sales_avg > 0) {
|
||||
const annual_demand = daily_sales_avg * 365;
|
||||
const order_cost = 25; // Fixed cost per order
|
||||
const holding_cost_percent = 0.25; // 25% annual holding cost
|
||||
|
||||
reorder_qty = Math.ceil(Math.sqrt((2 * annual_demand * order_cost) / holding_cost_percent));
|
||||
} else {
|
||||
// If no sales data, use a basic calculation
|
||||
reorder_qty = Math.max(safety_stock, config.low_stock_threshold);
|
||||
}
|
||||
|
||||
// Calculate overstocked amount
|
||||
const overstocked_amt = stock_status === 'Overstocked' ?
|
||||
stock - Math.ceil(daily_sales_avg * config.overstock_days) :
|
||||
0;
|
||||
|
||||
return {
|
||||
safety_stock,
|
||||
reorder_point,
|
||||
reorder_qty,
|
||||
overstocked_amt
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = calculateProductMetrics;
|
||||
@@ -1,309 +0,0 @@
|
||||
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress');
|
||||
const { getConnection } = require('./utils/db');
|
||||
|
||||
async function calculateSalesForecasts(startTime, totalProducts, processedCount, isCancelled = false) {
|
||||
const connection = await getConnection();
|
||||
try {
|
||||
if (isCancelled) {
|
||||
outputProgress({
|
||||
status: 'cancelled',
|
||||
operation: 'Sales forecasts calculation cancelled',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: null,
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
return processedCount;
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Starting sales forecasts calculation',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
// First, create a temporary table for forecast dates
|
||||
await connection.query(`
|
||||
CREATE TEMPORARY TABLE IF NOT EXISTS temp_forecast_dates (
|
||||
forecast_date DATE,
|
||||
day_of_week INT,
|
||||
month INT,
|
||||
PRIMARY KEY (forecast_date)
|
||||
)
|
||||
`);
|
||||
|
||||
await connection.query(`
|
||||
INSERT INTO temp_forecast_dates
|
||||
SELECT
|
||||
DATE_ADD(CURRENT_DATE, INTERVAL n DAY) as forecast_date,
|
||||
DAYOFWEEK(DATE_ADD(CURRENT_DATE, INTERVAL n DAY)) as day_of_week,
|
||||
MONTH(DATE_ADD(CURRENT_DATE, INTERVAL n DAY)) as month
|
||||
FROM (
|
||||
SELECT a.N + b.N * 10 as n
|
||||
FROM
|
||||
(SELECT 0 as N UNION SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 4 UNION
|
||||
SELECT 5 UNION SELECT 6 UNION SELECT 7 UNION SELECT 8 UNION SELECT 9) a,
|
||||
(SELECT 0 as N UNION SELECT 1 UNION SELECT 2) b
|
||||
ORDER BY n
|
||||
LIMIT 31
|
||||
) numbers
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.92);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Forecast dates prepared, calculating daily sales stats',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
if (isCancelled) return processedCount;
|
||||
|
||||
// Create temporary table for daily sales stats
|
||||
await connection.query(`
|
||||
CREATE TEMPORARY TABLE IF NOT EXISTS temp_daily_sales AS
|
||||
SELECT
|
||||
o.pid,
|
||||
DAYOFWEEK(o.date) as day_of_week,
|
||||
SUM(o.quantity) as daily_quantity,
|
||||
SUM(o.price * o.quantity) as daily_revenue,
|
||||
COUNT(DISTINCT DATE(o.date)) as day_count
|
||||
FROM orders o
|
||||
WHERE o.canceled = false
|
||||
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 90 DAY)
|
||||
GROUP BY o.pid, DAYOFWEEK(o.date)
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.94);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Daily sales stats calculated, preparing product stats',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
if (isCancelled) return processedCount;
|
||||
|
||||
// Create temporary table for product stats
|
||||
await connection.query(`
|
||||
CREATE TEMPORARY TABLE IF NOT EXISTS temp_product_stats AS
|
||||
SELECT
|
||||
pid,
|
||||
AVG(daily_revenue) as overall_avg_revenue,
|
||||
SUM(day_count) as total_days
|
||||
FROM temp_daily_sales
|
||||
GROUP BY pid
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.96);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Product stats prepared, calculating product-level forecasts',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
if (isCancelled) return processedCount;
|
||||
|
||||
// Calculate product-level forecasts
|
||||
await connection.query(`
|
||||
INSERT INTO sales_forecasts (
|
||||
pid,
|
||||
forecast_date,
|
||||
forecast_units,
|
||||
forecast_revenue,
|
||||
confidence_level,
|
||||
last_calculated_at
|
||||
)
|
||||
SELECT
|
||||
ds.pid,
|
||||
fd.forecast_date,
|
||||
GREATEST(0,
|
||||
AVG(ds.daily_quantity) *
|
||||
(1 + COALESCE(sf.seasonality_factor, 0))
|
||||
) as forecast_units,
|
||||
GREATEST(0,
|
||||
COALESCE(
|
||||
CASE
|
||||
WHEN SUM(ds.day_count) >= 4 THEN AVG(ds.daily_revenue)
|
||||
ELSE ps.overall_avg_revenue
|
||||
END *
|
||||
(1 + COALESCE(sf.seasonality_factor, 0)) *
|
||||
(0.95 + (RAND() * 0.1)),
|
||||
0
|
||||
)
|
||||
) as forecast_revenue,
|
||||
CASE
|
||||
WHEN ps.total_days >= 60 THEN 90
|
||||
WHEN ps.total_days >= 30 THEN 80
|
||||
WHEN ps.total_days >= 14 THEN 70
|
||||
ELSE 60
|
||||
END as confidence_level,
|
||||
NOW() as last_calculated_at
|
||||
FROM temp_daily_sales ds
|
||||
JOIN temp_product_stats ps ON ds.pid = ps.pid
|
||||
CROSS JOIN temp_forecast_dates fd
|
||||
LEFT JOIN sales_seasonality sf ON fd.month = sf.month
|
||||
GROUP BY ds.pid, fd.forecast_date, ps.overall_avg_revenue, ps.total_days, sf.seasonality_factor
|
||||
HAVING AVG(ds.daily_quantity) > 0
|
||||
ON DUPLICATE KEY UPDATE
|
||||
forecast_units = VALUES(forecast_units),
|
||||
forecast_revenue = VALUES(forecast_revenue),
|
||||
confidence_level = VALUES(confidence_level),
|
||||
last_calculated_at = NOW()
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.98);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Product forecasts calculated, preparing category stats',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
if (isCancelled) return processedCount;
|
||||
|
||||
// Create temporary table for category stats
|
||||
await connection.query(`
|
||||
CREATE TEMPORARY TABLE IF NOT EXISTS temp_category_sales AS
|
||||
SELECT
|
||||
pc.cat_id,
|
||||
DAYOFWEEK(o.date) as day_of_week,
|
||||
SUM(o.quantity) as daily_quantity,
|
||||
SUM(o.price * o.quantity) as daily_revenue,
|
||||
COUNT(DISTINCT DATE(o.date)) as day_count
|
||||
FROM orders o
|
||||
JOIN product_categories pc ON o.pid = pc.pid
|
||||
WHERE o.canceled = false
|
||||
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 90 DAY)
|
||||
GROUP BY pc.cat_id, DAYOFWEEK(o.date)
|
||||
`);
|
||||
|
||||
await connection.query(`
|
||||
CREATE TEMPORARY TABLE IF NOT EXISTS temp_category_stats AS
|
||||
SELECT
|
||||
cat_id,
|
||||
AVG(daily_revenue) as overall_avg_revenue,
|
||||
SUM(day_count) as total_days
|
||||
FROM temp_category_sales
|
||||
GROUP BY cat_id
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.99);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Category stats prepared, calculating category-level forecasts',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
if (isCancelled) return processedCount;
|
||||
|
||||
// Calculate category-level forecasts
|
||||
await connection.query(`
|
||||
INSERT INTO category_forecasts (
|
||||
category_id,
|
||||
forecast_date,
|
||||
forecast_units,
|
||||
forecast_revenue,
|
||||
confidence_level,
|
||||
last_calculated_at
|
||||
)
|
||||
SELECT
|
||||
cs.cat_id as category_id,
|
||||
fd.forecast_date,
|
||||
GREATEST(0,
|
||||
AVG(cs.daily_quantity) *
|
||||
(1 + COALESCE(sf.seasonality_factor, 0))
|
||||
) as forecast_units,
|
||||
GREATEST(0,
|
||||
COALESCE(
|
||||
CASE
|
||||
WHEN SUM(cs.day_count) >= 4 THEN AVG(cs.daily_revenue)
|
||||
ELSE ct.overall_avg_revenue
|
||||
END *
|
||||
(1 + COALESCE(sf.seasonality_factor, 0)) *
|
||||
(0.95 + (RAND() * 0.1)),
|
||||
0
|
||||
)
|
||||
) as forecast_revenue,
|
||||
CASE
|
||||
WHEN ct.total_days >= 60 THEN 90
|
||||
WHEN ct.total_days >= 30 THEN 80
|
||||
WHEN ct.total_days >= 14 THEN 70
|
||||
ELSE 60
|
||||
END as confidence_level,
|
||||
NOW() as last_calculated_at
|
||||
FROM temp_category_sales cs
|
||||
JOIN temp_category_stats ct ON cs.cat_id = ct.cat_id
|
||||
CROSS JOIN temp_forecast_dates fd
|
||||
LEFT JOIN sales_seasonality sf ON fd.month = sf.month
|
||||
GROUP BY cs.cat_id, fd.forecast_date, ct.overall_avg_revenue, ct.total_days, sf.seasonality_factor
|
||||
HAVING AVG(cs.daily_quantity) > 0
|
||||
ON DUPLICATE KEY UPDATE
|
||||
forecast_units = VALUES(forecast_units),
|
||||
forecast_revenue = VALUES(forecast_revenue),
|
||||
confidence_level = VALUES(confidence_level),
|
||||
last_calculated_at = NOW()
|
||||
`);
|
||||
|
||||
// Clean up temporary tables
|
||||
await connection.query(`
|
||||
DROP TEMPORARY TABLE IF EXISTS temp_forecast_dates;
|
||||
DROP TEMPORARY TABLE IF EXISTS temp_daily_sales;
|
||||
DROP TEMPORARY TABLE IF EXISTS temp_product_stats;
|
||||
DROP TEMPORARY TABLE IF EXISTS temp_category_sales;
|
||||
DROP TEMPORARY TABLE IF EXISTS temp_category_stats;
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 1.0);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Category forecasts calculated and temporary tables cleaned up',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
return processedCount;
|
||||
} catch (error) {
|
||||
logError(error, 'Error calculating sales forecasts');
|
||||
throw error;
|
||||
} finally {
|
||||
if (connection) {
|
||||
connection.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = calculateSalesForecasts;
|
||||
@@ -1,190 +0,0 @@
|
||||
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress');
|
||||
const { getConnection } = require('./utils/db');
|
||||
|
||||
async function calculateTimeAggregates(startTime, totalProducts, processedCount, isCancelled = false) {
|
||||
const connection = await getConnection();
|
||||
try {
|
||||
if (isCancelled) {
|
||||
outputProgress({
|
||||
status: 'cancelled',
|
||||
operation: 'Time aggregates calculation cancelled',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: null,
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
return processedCount;
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Starting time aggregates calculation',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
// Initial insert of time-based aggregates
|
||||
await connection.query(`
|
||||
INSERT INTO product_time_aggregates (
|
||||
pid,
|
||||
year,
|
||||
month,
|
||||
total_quantity_sold,
|
||||
total_revenue,
|
||||
total_cost,
|
||||
order_count,
|
||||
stock_received,
|
||||
stock_ordered,
|
||||
avg_price,
|
||||
profit_margin
|
||||
)
|
||||
WITH sales_data AS (
|
||||
SELECT
|
||||
o.pid,
|
||||
YEAR(o.date) as year,
|
||||
MONTH(o.date) as month,
|
||||
SUM(o.quantity) as total_quantity_sold,
|
||||
SUM((o.price - COALESCE(o.discount, 0)) * o.quantity) as total_revenue,
|
||||
SUM(COALESCE(p.cost_price, 0) * o.quantity) as total_cost,
|
||||
COUNT(DISTINCT o.order_number) as order_count,
|
||||
AVG(o.price - COALESCE(o.discount, 0)) as avg_price,
|
||||
CASE
|
||||
WHEN SUM((o.price - COALESCE(o.discount, 0)) * o.quantity) = 0 THEN 0
|
||||
ELSE ((SUM((o.price - COALESCE(o.discount, 0)) * o.quantity) -
|
||||
SUM(COALESCE(p.cost_price, 0) * o.quantity)) /
|
||||
SUM((o.price - COALESCE(o.discount, 0)) * o.quantity)) * 100
|
||||
END as profit_margin
|
||||
FROM orders o
|
||||
JOIN products p ON o.pid = p.pid
|
||||
WHERE o.canceled = 0
|
||||
GROUP BY o.pid, YEAR(o.date), MONTH(o.date)
|
||||
),
|
||||
purchase_data AS (
|
||||
SELECT
|
||||
pid,
|
||||
YEAR(date) as year,
|
||||
MONTH(date) as month,
|
||||
SUM(received) as stock_received,
|
||||
SUM(ordered) as stock_ordered
|
||||
FROM purchase_orders
|
||||
WHERE status = 50
|
||||
GROUP BY pid, YEAR(date), MONTH(date)
|
||||
)
|
||||
SELECT
|
||||
s.pid,
|
||||
s.year,
|
||||
s.month,
|
||||
s.total_quantity_sold,
|
||||
s.total_revenue,
|
||||
s.total_cost,
|
||||
s.order_count,
|
||||
COALESCE(p.stock_received, 0) as stock_received,
|
||||
COALESCE(p.stock_ordered, 0) as stock_ordered,
|
||||
s.avg_price,
|
||||
s.profit_margin
|
||||
FROM sales_data s
|
||||
LEFT JOIN purchase_data p
|
||||
ON s.pid = p.pid
|
||||
AND s.year = p.year
|
||||
AND s.month = p.month
|
||||
UNION
|
||||
SELECT
|
||||
p.pid,
|
||||
p.year,
|
||||
p.month,
|
||||
0 as total_quantity_sold,
|
||||
0 as total_revenue,
|
||||
0 as total_cost,
|
||||
0 as order_count,
|
||||
p.stock_received,
|
||||
p.stock_ordered,
|
||||
0 as avg_price,
|
||||
0 as profit_margin
|
||||
FROM purchase_data p
|
||||
LEFT JOIN sales_data s
|
||||
ON p.pid = s.pid
|
||||
AND p.year = s.year
|
||||
AND p.month = s.month
|
||||
WHERE s.pid IS NULL
|
||||
ON DUPLICATE KEY UPDATE
|
||||
total_quantity_sold = VALUES(total_quantity_sold),
|
||||
total_revenue = VALUES(total_revenue),
|
||||
total_cost = VALUES(total_cost),
|
||||
order_count = VALUES(order_count),
|
||||
stock_received = VALUES(stock_received),
|
||||
stock_ordered = VALUES(stock_ordered),
|
||||
avg_price = VALUES(avg_price),
|
||||
profit_margin = VALUES(profit_margin)
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.60);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Base time aggregates calculated, updating financial metrics',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
if (isCancelled) return processedCount;
|
||||
|
||||
// Update with financial metrics
|
||||
await connection.query(`
|
||||
UPDATE product_time_aggregates pta
|
||||
JOIN (
|
||||
SELECT
|
||||
p.pid,
|
||||
YEAR(o.date) as year,
|
||||
MONTH(o.date) as month,
|
||||
p.cost_price * p.stock_quantity as inventory_value,
|
||||
SUM(o.quantity * (o.price - p.cost_price)) as gross_profit,
|
||||
COUNT(DISTINCT DATE(o.date)) as days_in_period
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
WHERE o.canceled = false
|
||||
GROUP BY p.pid, YEAR(o.date), MONTH(o.date)
|
||||
) fin ON pta.pid = fin.pid
|
||||
AND pta.year = fin.year
|
||||
AND pta.month = fin.month
|
||||
SET
|
||||
pta.inventory_value = COALESCE(fin.inventory_value, 0),
|
||||
pta.gmroi = CASE
|
||||
WHEN COALESCE(fin.inventory_value, 0) > 0 AND fin.days_in_period > 0 THEN
|
||||
(COALESCE(fin.gross_profit, 0) * (365.0 / fin.days_in_period)) / COALESCE(fin.inventory_value, 0)
|
||||
ELSE 0
|
||||
END
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.65);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Financial metrics updated',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
return processedCount;
|
||||
} catch (error) {
|
||||
logError(error, 'Error calculating time aggregates');
|
||||
throw error;
|
||||
} finally {
|
||||
if (connection) {
|
||||
connection.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = calculateTimeAggregates;
|
||||
@@ -1,51 +0,0 @@
|
||||
const mysql = require('mysql2/promise');
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.resolve(__dirname, '../../..', '.env') });
|
||||
|
||||
// Database configuration
|
||||
const dbConfig = {
|
||||
host: process.env.DB_HOST,
|
||||
user: process.env.DB_USER,
|
||||
password: process.env.DB_PASSWORD,
|
||||
database: process.env.DB_NAME,
|
||||
waitForConnections: true,
|
||||
connectionLimit: 10,
|
||||
queueLimit: 0,
|
||||
// Add performance optimizations
|
||||
namedPlaceholders: true,
|
||||
maxPreparedStatements: 256,
|
||||
enableKeepAlive: true,
|
||||
keepAliveInitialDelay: 0,
|
||||
// Add memory optimizations
|
||||
flags: [
|
||||
'FOUND_ROWS',
|
||||
'LONG_PASSWORD',
|
||||
'PROTOCOL_41',
|
||||
'TRANSACTIONS',
|
||||
'SECURE_CONNECTION',
|
||||
'MULTI_RESULTS',
|
||||
'PS_MULTI_RESULTS',
|
||||
'PLUGIN_AUTH',
|
||||
'CONNECT_ATTRS',
|
||||
'PLUGIN_AUTH_LENENC_CLIENT_DATA',
|
||||
'SESSION_TRACK',
|
||||
'MULTI_STATEMENTS'
|
||||
]
|
||||
};
|
||||
|
||||
// Create a single pool instance to be reused
|
||||
const pool = mysql.createPool(dbConfig);
|
||||
|
||||
async function getConnection() {
|
||||
return await pool.getConnection();
|
||||
}
|
||||
|
||||
async function closePool() {
|
||||
await pool.end();
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
dbConfig,
|
||||
getConnection,
|
||||
closePool
|
||||
};
|
||||
@@ -1,158 +0,0 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// Helper function to format elapsed time
|
||||
function formatElapsedTime(elapsed) {
|
||||
// If elapsed is a timestamp, convert to elapsed milliseconds
|
||||
if (elapsed instanceof Date || elapsed > 1000000000000) {
|
||||
elapsed = Date.now() - elapsed;
|
||||
} else {
|
||||
// If elapsed is in seconds, convert to milliseconds
|
||||
elapsed = elapsed * 1000;
|
||||
}
|
||||
|
||||
const seconds = Math.floor(elapsed / 1000);
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const hours = Math.floor(minutes / 60);
|
||||
|
||||
if (hours > 0) {
|
||||
return `${hours}h ${minutes % 60}m`;
|
||||
} else if (minutes > 0) {
|
||||
return `${minutes}m ${seconds % 60}s`;
|
||||
} else {
|
||||
return `${seconds}s`;
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to estimate remaining time
|
||||
function estimateRemaining(startTime, current, total) {
|
||||
if (current === 0) return null;
|
||||
const elapsed = Date.now() - startTime;
|
||||
const rate = current / elapsed;
|
||||
const remaining = (total - current) / rate;
|
||||
|
||||
const minutes = Math.floor(remaining / 60000);
|
||||
const seconds = Math.floor((remaining % 60000) / 1000);
|
||||
|
||||
if (minutes > 0) {
|
||||
return `${minutes}m ${seconds}s`;
|
||||
} else {
|
||||
return `${seconds}s`;
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to calculate rate
|
||||
function calculateRate(startTime, current) {
|
||||
const elapsed = (Date.now() - startTime) / 1000; // Convert to seconds
|
||||
return elapsed > 0 ? Math.round(current / elapsed) : 0;
|
||||
}
|
||||
|
||||
// Set up logging
|
||||
const LOG_DIR = path.join(__dirname, '../../../logs');
|
||||
const ERROR_LOG = path.join(LOG_DIR, 'import-errors.log');
|
||||
const IMPORT_LOG = path.join(LOG_DIR, 'import.log');
|
||||
const STATUS_FILE = path.join(LOG_DIR, 'metrics-status.json');
|
||||
|
||||
// Ensure log directory exists
|
||||
if (!fs.existsSync(LOG_DIR)) {
|
||||
fs.mkdirSync(LOG_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
// Helper function to log errors
|
||||
function logError(error, context = '') {
|
||||
const timestamp = new Date().toISOString();
|
||||
const errorMessage = `[${timestamp}] ${context}\nError: ${error.message}\nStack: ${error.stack}\n\n`;
|
||||
|
||||
// Log to error file
|
||||
fs.appendFileSync(ERROR_LOG, errorMessage);
|
||||
|
||||
// Also log to console
|
||||
console.error(`\n${context}\nError: ${error.message}`);
|
||||
}
|
||||
|
||||
// Helper function to log import progress
|
||||
function logImport(message) {
|
||||
const timestamp = new Date().toISOString();
|
||||
const logMessage = `[${timestamp}] ${message}\n`;
|
||||
fs.appendFileSync(IMPORT_LOG, logMessage);
|
||||
}
|
||||
|
||||
// Helper function to output progress
|
||||
function outputProgress(data) {
|
||||
// Save progress to file for resumption
|
||||
saveProgress(data);
|
||||
// Format as SSE event
|
||||
const event = {
|
||||
progress: data
|
||||
};
|
||||
// Always send to stdout for frontend
|
||||
process.stdout.write(JSON.stringify(event) + '\n');
|
||||
|
||||
// Log significant events to disk
|
||||
const isSignificant =
|
||||
// Operation starts
|
||||
(data.operation && !data.current) ||
|
||||
// Operation completions and errors
|
||||
data.status === 'complete' ||
|
||||
data.status === 'error' ||
|
||||
// Major phase changes
|
||||
data.operation?.includes('Starting ABC classification') ||
|
||||
data.operation?.includes('Starting time-based aggregates') ||
|
||||
data.operation?.includes('Starting vendor metrics');
|
||||
|
||||
if (isSignificant) {
|
||||
logImport(`${data.operation || 'Operation'}${data.message ? ': ' + data.message : ''}${data.error ? ' Error: ' + data.error : ''}${data.status ? ' Status: ' + data.status : ''}`);
|
||||
}
|
||||
}
|
||||
|
||||
function saveProgress(progress) {
|
||||
try {
|
||||
fs.writeFileSync(STATUS_FILE, JSON.stringify({
|
||||
...progress,
|
||||
timestamp: Date.now()
|
||||
}));
|
||||
} catch (err) {
|
||||
console.error('Failed to save progress:', err);
|
||||
}
|
||||
}
|
||||
|
||||
function clearProgress() {
|
||||
try {
|
||||
if (fs.existsSync(STATUS_FILE)) {
|
||||
fs.unlinkSync(STATUS_FILE);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to clear progress:', err);
|
||||
}
|
||||
}
|
||||
|
||||
function getProgress() {
|
||||
try {
|
||||
if (fs.existsSync(STATUS_FILE)) {
|
||||
const progress = JSON.parse(fs.readFileSync(STATUS_FILE, 'utf8'));
|
||||
// Check if the progress is still valid (less than 1 hour old)
|
||||
if (progress.timestamp && Date.now() - progress.timestamp < 3600000) {
|
||||
return progress;
|
||||
} else {
|
||||
// Clear old progress
|
||||
clearProgress();
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to read progress:', err);
|
||||
clearProgress();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
formatElapsedTime,
|
||||
estimateRemaining,
|
||||
calculateRate,
|
||||
logError,
|
||||
logImport,
|
||||
outputProgress,
|
||||
saveProgress,
|
||||
clearProgress,
|
||||
getProgress
|
||||
};
|
||||
@@ -1,173 +0,0 @@
|
||||
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate, logError } = require('./utils/progress');
|
||||
const { getConnection } = require('./utils/db');
|
||||
|
||||
async function calculateVendorMetrics(startTime, totalProducts, processedCount, isCancelled = false) {
|
||||
const connection = await getConnection();
|
||||
try {
|
||||
if (isCancelled) {
|
||||
outputProgress({
|
||||
status: 'cancelled',
|
||||
operation: 'Vendor metrics calculation cancelled',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: null,
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
return processedCount;
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Starting vendor metrics calculation',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
// First ensure all vendors exist in vendor_details
|
||||
await connection.query(`
|
||||
INSERT IGNORE INTO vendor_details (vendor, status, created_at, updated_at)
|
||||
SELECT DISTINCT
|
||||
vendor,
|
||||
'active' as status,
|
||||
NOW() as created_at,
|
||||
NOW() as updated_at
|
||||
FROM products
|
||||
WHERE vendor IS NOT NULL
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.8);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Vendor details updated, calculating metrics',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
if (isCancelled) return processedCount;
|
||||
|
||||
// Now calculate vendor metrics
|
||||
await connection.query(`
|
||||
INSERT INTO vendor_metrics (
|
||||
vendor,
|
||||
total_revenue,
|
||||
total_orders,
|
||||
total_late_orders,
|
||||
avg_lead_time_days,
|
||||
on_time_delivery_rate,
|
||||
order_fill_rate,
|
||||
avg_order_value,
|
||||
active_products,
|
||||
total_products,
|
||||
status,
|
||||
last_calculated_at
|
||||
)
|
||||
WITH vendor_sales AS (
|
||||
SELECT
|
||||
p.vendor,
|
||||
SUM(o.quantity * o.price) as total_revenue,
|
||||
COUNT(DISTINCT o.id) as total_orders,
|
||||
COUNT(DISTINCT p.pid) as active_products
|
||||
FROM products p
|
||||
JOIN orders o ON p.pid = o.pid
|
||||
WHERE o.canceled = false
|
||||
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
|
||||
GROUP BY p.vendor
|
||||
),
|
||||
vendor_po AS (
|
||||
SELECT
|
||||
p.vendor,
|
||||
COUNT(DISTINCT CASE WHEN po.receiving_status = 40 THEN po.id END) as received_orders,
|
||||
COUNT(DISTINCT po.id) as total_orders,
|
||||
AVG(CASE
|
||||
WHEN po.receiving_status = 40
|
||||
THEN DATEDIFF(po.received_date, po.date)
|
||||
END) as avg_lead_time_days
|
||||
FROM products p
|
||||
JOIN purchase_orders po ON p.pid = po.pid
|
||||
WHERE po.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
|
||||
GROUP BY p.vendor
|
||||
),
|
||||
vendor_products AS (
|
||||
SELECT
|
||||
vendor,
|
||||
COUNT(DISTINCT pid) as total_products
|
||||
FROM products
|
||||
GROUP BY vendor
|
||||
)
|
||||
SELECT
|
||||
vs.vendor,
|
||||
COALESCE(vs.total_revenue, 0) as total_revenue,
|
||||
COALESCE(vp.total_orders, 0) as total_orders,
|
||||
COALESCE(vp.total_orders - vp.received_orders, 0) as total_late_orders,
|
||||
COALESCE(vp.avg_lead_time_days, 0) as avg_lead_time_days,
|
||||
CASE
|
||||
WHEN vp.total_orders > 0
|
||||
THEN (vp.received_orders / vp.total_orders) * 100
|
||||
ELSE 0
|
||||
END as on_time_delivery_rate,
|
||||
CASE
|
||||
WHEN vp.total_orders > 0
|
||||
THEN (vp.received_orders / vp.total_orders) * 100
|
||||
ELSE 0
|
||||
END as order_fill_rate,
|
||||
CASE
|
||||
WHEN vs.total_orders > 0
|
||||
THEN vs.total_revenue / vs.total_orders
|
||||
ELSE 0
|
||||
END as avg_order_value,
|
||||
COALESCE(vs.active_products, 0) as active_products,
|
||||
COALESCE(vpr.total_products, 0) as total_products,
|
||||
'active' as status,
|
||||
NOW() as last_calculated_at
|
||||
FROM vendor_sales vs
|
||||
LEFT JOIN vendor_po vp ON vs.vendor = vp.vendor
|
||||
LEFT JOIN vendor_products vpr ON vs.vendor = vpr.vendor
|
||||
WHERE vs.vendor IS NOT NULL
|
||||
ON DUPLICATE KEY UPDATE
|
||||
total_revenue = VALUES(total_revenue),
|
||||
total_orders = VALUES(total_orders),
|
||||
total_late_orders = VALUES(total_late_orders),
|
||||
avg_lead_time_days = VALUES(avg_lead_time_days),
|
||||
on_time_delivery_rate = VALUES(on_time_delivery_rate),
|
||||
order_fill_rate = VALUES(order_fill_rate),
|
||||
avg_order_value = VALUES(avg_order_value),
|
||||
active_products = VALUES(active_products),
|
||||
total_products = VALUES(total_products),
|
||||
status = VALUES(status),
|
||||
last_calculated_at = VALUES(last_calculated_at)
|
||||
`);
|
||||
|
||||
processedCount = Math.floor(totalProducts * 0.9);
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Vendor metrics calculated',
|
||||
current: processedCount,
|
||||
total: totalProducts,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, processedCount, totalProducts),
|
||||
rate: calculateRate(startTime, processedCount),
|
||||
percentage: ((processedCount / totalProducts) * 100).toFixed(1)
|
||||
});
|
||||
|
||||
return processedCount;
|
||||
} catch (error) {
|
||||
logError(error, 'Error calculating vendor metrics');
|
||||
throw error;
|
||||
} finally {
|
||||
if (connection) {
|
||||
connection.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = calculateVendorMetrics;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,180 +0,0 @@
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
const axios = require('axios');
|
||||
const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } = require('../metrics/utils/progress');
|
||||
|
||||
// Change working directory to script directory
|
||||
process.chdir(path.dirname(__filename));
|
||||
|
||||
require('dotenv').config({ path: path.resolve(__dirname, '..', '.env') });
|
||||
|
||||
const FILES = [
|
||||
{
|
||||
name: '39f2x83-products.csv',
|
||||
url: process.env.PRODUCTS_CSV_URL
|
||||
},
|
||||
{
|
||||
name: '39f2x83-orders.csv',
|
||||
url: process.env.ORDERS_CSV_URL
|
||||
},
|
||||
{
|
||||
name: '39f2x83-purchase_orders.csv',
|
||||
url: process.env.PURCHASE_ORDERS_CSV_URL
|
||||
}
|
||||
];
|
||||
|
||||
let isCancelled = false;
|
||||
|
||||
function cancelUpdate() {
|
||||
isCancelled = true;
|
||||
outputProgress({
|
||||
status: 'cancelled',
|
||||
operation: 'CSV update cancelled',
|
||||
current: 0,
|
||||
total: FILES.length,
|
||||
elapsed: null,
|
||||
remaining: null,
|
||||
rate: 0
|
||||
});
|
||||
}
|
||||
|
||||
async function downloadFile(file, index, startTime) {
|
||||
if (isCancelled) return;
|
||||
|
||||
const csvDir = path.join(__dirname, '../csv');
|
||||
if (!fs.existsSync(csvDir)) {
|
||||
fs.mkdirSync(csvDir, { recursive: true });
|
||||
}
|
||||
|
||||
const writer = fs.createWriteStream(path.join(csvDir, file.name));
|
||||
|
||||
try {
|
||||
const response = await axios({
|
||||
url: file.url,
|
||||
method: 'GET',
|
||||
responseType: 'stream'
|
||||
});
|
||||
|
||||
const totalLength = response.headers['content-length'];
|
||||
let downloadedLength = 0;
|
||||
let lastProgressUpdate = Date.now();
|
||||
const PROGRESS_INTERVAL = 1000; // Update progress every second
|
||||
|
||||
response.data.on('data', (chunk) => {
|
||||
if (isCancelled) {
|
||||
writer.end();
|
||||
return;
|
||||
}
|
||||
|
||||
downloadedLength += chunk.length;
|
||||
|
||||
// Update progress based on time interval
|
||||
const now = Date.now();
|
||||
if (now - lastProgressUpdate >= PROGRESS_INTERVAL) {
|
||||
const progress = (downloadedLength / totalLength) * 100;
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: `Downloading ${file.name}`,
|
||||
current: index + (downloadedLength / totalLength),
|
||||
total: FILES.length,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, index + (downloadedLength / totalLength), FILES.length),
|
||||
rate: calculateRate(startTime, index + (downloadedLength / totalLength)),
|
||||
percentage: progress.toFixed(1),
|
||||
file_progress: {
|
||||
name: file.name,
|
||||
downloaded: downloadedLength,
|
||||
total: totalLength,
|
||||
percentage: progress.toFixed(1)
|
||||
}
|
||||
});
|
||||
lastProgressUpdate = now;
|
||||
}
|
||||
});
|
||||
|
||||
response.data.pipe(writer);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
writer.on('finish', resolve);
|
||||
writer.on('error', reject);
|
||||
});
|
||||
} catch (error) {
|
||||
fs.unlinkSync(path.join(csvDir, file.name));
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Main function to update all files
|
||||
async function updateFiles() {
|
||||
const startTime = Date.now();
|
||||
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'Starting CSV update',
|
||||
current: 0,
|
||||
total: FILES.length,
|
||||
elapsed: '0s',
|
||||
remaining: null,
|
||||
rate: 0,
|
||||
percentage: '0'
|
||||
});
|
||||
|
||||
try {
|
||||
for (let i = 0; i < FILES.length; i++) {
|
||||
if (isCancelled) {
|
||||
return;
|
||||
}
|
||||
|
||||
const file = FILES[i];
|
||||
await downloadFile(file, i, startTime);
|
||||
|
||||
outputProgress({
|
||||
status: 'running',
|
||||
operation: 'CSV update in progress',
|
||||
current: i + 1,
|
||||
total: FILES.length,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: estimateRemaining(startTime, i + 1, FILES.length),
|
||||
rate: calculateRate(startTime, i + 1),
|
||||
percentage: (((i + 1) / FILES.length) * 100).toFixed(1)
|
||||
});
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
status: 'complete',
|
||||
operation: 'CSV update complete',
|
||||
current: FILES.length,
|
||||
total: FILES.length,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: '0s',
|
||||
rate: calculateRate(startTime, FILES.length),
|
||||
percentage: '100'
|
||||
});
|
||||
} catch (error) {
|
||||
outputProgress({
|
||||
status: 'error',
|
||||
operation: 'CSV update failed',
|
||||
error: error.message,
|
||||
current: 0,
|
||||
total: FILES.length,
|
||||
elapsed: formatElapsedTime(startTime),
|
||||
remaining: null,
|
||||
rate: 0
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// Run the update only if this is the main module
|
||||
if (require.main === module) {
|
||||
updateFiles().catch((error) => {
|
||||
console.error('Error updating CSV files:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
// Export the functions needed by the route
|
||||
module.exports = {
|
||||
updateFiles,
|
||||
cancelUpdate
|
||||
};
|
||||
@@ -1,547 +0,0 @@
|
||||
const mysql = require('mysql2/promise');
|
||||
const path = require('path');
|
||||
const dotenv = require('dotenv');
|
||||
const fs = require('fs');
|
||||
|
||||
dotenv.config({ path: path.join(__dirname, '../.env') });
|
||||
|
||||
const dbConfig = {
|
||||
host: process.env.DB_HOST,
|
||||
user: process.env.DB_USER,
|
||||
password: process.env.DB_PASSWORD,
|
||||
database: process.env.DB_NAME,
|
||||
multipleStatements: true
|
||||
};
|
||||
|
||||
// Helper function to output progress in JSON format
|
||||
function outputProgress(data) {
|
||||
if (!data.status) {
|
||||
data = {
|
||||
status: 'running',
|
||||
...data
|
||||
};
|
||||
}
|
||||
console.log(JSON.stringify(data));
|
||||
}
|
||||
|
||||
// Core tables that must be created
|
||||
const CORE_TABLES = [
|
||||
'products',
|
||||
'orders',
|
||||
'purchase_orders',
|
||||
'categories',
|
||||
'product_categories'
|
||||
];
|
||||
|
||||
// Config tables that must be created
|
||||
const CONFIG_TABLES = [
|
||||
'stock_thresholds',
|
||||
'lead_time_thresholds',
|
||||
'sales_velocity_config',
|
||||
'abc_classification_config',
|
||||
'safety_stock_config',
|
||||
'sales_seasonality',
|
||||
'turnover_config'
|
||||
];
|
||||
|
||||
// Split SQL into individual statements
|
||||
function splitSQLStatements(sql) {
|
||||
// First, normalize line endings
|
||||
sql = sql.replace(/\r\n/g, '\n');
|
||||
|
||||
// Track statement boundaries
|
||||
let statements = [];
|
||||
let currentStatement = '';
|
||||
let inString = false;
|
||||
let stringChar = '';
|
||||
|
||||
// Process character by character
|
||||
for (let i = 0; i < sql.length; i++) {
|
||||
const char = sql[i];
|
||||
const nextChar = sql[i + 1] || '';
|
||||
|
||||
// Handle string literals
|
||||
if ((char === "'" || char === '"') && sql[i - 1] !== '\\') {
|
||||
if (!inString) {
|
||||
inString = true;
|
||||
stringChar = char;
|
||||
} else if (char === stringChar) {
|
||||
inString = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Handle comments
|
||||
if (!inString && char === '-' && nextChar === '-') {
|
||||
// Skip to end of line
|
||||
while (i < sql.length && sql[i] !== '\n') i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!inString && char === '/' && nextChar === '*') {
|
||||
// Skip until closing */
|
||||
i += 2;
|
||||
while (i < sql.length && (sql[i] !== '*' || sql[i + 1] !== '/')) i++;
|
||||
i++; // Skip the closing /
|
||||
continue;
|
||||
}
|
||||
|
||||
// Handle statement boundaries
|
||||
if (!inString && char === ';') {
|
||||
if (currentStatement.trim()) {
|
||||
statements.push(currentStatement.trim());
|
||||
}
|
||||
currentStatement = '';
|
||||
} else {
|
||||
currentStatement += char;
|
||||
}
|
||||
}
|
||||
|
||||
// Add the last statement if it exists
|
||||
if (currentStatement.trim()) {
|
||||
statements.push(currentStatement.trim());
|
||||
}
|
||||
|
||||
return statements;
|
||||
}
|
||||
|
||||
async function resetDatabase() {
|
||||
outputProgress({
|
||||
operation: 'Starting database reset',
|
||||
message: 'Connecting to database...'
|
||||
});
|
||||
|
||||
// Debug: Log current directory and file paths
|
||||
outputProgress({
|
||||
operation: 'Debug paths',
|
||||
message: {
|
||||
currentDir: process.cwd(),
|
||||
__dirname: __dirname,
|
||||
schemaPath: path.join(__dirname, '../db/schema.sql')
|
||||
}
|
||||
});
|
||||
|
||||
const connection = await mysql.createConnection(dbConfig);
|
||||
|
||||
try {
|
||||
// Check MySQL privileges
|
||||
outputProgress({
|
||||
operation: 'Checking privileges',
|
||||
message: 'Verifying MySQL user privileges...'
|
||||
});
|
||||
|
||||
const [grants] = await connection.query('SHOW GRANTS');
|
||||
outputProgress({
|
||||
operation: 'User privileges',
|
||||
message: {
|
||||
grants: grants.map(g => Object.values(g)[0])
|
||||
}
|
||||
});
|
||||
|
||||
// Enable warnings as errors
|
||||
await connection.query('SET SESSION sql_notes = 1');
|
||||
|
||||
// Log database config (without sensitive info)
|
||||
outputProgress({
|
||||
operation: 'Database config',
|
||||
message: `Using database: ${dbConfig.database} on host: ${dbConfig.host}`
|
||||
});
|
||||
|
||||
// Get list of all tables in the current database
|
||||
outputProgress({
|
||||
operation: 'Getting table list',
|
||||
message: 'Retrieving all table names...'
|
||||
});
|
||||
|
||||
const [tables] = await connection.query(`
|
||||
SELECT GROUP_CONCAT(table_name) as tables
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = DATABASE()
|
||||
AND table_name NOT IN ('users', 'import_history')
|
||||
`);
|
||||
|
||||
if (!tables[0].tables) {
|
||||
outputProgress({
|
||||
operation: 'No tables found',
|
||||
message: 'Database is already empty'
|
||||
});
|
||||
} else {
|
||||
outputProgress({
|
||||
operation: 'Dropping tables',
|
||||
message: 'Dropping all existing tables...'
|
||||
});
|
||||
|
||||
await connection.query('SET FOREIGN_KEY_CHECKS = 0');
|
||||
const dropQuery = `
|
||||
DROP TABLE IF EXISTS
|
||||
${tables[0].tables
|
||||
.split(',')
|
||||
.filter(table => table !== 'users')
|
||||
.map(table => '`' + table + '`')
|
||||
.join(', ')}
|
||||
`;
|
||||
await connection.query(dropQuery);
|
||||
await connection.query('SET FOREIGN_KEY_CHECKS = 1');
|
||||
}
|
||||
|
||||
// Read and execute main schema (core tables)
|
||||
outputProgress({
|
||||
operation: 'Running database setup',
|
||||
message: 'Creating core tables...'
|
||||
});
|
||||
const schemaPath = path.join(__dirname, '../db/schema.sql');
|
||||
|
||||
// Verify file exists
|
||||
if (!fs.existsSync(schemaPath)) {
|
||||
throw new Error(`Schema file not found at: ${schemaPath}`);
|
||||
}
|
||||
|
||||
const schemaSQL = fs.readFileSync(schemaPath, 'utf8');
|
||||
|
||||
outputProgress({
|
||||
operation: 'Schema file',
|
||||
message: {
|
||||
path: schemaPath,
|
||||
exists: fs.existsSync(schemaPath),
|
||||
size: fs.statSync(schemaPath).size,
|
||||
firstFewLines: schemaSQL.split('\n').slice(0, 5).join('\n')
|
||||
}
|
||||
});
|
||||
|
||||
// Execute schema statements one at a time
|
||||
const statements = splitSQLStatements(schemaSQL);
|
||||
outputProgress({
|
||||
operation: 'SQL Execution',
|
||||
message: {
|
||||
totalStatements: statements.length,
|
||||
statements: statements.map((stmt, i) => ({
|
||||
number: i + 1,
|
||||
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : '')
|
||||
}))
|
||||
}
|
||||
});
|
||||
|
||||
for (let i = 0; i < statements.length; i++) {
|
||||
const stmt = statements[i];
|
||||
try {
|
||||
const [result, fields] = await connection.query(stmt);
|
||||
|
||||
// Check for warnings
|
||||
const [warnings] = await connection.query('SHOW WARNINGS');
|
||||
if (warnings && warnings.length > 0) {
|
||||
outputProgress({
|
||||
status: 'warning',
|
||||
operation: 'SQL Warning',
|
||||
statement: i + 1,
|
||||
warnings: warnings
|
||||
});
|
||||
}
|
||||
|
||||
// Verify if table was created (if this was a CREATE TABLE statement)
|
||||
if (stmt.trim().toLowerCase().startsWith('create table')) {
|
||||
const tableName = stmt.match(/create\s+table\s+(?:if\s+not\s+exists\s+)?`?(\w+)`?/i)?.[1];
|
||||
if (tableName) {
|
||||
const [tableExists] = await connection.query(`
|
||||
SELECT COUNT(*) as count
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = DATABASE()
|
||||
AND table_name = ?
|
||||
`, [tableName]);
|
||||
|
||||
outputProgress({
|
||||
operation: 'Table Creation Verification',
|
||||
message: {
|
||||
table: tableName,
|
||||
exists: tableExists[0].count > 0
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
operation: 'SQL Progress',
|
||||
message: {
|
||||
statement: i + 1,
|
||||
total: statements.length,
|
||||
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : ''),
|
||||
affectedRows: result.affectedRows
|
||||
}
|
||||
});
|
||||
} catch (sqlError) {
|
||||
outputProgress({
|
||||
status: 'error',
|
||||
operation: 'SQL Error',
|
||||
error: sqlError.message,
|
||||
sqlState: sqlError.sqlState,
|
||||
errno: sqlError.errno,
|
||||
statement: stmt,
|
||||
statementNumber: i + 1
|
||||
});
|
||||
throw sqlError;
|
||||
}
|
||||
}
|
||||
|
||||
// List all tables in the database after schema execution
|
||||
outputProgress({
|
||||
operation: 'Debug database',
|
||||
message: {
|
||||
currentDatabase: (await connection.query('SELECT DATABASE() as db'))[0][0].db
|
||||
}
|
||||
});
|
||||
|
||||
const [allTables] = await connection.query(`
|
||||
SELECT
|
||||
table_schema,
|
||||
table_name,
|
||||
engine,
|
||||
create_time,
|
||||
table_rows
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = DATABASE()
|
||||
`);
|
||||
|
||||
if (allTables.length === 0) {
|
||||
outputProgress({
|
||||
operation: 'Warning',
|
||||
message: 'No tables found in database after schema execution'
|
||||
});
|
||||
} else {
|
||||
outputProgress({
|
||||
operation: 'Tables after schema execution',
|
||||
message: {
|
||||
count: allTables.length,
|
||||
tables: allTables.map(t => ({
|
||||
schema: t.table_schema,
|
||||
name: t.table_name,
|
||||
engine: t.engine,
|
||||
created: t.create_time,
|
||||
rows: t.table_rows
|
||||
}))
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Also check table status
|
||||
const [tableStatus] = await connection.query('SHOW TABLE STATUS');
|
||||
outputProgress({
|
||||
operation: 'Table Status',
|
||||
message: {
|
||||
tables: tableStatus.map(t => ({
|
||||
name: t.Name,
|
||||
engine: t.Engine,
|
||||
version: t.Version,
|
||||
rowFormat: t.Row_format,
|
||||
rows: t.Rows,
|
||||
createTime: t.Create_time,
|
||||
updateTime: t.Update_time
|
||||
}))
|
||||
}
|
||||
});
|
||||
|
||||
// Verify core tables were created using SHOW TABLES
|
||||
const [showTables] = await connection.query('SHOW TABLES');
|
||||
const existingTables = showTables.map(t => Object.values(t)[0]);
|
||||
|
||||
outputProgress({
|
||||
operation: 'Core tables verification',
|
||||
message: {
|
||||
found: existingTables,
|
||||
expected: CORE_TABLES
|
||||
}
|
||||
});
|
||||
|
||||
const missingCoreTables = CORE_TABLES.filter(
|
||||
t => !existingTables.includes(t)
|
||||
);
|
||||
|
||||
if (missingCoreTables.length > 0) {
|
||||
throw new Error(
|
||||
`Failed to create core tables: ${missingCoreTables.join(', ')}`
|
||||
);
|
||||
}
|
||||
|
||||
// Verify all core tables use InnoDB
|
||||
const [engineStatus] = await connection.query('SHOW TABLE STATUS WHERE Name IN (?)', [CORE_TABLES]);
|
||||
const nonInnoDBTables = engineStatus.filter(t => t.Engine !== 'InnoDB');
|
||||
|
||||
if (nonInnoDBTables.length > 0) {
|
||||
throw new Error(
|
||||
`Tables using non-InnoDB engine: ${nonInnoDBTables.map(t => t.Name).join(', ')}`
|
||||
);
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
operation: 'Core tables created',
|
||||
message: `Successfully created tables: ${CORE_TABLES.join(', ')}`
|
||||
});
|
||||
|
||||
// Read and execute config schema
|
||||
outputProgress({
|
||||
operation: 'Running config setup',
|
||||
message: 'Creating configuration tables...'
|
||||
});
|
||||
const configSchemaSQL = fs.readFileSync(
|
||||
path.join(__dirname, '../db/config-schema.sql'),
|
||||
'utf8'
|
||||
);
|
||||
|
||||
// Execute config schema statements one at a time
|
||||
const configStatements = splitSQLStatements(configSchemaSQL);
|
||||
outputProgress({
|
||||
operation: 'Config SQL Execution',
|
||||
message: {
|
||||
totalStatements: configStatements.length,
|
||||
statements: configStatements.map((stmt, i) => ({
|
||||
number: i + 1,
|
||||
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : '')
|
||||
}))
|
||||
}
|
||||
});
|
||||
|
||||
for (let i = 0; i < configStatements.length; i++) {
|
||||
const stmt = configStatements[i];
|
||||
try {
|
||||
const [result, fields] = await connection.query(stmt);
|
||||
|
||||
// Check for warnings
|
||||
const [warnings] = await connection.query('SHOW WARNINGS');
|
||||
if (warnings && warnings.length > 0) {
|
||||
outputProgress({
|
||||
status: 'warning',
|
||||
operation: 'Config SQL Warning',
|
||||
statement: i + 1,
|
||||
warnings: warnings
|
||||
});
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
operation: 'Config SQL Progress',
|
||||
message: {
|
||||
statement: i + 1,
|
||||
total: configStatements.length,
|
||||
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : ''),
|
||||
affectedRows: result.affectedRows
|
||||
}
|
||||
});
|
||||
} catch (sqlError) {
|
||||
outputProgress({
|
||||
status: 'error',
|
||||
operation: 'Config SQL Error',
|
||||
error: sqlError.message,
|
||||
sqlState: sqlError.sqlState,
|
||||
errno: sqlError.errno,
|
||||
statement: stmt,
|
||||
statementNumber: i + 1
|
||||
});
|
||||
throw sqlError;
|
||||
}
|
||||
}
|
||||
|
||||
// Verify config tables were created
|
||||
const [showConfigTables] = await connection.query('SHOW TABLES');
|
||||
const existingConfigTables = showConfigTables.map(t => Object.values(t)[0]);
|
||||
|
||||
outputProgress({
|
||||
operation: 'Config tables verification',
|
||||
message: {
|
||||
found: existingConfigTables,
|
||||
expected: CONFIG_TABLES
|
||||
}
|
||||
});
|
||||
|
||||
const missingConfigTables = CONFIG_TABLES.filter(
|
||||
t => !existingConfigTables.includes(t)
|
||||
);
|
||||
|
||||
if (missingConfigTables.length > 0) {
|
||||
throw new Error(
|
||||
`Failed to create config tables: ${missingConfigTables.join(', ')}`
|
||||
);
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
operation: 'Config tables created',
|
||||
message: `Successfully created tables: ${CONFIG_TABLES.join(', ')}`
|
||||
});
|
||||
|
||||
// Read and execute metrics schema (metrics tables)
|
||||
outputProgress({
|
||||
operation: 'Running metrics setup',
|
||||
message: 'Creating metrics tables...'
|
||||
});
|
||||
const metricsSchemaSQL = fs.readFileSync(
|
||||
path.join(__dirname, '../db/metrics-schema.sql'),
|
||||
'utf8'
|
||||
);
|
||||
|
||||
// Execute metrics schema statements one at a time
|
||||
const metricsStatements = splitSQLStatements(metricsSchemaSQL);
|
||||
outputProgress({
|
||||
operation: 'Metrics SQL Execution',
|
||||
message: {
|
||||
totalStatements: metricsStatements.length,
|
||||
statements: metricsStatements.map((stmt, i) => ({
|
||||
number: i + 1,
|
||||
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : '')
|
||||
}))
|
||||
}
|
||||
});
|
||||
|
||||
for (let i = 0; i < metricsStatements.length; i++) {
|
||||
const stmt = metricsStatements[i];
|
||||
try {
|
||||
const [result, fields] = await connection.query(stmt);
|
||||
|
||||
// Check for warnings
|
||||
const [warnings] = await connection.query('SHOW WARNINGS');
|
||||
if (warnings && warnings.length > 0) {
|
||||
outputProgress({
|
||||
status: 'warning',
|
||||
operation: 'Metrics SQL Warning',
|
||||
statement: i + 1,
|
||||
warnings: warnings
|
||||
});
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
operation: 'Metrics SQL Progress',
|
||||
message: {
|
||||
statement: i + 1,
|
||||
total: metricsStatements.length,
|
||||
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : ''),
|
||||
affectedRows: result.affectedRows
|
||||
}
|
||||
});
|
||||
} catch (sqlError) {
|
||||
outputProgress({
|
||||
status: 'error',
|
||||
operation: 'Metrics SQL Error',
|
||||
error: sqlError.message,
|
||||
sqlState: sqlError.sqlState,
|
||||
errno: sqlError.errno,
|
||||
statement: stmt,
|
||||
statementNumber: i + 1
|
||||
});
|
||||
throw sqlError;
|
||||
}
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
status: 'complete',
|
||||
operation: 'Database reset complete',
|
||||
message: 'Database has been reset and all tables recreated'
|
||||
});
|
||||
} catch (error) {
|
||||
outputProgress({
|
||||
status: 'error',
|
||||
operation: 'Failed to reset database',
|
||||
error: error.message,
|
||||
stack: error.stack
|
||||
});
|
||||
process.exit(1);
|
||||
} finally {
|
||||
await connection.end();
|
||||
}
|
||||
}
|
||||
|
||||
// Run the reset
|
||||
resetDatabase();
|
||||
@@ -1,331 +0,0 @@
|
||||
const mysql = require('mysql2/promise');
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
require('dotenv').config({ path: path.resolve(__dirname, '../.env') });
|
||||
|
||||
const dbConfig = {
|
||||
host: process.env.DB_HOST,
|
||||
user: process.env.DB_USER,
|
||||
password: process.env.DB_PASSWORD,
|
||||
database: process.env.DB_NAME,
|
||||
multipleStatements: true
|
||||
};
|
||||
|
||||
function outputProgress(data) {
|
||||
if (!data.status) {
|
||||
data = {
|
||||
status: 'running',
|
||||
...data
|
||||
};
|
||||
}
|
||||
console.log(JSON.stringify(data));
|
||||
}
|
||||
|
||||
// Explicitly define all metrics-related tables in dependency order
|
||||
const METRICS_TABLES = [
|
||||
'brand_metrics',
|
||||
'brand_time_metrics',
|
||||
'category_forecasts',
|
||||
'category_metrics',
|
||||
'category_sales_metrics',
|
||||
'category_time_metrics',
|
||||
'product_metrics',
|
||||
'product_time_aggregates',
|
||||
'sales_forecasts',
|
||||
'temp_purchase_metrics',
|
||||
'temp_sales_metrics',
|
||||
'vendor_metrics', //before vendor_details for foreign key
|
||||
'vendor_time_metrics', //before vendor_details for foreign key
|
||||
'vendor_details'
|
||||
];
|
||||
|
||||
// Split SQL into individual statements
|
||||
function splitSQLStatements(sql) {
|
||||
sql = sql.replace(/\r\n/g, '\n');
|
||||
let statements = [];
|
||||
let currentStatement = '';
|
||||
let inString = false;
|
||||
let stringChar = '';
|
||||
|
||||
for (let i = 0; i < sql.length; i++) {
|
||||
const char = sql[i];
|
||||
const nextChar = sql[i + 1] || '';
|
||||
|
||||
if ((char === "'" || char === '"') && sql[i - 1] !== '\\') {
|
||||
if (!inString) {
|
||||
inString = true;
|
||||
stringChar = char;
|
||||
} else if (char === stringChar) {
|
||||
inString = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (!inString && char === '-' && nextChar === '-') {
|
||||
while (i < sql.length && sql[i] !== '\n') i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!inString && char === '/' && nextChar === '*') {
|
||||
i += 2;
|
||||
while (i < sql.length && (sql[i] !== '*' || sql[i + 1] !== '/')) i++;
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!inString && char === ';') {
|
||||
if (currentStatement.trim()) {
|
||||
statements.push(currentStatement.trim());
|
||||
}
|
||||
currentStatement = '';
|
||||
} else {
|
||||
currentStatement += char;
|
||||
}
|
||||
}
|
||||
|
||||
if (currentStatement.trim()) {
|
||||
statements.push(currentStatement.trim());
|
||||
}
|
||||
|
||||
return statements;
|
||||
}
|
||||
|
||||
async function resetMetrics() {
|
||||
let connection;
|
||||
try {
|
||||
outputProgress({
|
||||
operation: 'Starting metrics reset',
|
||||
message: 'Connecting to database...'
|
||||
});
|
||||
|
||||
connection = await mysql.createConnection(dbConfig);
|
||||
await connection.beginTransaction();
|
||||
|
||||
// First verify current state
|
||||
const [initialTables] = await connection.query(`
|
||||
SELECT TABLE_NAME as name
|
||||
FROM information_schema.tables
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME IN (?)
|
||||
`, [METRICS_TABLES]);
|
||||
|
||||
outputProgress({
|
||||
operation: 'Initial state',
|
||||
message: `Found ${initialTables.length} existing metrics tables: ${initialTables.map(t => t.name).join(', ')}`
|
||||
});
|
||||
|
||||
// Disable foreign key checks at the start
|
||||
await connection.query('SET FOREIGN_KEY_CHECKS = 0');
|
||||
|
||||
// Drop all metrics tables in reverse order to handle dependencies
|
||||
outputProgress({
|
||||
operation: 'Dropping metrics tables',
|
||||
message: 'Removing existing metrics tables...'
|
||||
});
|
||||
|
||||
for (const table of [...METRICS_TABLES].reverse()) {
|
||||
try {
|
||||
await connection.query(`DROP TABLE IF EXISTS ${table}`);
|
||||
|
||||
// Verify the table was actually dropped
|
||||
const [checkDrop] = await connection.query(`
|
||||
SELECT COUNT(*) as count
|
||||
FROM information_schema.tables
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = ?
|
||||
`, [table]);
|
||||
|
||||
if (checkDrop[0].count > 0) {
|
||||
throw new Error(`Failed to drop table ${table} - table still exists`);
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
operation: 'Table dropped',
|
||||
message: `Successfully dropped table: ${table}`
|
||||
});
|
||||
} catch (err) {
|
||||
outputProgress({
|
||||
status: 'error',
|
||||
operation: 'Drop table error',
|
||||
message: `Error dropping table ${table}: ${err.message}`
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
// Verify all tables were dropped
|
||||
const [afterDrop] = await connection.query(`
|
||||
SELECT TABLE_NAME as name
|
||||
FROM information_schema.tables
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME IN (?)
|
||||
`, [METRICS_TABLES]);
|
||||
|
||||
if (afterDrop.length > 0) {
|
||||
throw new Error(`Failed to drop all tables. Remaining tables: ${afterDrop.map(t => t.name).join(', ')}`);
|
||||
}
|
||||
|
||||
// Read metrics schema
|
||||
outputProgress({
|
||||
operation: 'Reading schema',
|
||||
message: 'Loading metrics schema file...'
|
||||
});
|
||||
|
||||
const schemaPath = path.resolve(__dirname, '../db/metrics-schema.sql');
|
||||
if (!fs.existsSync(schemaPath)) {
|
||||
throw new Error(`Schema file not found at: ${schemaPath}`);
|
||||
}
|
||||
|
||||
const schemaSQL = fs.readFileSync(schemaPath, 'utf8');
|
||||
const statements = splitSQLStatements(schemaSQL);
|
||||
|
||||
outputProgress({
|
||||
operation: 'Schema loaded',
|
||||
message: `Found ${statements.length} SQL statements to execute`
|
||||
});
|
||||
|
||||
// Execute schema statements
|
||||
for (let i = 0; i < statements.length; i++) {
|
||||
const stmt = statements[i];
|
||||
try {
|
||||
await connection.query(stmt);
|
||||
|
||||
// Check for warnings
|
||||
const [warnings] = await connection.query('SHOW WARNINGS');
|
||||
if (warnings && warnings.length > 0) {
|
||||
outputProgress({
|
||||
status: 'warning',
|
||||
operation: 'SQL Warning',
|
||||
message: {
|
||||
statement: i + 1,
|
||||
warnings: warnings
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// If this is a CREATE TABLE statement, verify the table was created
|
||||
if (stmt.trim().toLowerCase().startsWith('create table')) {
|
||||
const tableName = stmt.match(/create\s+table\s+(?:if\s+not\s+exists\s+)?`?(\w+)`?/i)?.[1];
|
||||
if (tableName) {
|
||||
const [checkCreate] = await connection.query(`
|
||||
SELECT TABLE_NAME as name, CREATE_TIME as created
|
||||
FROM information_schema.tables
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = ?
|
||||
`, [tableName]);
|
||||
|
||||
if (checkCreate.length === 0) {
|
||||
throw new Error(`Failed to create table ${tableName} - table does not exist after CREATE statement`);
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
operation: 'Table created',
|
||||
message: `Successfully created table: ${tableName} at ${checkCreate[0].created}`
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
outputProgress({
|
||||
operation: 'SQL Progress',
|
||||
message: {
|
||||
statement: i + 1,
|
||||
total: statements.length,
|
||||
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : '')
|
||||
}
|
||||
});
|
||||
} catch (sqlError) {
|
||||
outputProgress({
|
||||
status: 'error',
|
||||
operation: 'SQL Error',
|
||||
message: {
|
||||
error: sqlError.message,
|
||||
sqlState: sqlError.sqlState,
|
||||
errno: sqlError.errno,
|
||||
statement: stmt,
|
||||
statementNumber: i + 1
|
||||
}
|
||||
});
|
||||
throw sqlError;
|
||||
}
|
||||
}
|
||||
|
||||
// Re-enable foreign key checks after all tables are created
|
||||
await connection.query('SET FOREIGN_KEY_CHECKS = 1');
|
||||
|
||||
// Verify metrics tables were created
|
||||
outputProgress({
|
||||
operation: 'Verifying metrics tables',
|
||||
message: 'Checking all metrics tables were created...'
|
||||
});
|
||||
|
||||
const [metricsTablesResult] = await connection.query(`
|
||||
SELECT
|
||||
TABLE_NAME as name,
|
||||
TABLE_ROWS as \`rows\`,
|
||||
CREATE_TIME as created
|
||||
FROM information_schema.tables
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME IN (?)
|
||||
`, [METRICS_TABLES]);
|
||||
|
||||
outputProgress({
|
||||
operation: 'Tables found',
|
||||
message: `Found ${metricsTablesResult.length} tables: ${metricsTablesResult.map(t =>
|
||||
`${t.name} (created: ${t.created})`
|
||||
).join(', ')}`
|
||||
});
|
||||
|
||||
const existingMetricsTables = metricsTablesResult.map(t => t.name);
|
||||
const missingMetricsTables = METRICS_TABLES.filter(t => !existingMetricsTables.includes(t));
|
||||
|
||||
if (missingMetricsTables.length > 0) {
|
||||
// Do one final check of the actual tables
|
||||
const [finalCheck] = await connection.query('SHOW TABLES');
|
||||
outputProgress({
|
||||
operation: 'Final table check',
|
||||
message: `All database tables: ${finalCheck.map(t => Object.values(t)[0]).join(', ')}`
|
||||
});
|
||||
throw new Error(`Failed to create metrics tables: ${missingMetricsTables.join(', ')}`);
|
||||
}
|
||||
|
||||
await connection.commit();
|
||||
|
||||
outputProgress({
|
||||
status: 'complete',
|
||||
operation: 'Reset complete',
|
||||
message: 'All metrics tables have been reset successfully'
|
||||
});
|
||||
} catch (error) {
|
||||
outputProgress({
|
||||
status: 'error',
|
||||
operation: 'Reset failed',
|
||||
message: error.message,
|
||||
stack: error.stack
|
||||
});
|
||||
|
||||
if (connection) {
|
||||
await connection.rollback();
|
||||
// Make sure to re-enable foreign key checks even if there's an error
|
||||
await connection.query('SET FOREIGN_KEY_CHECKS = 1').catch(() => {});
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
if (connection) {
|
||||
// One final attempt to ensure foreign key checks are enabled
|
||||
await connection.query('SET FOREIGN_KEY_CHECKS = 1').catch(() => {});
|
||||
await connection.end();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Export if required as a module
|
||||
if (typeof module !== 'undefined' && module.exports) {
|
||||
module.exports = resetMetrics;
|
||||
}
|
||||
|
||||
// Run if called from command line
|
||||
if (require.main === module) {
|
||||
resetMetrics().catch(error => {
|
||||
console.error('Error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -1,180 +0,0 @@
|
||||
const readline = require('readline');
|
||||
|
||||
const rl = readline.createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout
|
||||
});
|
||||
|
||||
const question = (query) => new Promise((resolve) => rl.question(query, resolve));
|
||||
|
||||
async function loadScript(name) {
|
||||
try {
|
||||
return await require(name);
|
||||
} catch (error) {
|
||||
console.error(`Failed to load script ${name}:`, error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function runWithTimeout(fn) {
|
||||
return new Promise((resolve, reject) => {
|
||||
// Create a child process for the script
|
||||
const child = require('child_process').fork(fn.toString(), [], {
|
||||
stdio: 'inherit'
|
||||
});
|
||||
|
||||
child.on('exit', (code) => {
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(new Error(`Script exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
|
||||
child.on('error', (err) => {
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function clearScreen() {
|
||||
process.stdout.write('\x1Bc');
|
||||
}
|
||||
|
||||
const scripts = {
|
||||
'Import Scripts': {
|
||||
'1': { name: 'Full Import From Production', path: './import-from-prod' },
|
||||
'2': { name: 'Individual Import Scripts ▸', submenu: {
|
||||
'1': { name: 'Import Orders', path: './import/orders', key: 'importOrders' },
|
||||
'2': { name: 'Import Products', path: './import/products', key: 'importProducts' },
|
||||
'3': { name: 'Import Purchase Orders', path: './import/purchase-orders' },
|
||||
'4': { name: 'Import Categories', path: './import/categories' },
|
||||
'b': { name: 'Back to Main Menu' }
|
||||
}}
|
||||
},
|
||||
'Metrics': {
|
||||
'3': { name: 'Calculate All Metrics', path: './calculate-metrics' },
|
||||
'4': { name: 'Individual Metric Scripts ▸', submenu: {
|
||||
'1': { name: 'Brand Metrics', path: './metrics/brand-metrics' },
|
||||
'2': { name: 'Category Metrics', path: './metrics/category-metrics' },
|
||||
'3': { name: 'Financial Metrics', path: './metrics/financial-metrics' },
|
||||
'4': { name: 'Product Metrics', path: './metrics/product-metrics' },
|
||||
'5': { name: 'Sales Forecasts', path: './metrics/sales-forecasts' },
|
||||
'6': { name: 'Time Aggregates', path: './metrics/time-aggregates' },
|
||||
'7': { name: 'Vendor Metrics', path: './metrics/vendor-metrics' },
|
||||
'b': { name: 'Back to Main Menu' }
|
||||
}}
|
||||
},
|
||||
'Database Management': {
|
||||
'5': { name: 'Test Production Connection', path: './test-prod-connection' }
|
||||
},
|
||||
'Reset Scripts': {
|
||||
'6': { name: 'Reset Database', path: './reset-db' },
|
||||
'7': { name: 'Reset Metrics', path: './reset-metrics' }
|
||||
}
|
||||
};
|
||||
|
||||
let lastRun = null;
|
||||
|
||||
async function displayMenu(menuItems, title = 'Inventory Management Script Runner') {
|
||||
clearScreen();
|
||||
console.log(`\n${title}\n`);
|
||||
|
||||
for (const [category, items] of Object.entries(menuItems)) {
|
||||
console.log(`\n${category}:`);
|
||||
Object.entries(items).forEach(([key, script]) => {
|
||||
console.log(`${key}. ${script.name}`);
|
||||
});
|
||||
}
|
||||
|
||||
if (lastRun) {
|
||||
console.log('\nQuick Access:');
|
||||
console.log(`r. Repeat Last Script (${lastRun.name})`);
|
||||
}
|
||||
|
||||
console.log('\nq. Quit\n');
|
||||
}
|
||||
|
||||
async function handleSubmenu(submenu, title) {
|
||||
while (true) {
|
||||
await displayMenu({"Individual Scripts": submenu}, title);
|
||||
const choice = await question('Select an option (or b to go back): ');
|
||||
|
||||
if (choice.toLowerCase() === 'b') {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (submenu[choice]) {
|
||||
return submenu[choice];
|
||||
}
|
||||
|
||||
console.log('Invalid selection. Please try again.');
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
}
|
||||
}
|
||||
|
||||
async function runScript(script) {
|
||||
console.log(`\nRunning: ${script.name}`);
|
||||
try {
|
||||
const scriptPath = require.resolve(script.path);
|
||||
await runWithTimeout(scriptPath);
|
||||
console.log('\nScript completed successfully');
|
||||
lastRun = script;
|
||||
} catch (error) {
|
||||
console.error('\nError running script:', error);
|
||||
}
|
||||
await question('\nPress Enter to continue...');
|
||||
}
|
||||
|
||||
async function main() {
|
||||
while (true) {
|
||||
await displayMenu(scripts);
|
||||
|
||||
const choice = await question('Select an option: ');
|
||||
|
||||
if (choice.toLowerCase() === 'q') {
|
||||
break;
|
||||
}
|
||||
|
||||
if (choice.toLowerCase() === 'r' && lastRun) {
|
||||
await runScript(lastRun);
|
||||
continue;
|
||||
}
|
||||
|
||||
let selectedScript = null;
|
||||
for (const category of Object.values(scripts)) {
|
||||
if (category[choice]) {
|
||||
selectedScript = category[choice];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!selectedScript) {
|
||||
console.log('Invalid selection. Please try again.');
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (selectedScript.submenu) {
|
||||
const submenuChoice = await handleSubmenu(
|
||||
selectedScript.submenu,
|
||||
selectedScript.name
|
||||
);
|
||||
if (submenuChoice && submenuChoice.path) {
|
||||
await runScript(submenuChoice);
|
||||
}
|
||||
} else if (selectedScript.path) {
|
||||
await runScript(selectedScript);
|
||||
}
|
||||
}
|
||||
|
||||
rl.close();
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main().catch(error => {
|
||||
console.error('Fatal error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
const mysql = require('mysql2/promise');
|
||||
const { Client } = require('ssh2');
|
||||
const dotenv = require('dotenv');
|
||||
const path = require('path');
|
||||
|
||||
dotenv.config({ path: path.join(__dirname, '../.env') });
|
||||
|
||||
// SSH configuration
|
||||
const sshConfig = {
|
||||
host: process.env.PROD_SSH_HOST,
|
||||
port: process.env.PROD_SSH_PORT || 22,
|
||||
username: process.env.PROD_SSH_USER,
|
||||
privateKey: process.env.PROD_SSH_KEY_PATH ? require('fs').readFileSync(process.env.PROD_SSH_KEY_PATH) : undefined
|
||||
};
|
||||
|
||||
// Database configuration
|
||||
const dbConfig = {
|
||||
host: process.env.PROD_DB_HOST || 'localhost', // Usually localhost when tunneling
|
||||
user: process.env.PROD_DB_USER,
|
||||
password: process.env.PROD_DB_PASSWORD,
|
||||
database: process.env.PROD_DB_NAME,
|
||||
port: process.env.PROD_DB_PORT || 3306
|
||||
};
|
||||
|
||||
async function testConnection() {
|
||||
const ssh = new Client();
|
||||
|
||||
try {
|
||||
// Create new Promise for SSH connection
|
||||
await new Promise((resolve, reject) => {
|
||||
ssh.on('ready', resolve)
|
||||
.on('error', reject)
|
||||
.connect(sshConfig);
|
||||
});
|
||||
|
||||
console.log('SSH Connection successful!');
|
||||
|
||||
// Forward local port to remote MySQL port
|
||||
const tunnel = await new Promise((resolve, reject) => {
|
||||
ssh.forwardOut(
|
||||
'127.0.0.1',
|
||||
0,
|
||||
dbConfig.host,
|
||||
dbConfig.port,
|
||||
(err, stream) => {
|
||||
if (err) reject(err);
|
||||
resolve(stream);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
console.log('Port forwarding established');
|
||||
|
||||
// Create MySQL connection over SSH tunnel
|
||||
const connection = await mysql.createConnection({
|
||||
...dbConfig,
|
||||
stream: tunnel
|
||||
});
|
||||
|
||||
console.log('MySQL Connection successful!');
|
||||
|
||||
// Test query
|
||||
const [rows] = await connection.query('SELECT COUNT(*) as count FROM products');
|
||||
console.log('Query successful! Product count:', rows[0].count);
|
||||
|
||||
// Clean up
|
||||
await connection.end();
|
||||
ssh.end();
|
||||
console.log('Connections closed successfully');
|
||||
return rows[0].count;
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error:', error);
|
||||
if (ssh) ssh.end();
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
// If running directly (not imported)
|
||||
if (require.main === module) {
|
||||
testConnection()
|
||||
.then(() => process.exit(0))
|
||||
.catch(error => {
|
||||
console.error('Test failed:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { testConnection };
|
||||
@@ -4,7 +4,9 @@ const cors = require('cors');
|
||||
const corsMiddleware = cors({
|
||||
origin: [
|
||||
'https://inventory.kent.pw',
|
||||
'http://localhost:5173',
|
||||
'http://localhost:5175',
|
||||
'https://tools.acherryontop.com',
|
||||
'https://tools.acherryontop.com',
|
||||
/^http:\/\/192\.168\.\d+\.\d+(:\d+)?$/,
|
||||
/^http:\/\/10\.\d+\.\d+\.\d+(:\d+)?$/
|
||||
],
|
||||
@@ -26,7 +28,7 @@ const corsErrorHandler = (err, req, res, next) => {
|
||||
res.status(403).json({
|
||||
error: 'CORS not allowed',
|
||||
origin: req.get('Origin'),
|
||||
message: 'Origin not in allowed list: https://inventory.kent.pw, localhost:5173, 192.168.x.x, or 10.x.x.x'
|
||||
message: 'Origin not in allowed list: https://inventory.kent.pw, https://tools.acherryontop.com, https://tools.acherryontop.com, localhost:5175, 192.168.x.x, or 10.x.x.x'
|
||||
});
|
||||
} else {
|
||||
next(err);
|
||||
|
||||
1
inventory-server/src/routes/ai-validation.js
Normal file
1
inventory-server/src/routes/ai-validation.js
Normal file
@@ -0,0 +1 @@
|
||||
|
||||
@@ -1,699 +1 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
|
||||
// Get overall analytics stats
|
||||
router.get('/stats', async (req, res) => {
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
|
||||
const [results] = await pool.query(`
|
||||
SELECT
|
||||
COALESCE(
|
||||
ROUND(
|
||||
(SUM(o.price * o.quantity - p.cost_price * o.quantity) /
|
||||
NULLIF(SUM(o.price * o.quantity), 0)) * 100, 1
|
||||
),
|
||||
0
|
||||
) as profitMargin,
|
||||
COALESCE(
|
||||
ROUND(
|
||||
(AVG(p.price / NULLIF(p.cost_price, 0) - 1) * 100), 1
|
||||
),
|
||||
0
|
||||
) as averageMarkup,
|
||||
COALESCE(
|
||||
ROUND(
|
||||
SUM(o.quantity) / NULLIF(AVG(p.stock_quantity), 0), 2
|
||||
),
|
||||
0
|
||||
) as stockTurnoverRate,
|
||||
COALESCE(COUNT(DISTINCT p.vendor), 0) as vendorCount,
|
||||
COALESCE(COUNT(DISTINCT p.categories), 0) as categoryCount,
|
||||
COALESCE(
|
||||
ROUND(
|
||||
AVG(o.price * o.quantity), 2
|
||||
),
|
||||
0
|
||||
) as averageOrderValue
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
`);
|
||||
|
||||
// Ensure all values are numbers
|
||||
const stats = {
|
||||
profitMargin: Number(results[0].profitMargin) || 0,
|
||||
averageMarkup: Number(results[0].averageMarkup) || 0,
|
||||
stockTurnoverRate: Number(results[0].stockTurnoverRate) || 0,
|
||||
vendorCount: Number(results[0].vendorCount) || 0,
|
||||
categoryCount: Number(results[0].categoryCount) || 0,
|
||||
averageOrderValue: Number(results[0].averageOrderValue) || 0
|
||||
};
|
||||
|
||||
res.json(stats);
|
||||
} catch (error) {
|
||||
console.error('Error fetching analytics stats:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch analytics stats' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get profit analysis data
|
||||
router.get('/profit', async (req, res) => {
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
|
||||
// Get profit margins by category with full path
|
||||
const [byCategory] = await pool.query(`
|
||||
WITH RECURSIVE category_path AS (
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
c.parent_id,
|
||||
CAST(c.name AS CHAR(1000)) as path
|
||||
FROM categories c
|
||||
WHERE c.parent_id IS NULL
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
c.parent_id,
|
||||
CONCAT(cp.path, ' > ', c.name)
|
||||
FROM categories c
|
||||
JOIN category_path cp ON c.parent_id = cp.cat_id
|
||||
)
|
||||
SELECT
|
||||
c.name as category,
|
||||
cp.path as categoryPath,
|
||||
ROUND(
|
||||
(SUM(o.price * o.quantity - p.cost_price * o.quantity) /
|
||||
NULLIF(SUM(o.price * o.quantity), 0)) * 100, 1
|
||||
) as profitMargin,
|
||||
CAST(SUM(o.price * o.quantity) AS DECIMAL(15,3)) as revenue,
|
||||
CAST(SUM(p.cost_price * o.quantity) AS DECIMAL(15,3)) as cost
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
JOIN product_categories pc ON p.pid = pc.pid
|
||||
JOIN categories c ON pc.cat_id = c.cat_id
|
||||
JOIN category_path cp ON c.cat_id = cp.cat_id
|
||||
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
GROUP BY c.name, cp.path
|
||||
ORDER BY profitMargin DESC
|
||||
LIMIT 10
|
||||
`);
|
||||
|
||||
// Get profit margin trend over time
|
||||
const [overTime] = await pool.query(`
|
||||
SELECT
|
||||
formatted_date as date,
|
||||
ROUND(
|
||||
(SUM(o.price * o.quantity - p.cost_price * o.quantity) /
|
||||
NULLIF(SUM(o.price * o.quantity), 0)) * 100, 1
|
||||
) as profitMargin,
|
||||
CAST(SUM(o.price * o.quantity) AS DECIMAL(15,3)) as revenue,
|
||||
CAST(SUM(p.cost_price * o.quantity) AS DECIMAL(15,3)) as cost
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
CROSS JOIN (
|
||||
SELECT DATE_FORMAT(o.date, '%Y-%m-%d') as formatted_date
|
||||
FROM orders o
|
||||
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
GROUP BY DATE_FORMAT(o.date, '%Y-%m-%d')
|
||||
) dates
|
||||
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
AND DATE_FORMAT(o.date, '%Y-%m-%d') = dates.formatted_date
|
||||
GROUP BY formatted_date
|
||||
ORDER BY formatted_date
|
||||
`);
|
||||
|
||||
// Get top performing products with category paths
|
||||
const [topProducts] = await pool.query(`
|
||||
WITH RECURSIVE category_path AS (
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
c.parent_id,
|
||||
CAST(c.name AS CHAR(1000)) as path
|
||||
FROM categories c
|
||||
WHERE c.parent_id IS NULL
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
c.parent_id,
|
||||
CONCAT(cp.path, ' > ', c.name)
|
||||
FROM categories c
|
||||
JOIN category_path cp ON c.parent_id = cp.cat_id
|
||||
)
|
||||
SELECT
|
||||
p.title as product,
|
||||
c.name as category,
|
||||
cp.path as categoryPath,
|
||||
ROUND(
|
||||
(SUM(o.price * o.quantity - p.cost_price * o.quantity) /
|
||||
NULLIF(SUM(o.price * o.quantity), 0)) * 100, 1
|
||||
) as profitMargin,
|
||||
CAST(SUM(o.price * o.quantity) AS DECIMAL(15,3)) as revenue,
|
||||
CAST(SUM(p.cost_price * o.quantity) AS DECIMAL(15,3)) as cost
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
JOIN product_categories pc ON p.pid = pc.pid
|
||||
JOIN categories c ON pc.cat_id = c.cat_id
|
||||
JOIN category_path cp ON c.cat_id = cp.cat_id
|
||||
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
GROUP BY p.pid, p.title, c.name, cp.path
|
||||
HAVING revenue > 0
|
||||
ORDER BY profitMargin DESC
|
||||
LIMIT 10
|
||||
`);
|
||||
|
||||
res.json({ byCategory, overTime, topProducts });
|
||||
} catch (error) {
|
||||
console.error('Error fetching profit analysis:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch profit analysis' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get vendor performance data
|
||||
router.get('/vendors', async (req, res) => {
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
|
||||
console.log('Fetching vendor performance data...');
|
||||
|
||||
// First check if we have any vendors with sales
|
||||
const [checkData] = await pool.query(`
|
||||
SELECT COUNT(DISTINCT p.vendor) as vendor_count,
|
||||
COUNT(DISTINCT o.order_number) as order_count
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
WHERE p.vendor IS NOT NULL
|
||||
`);
|
||||
|
||||
console.log('Vendor data check:', checkData[0]);
|
||||
|
||||
// Get vendor performance metrics
|
||||
const [performance] = await pool.query(`
|
||||
WITH monthly_sales AS (
|
||||
SELECT
|
||||
p.vendor,
|
||||
CAST(SUM(CASE
|
||||
WHEN o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
THEN o.price * o.quantity
|
||||
ELSE 0
|
||||
END) AS DECIMAL(15,3)) as current_month,
|
||||
CAST(SUM(CASE
|
||||
WHEN o.date >= DATE_SUB(CURDATE(), INTERVAL 60 DAY)
|
||||
AND o.date < DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
THEN o.price * o.quantity
|
||||
ELSE 0
|
||||
END) AS DECIMAL(15,3)) as previous_month
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
WHERE p.vendor IS NOT NULL
|
||||
AND o.date >= DATE_SUB(CURDATE(), INTERVAL 60 DAY)
|
||||
GROUP BY p.vendor
|
||||
)
|
||||
SELECT
|
||||
p.vendor,
|
||||
CAST(SUM(o.price * o.quantity) AS DECIMAL(15,3)) as salesVolume,
|
||||
COALESCE(ROUND(
|
||||
(SUM(o.price * o.quantity - p.cost_price * o.quantity) /
|
||||
NULLIF(SUM(o.price * o.quantity), 0)) * 100, 1
|
||||
), 0) as profitMargin,
|
||||
COALESCE(ROUND(
|
||||
SUM(o.quantity) / NULLIF(AVG(p.stock_quantity), 0), 1
|
||||
), 0) as stockTurnover,
|
||||
COUNT(DISTINCT p.pid) as productCount,
|
||||
ROUND(
|
||||
((ms.current_month / NULLIF(ms.previous_month, 0)) - 1) * 100,
|
||||
1
|
||||
) as growth
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
LEFT JOIN monthly_sales ms ON p.vendor = ms.vendor
|
||||
WHERE p.vendor IS NOT NULL
|
||||
AND o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
GROUP BY p.vendor, ms.current_month, ms.previous_month
|
||||
ORDER BY salesVolume DESC
|
||||
LIMIT 10
|
||||
`);
|
||||
|
||||
console.log('Performance data:', performance);
|
||||
|
||||
// Get vendor comparison data
|
||||
const [comparison] = await pool.query(`
|
||||
SELECT
|
||||
p.vendor,
|
||||
CAST(COALESCE(ROUND(SUM(o.price * o.quantity) / NULLIF(COUNT(DISTINCT p.pid), 0), 2), 0) AS DECIMAL(15,3)) as salesPerProduct,
|
||||
COALESCE(ROUND(AVG((o.price - p.cost_price) / NULLIF(o.price, 0) * 100), 1), 0) as averageMargin,
|
||||
COUNT(DISTINCT p.pid) as size
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid AND o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
WHERE p.vendor IS NOT NULL
|
||||
GROUP BY p.vendor
|
||||
ORDER BY salesPerProduct DESC
|
||||
LIMIT 20
|
||||
`);
|
||||
|
||||
console.log('Comparison data:', comparison);
|
||||
|
||||
// Get vendor sales trends
|
||||
const [trends] = await pool.query(`
|
||||
SELECT
|
||||
p.vendor,
|
||||
DATE_FORMAT(o.date, '%b %Y') as month,
|
||||
CAST(COALESCE(SUM(o.price * o.quantity), 0) AS DECIMAL(15,3)) as sales
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
WHERE p.vendor IS NOT NULL
|
||||
AND o.date >= DATE_SUB(CURDATE(), INTERVAL 6 MONTH)
|
||||
GROUP BY
|
||||
p.vendor,
|
||||
DATE_FORMAT(o.date, '%b %Y'),
|
||||
DATE_FORMAT(o.date, '%Y-%m')
|
||||
ORDER BY
|
||||
p.vendor,
|
||||
DATE_FORMAT(o.date, '%Y-%m')
|
||||
`);
|
||||
|
||||
console.log('Trends data:', trends);
|
||||
|
||||
res.json({ performance, comparison, trends });
|
||||
} catch (error) {
|
||||
console.error('Error fetching vendor performance:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch vendor performance' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get stock analysis data
|
||||
router.get('/stock', async (req, res) => {
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
|
||||
// Get global configuration values
|
||||
const [configs] = await pool.query(`
|
||||
SELECT
|
||||
st.low_stock_threshold,
|
||||
tc.calculation_period_days as turnover_period
|
||||
FROM stock_thresholds st
|
||||
CROSS JOIN turnover_config tc
|
||||
WHERE st.id = 1 AND tc.id = 1
|
||||
`);
|
||||
|
||||
const config = configs[0] || {
|
||||
low_stock_threshold: 5,
|
||||
turnover_period: 30
|
||||
};
|
||||
|
||||
// Get turnover by category
|
||||
const [turnoverByCategory] = await pool.query(`
|
||||
SELECT
|
||||
c.name as category,
|
||||
ROUND(SUM(o.quantity) / NULLIF(AVG(p.stock_quantity), 0), 1) as turnoverRate,
|
||||
ROUND(AVG(p.stock_quantity), 0) as averageStock,
|
||||
SUM(o.quantity) as totalSales
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
JOIN product_categories pc ON p.pid = pc.pid
|
||||
JOIN categories c ON pc.cat_id = c.cat_id
|
||||
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL ? DAY)
|
||||
GROUP BY c.name
|
||||
HAVING turnoverRate > 0
|
||||
ORDER BY turnoverRate DESC
|
||||
LIMIT 10
|
||||
`, [config.turnover_period]);
|
||||
|
||||
// Get stock levels over time
|
||||
const [stockLevels] = await pool.query(`
|
||||
SELECT
|
||||
DATE_FORMAT(o.date, '%Y-%m-%d') as date,
|
||||
SUM(CASE WHEN p.stock_quantity > ? THEN 1 ELSE 0 END) as inStock,
|
||||
SUM(CASE WHEN p.stock_quantity <= ? AND p.stock_quantity > 0 THEN 1 ELSE 0 END) as lowStock,
|
||||
SUM(CASE WHEN p.stock_quantity = 0 THEN 1 ELSE 0 END) as outOfStock
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL ? DAY)
|
||||
GROUP BY DATE_FORMAT(o.date, '%Y-%m-%d')
|
||||
ORDER BY date
|
||||
`, [
|
||||
config.low_stock_threshold,
|
||||
config.low_stock_threshold,
|
||||
config.turnover_period
|
||||
]);
|
||||
|
||||
// Get critical stock items
|
||||
const [criticalItems] = await pool.query(`
|
||||
WITH product_thresholds AS (
|
||||
SELECT
|
||||
p.pid,
|
||||
COALESCE(
|
||||
(SELECT reorder_days
|
||||
FROM stock_thresholds st
|
||||
WHERE st.vendor = p.vendor LIMIT 1),
|
||||
(SELECT reorder_days
|
||||
FROM stock_thresholds st
|
||||
WHERE st.vendor IS NULL LIMIT 1),
|
||||
14
|
||||
) as reorder_days
|
||||
FROM products p
|
||||
)
|
||||
SELECT
|
||||
p.title as product,
|
||||
p.SKU as sku,
|
||||
p.stock_quantity as stockQuantity,
|
||||
GREATEST(ROUND(AVG(o.quantity) * pt.reorder_days), ?) as reorderPoint,
|
||||
ROUND(SUM(o.quantity) / NULLIF(p.stock_quantity, 0), 1) as turnoverRate,
|
||||
CASE
|
||||
WHEN p.stock_quantity = 0 THEN 0
|
||||
ELSE ROUND(p.stock_quantity / NULLIF((SUM(o.quantity) / ?), 0))
|
||||
END as daysUntilStockout
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
JOIN product_thresholds pt ON p.pid = pt.pid
|
||||
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL ? DAY)
|
||||
AND p.managing_stock = true
|
||||
GROUP BY p.pid
|
||||
HAVING daysUntilStockout < ? AND daysUntilStockout >= 0
|
||||
ORDER BY daysUntilStockout
|
||||
LIMIT 10
|
||||
`, [
|
||||
config.low_stock_threshold,
|
||||
config.turnover_period,
|
||||
config.turnover_period,
|
||||
config.turnover_period
|
||||
]);
|
||||
|
||||
res.json({ turnoverByCategory, stockLevels, criticalItems });
|
||||
} catch (error) {
|
||||
console.error('Error fetching stock analysis:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch stock analysis' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get price analysis data
|
||||
router.get('/pricing', async (req, res) => {
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
|
||||
// Get price points analysis
|
||||
const [pricePoints] = await pool.query(`
|
||||
SELECT
|
||||
CAST(p.price AS DECIMAL(15,3)) as price,
|
||||
CAST(SUM(o.quantity) AS DECIMAL(15,3)) as salesVolume,
|
||||
CAST(SUM(o.price * o.quantity) AS DECIMAL(15,3)) as revenue,
|
||||
c.name as category
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
JOIN product_categories pc ON p.pid = pc.pid
|
||||
JOIN categories c ON pc.cat_id = c.cat_id
|
||||
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
GROUP BY p.price, c.name
|
||||
HAVING salesVolume > 0
|
||||
ORDER BY revenue DESC
|
||||
LIMIT 50
|
||||
`);
|
||||
|
||||
// Get price elasticity data (price changes vs demand)
|
||||
const [elasticity] = await pool.query(`
|
||||
SELECT
|
||||
DATE_FORMAT(o.date, '%Y-%m-%d') as date,
|
||||
CAST(AVG(o.price) AS DECIMAL(15,3)) as price,
|
||||
CAST(SUM(o.quantity) AS DECIMAL(15,3)) as demand
|
||||
FROM orders o
|
||||
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
GROUP BY DATE_FORMAT(o.date, '%Y-%m-%d')
|
||||
ORDER BY date
|
||||
`);
|
||||
|
||||
// Get price optimization recommendations
|
||||
const [recommendations] = await pool.query(`
|
||||
SELECT
|
||||
p.title as product,
|
||||
CAST(p.price AS DECIMAL(15,3)) as currentPrice,
|
||||
CAST(
|
||||
ROUND(
|
||||
CASE
|
||||
WHEN AVG(o.quantity) > 10 THEN p.price * 1.1
|
||||
WHEN AVG(o.quantity) < 2 THEN p.price * 0.9
|
||||
ELSE p.price
|
||||
END, 2
|
||||
) AS DECIMAL(15,3)
|
||||
) as recommendedPrice,
|
||||
CAST(
|
||||
ROUND(
|
||||
SUM(o.price * o.quantity) *
|
||||
CASE
|
||||
WHEN AVG(o.quantity) > 10 THEN 1.15
|
||||
WHEN AVG(o.quantity) < 2 THEN 0.95
|
||||
ELSE 1
|
||||
END, 2
|
||||
) AS DECIMAL(15,3)
|
||||
) as potentialRevenue,
|
||||
CASE
|
||||
WHEN AVG(o.quantity) > 10 THEN 85
|
||||
WHEN AVG(o.quantity) < 2 THEN 75
|
||||
ELSE 65
|
||||
END as confidence
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
GROUP BY p.pid, p.price
|
||||
HAVING ABS(recommendedPrice - currentPrice) > 0
|
||||
ORDER BY potentialRevenue - CAST(SUM(o.price * o.quantity) AS DECIMAL(15,3)) DESC
|
||||
LIMIT 10
|
||||
`);
|
||||
|
||||
res.json({ pricePoints, elasticity, recommendations });
|
||||
} catch (error) {
|
||||
console.error('Error fetching price analysis:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch price analysis' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get category performance data
|
||||
router.get('/categories', async (req, res) => {
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
|
||||
// Common CTE for category paths
|
||||
const categoryPathCTE = `
|
||||
WITH RECURSIVE category_path AS (
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
c.parent_id,
|
||||
CAST(c.name AS CHAR(1000)) as path
|
||||
FROM categories c
|
||||
WHERE c.parent_id IS NULL
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
c.parent_id,
|
||||
CONCAT(cp.path, ' > ', c.name)
|
||||
FROM categories c
|
||||
JOIN category_path cp ON c.parent_id = cp.cat_id
|
||||
)
|
||||
`;
|
||||
|
||||
// Get category performance metrics with full path
|
||||
const [performance] = await pool.query(`
|
||||
${categoryPathCTE},
|
||||
monthly_sales AS (
|
||||
SELECT
|
||||
c.name,
|
||||
cp.path,
|
||||
SUM(CASE
|
||||
WHEN o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
THEN o.price * o.quantity
|
||||
ELSE 0
|
||||
END) as current_month,
|
||||
SUM(CASE
|
||||
WHEN o.date >= DATE_SUB(CURDATE(), INTERVAL 60 DAY)
|
||||
AND o.date < DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
THEN o.price * o.quantity
|
||||
ELSE 0
|
||||
END) as previous_month
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
JOIN product_categories pc ON p.pid = pc.pid
|
||||
JOIN categories c ON pc.cat_id = c.cat_id
|
||||
JOIN category_path cp ON c.cat_id = cp.cat_id
|
||||
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 60 DAY)
|
||||
GROUP BY c.name, cp.path
|
||||
)
|
||||
SELECT
|
||||
c.name as category,
|
||||
cp.path as categoryPath,
|
||||
SUM(o.price * o.quantity) as revenue,
|
||||
SUM(o.price * o.quantity - p.cost_price * o.quantity) as profit,
|
||||
ROUND(
|
||||
((ms.current_month / NULLIF(ms.previous_month, 0)) - 1) * 100,
|
||||
1
|
||||
) as growth,
|
||||
COUNT(DISTINCT p.pid) as productCount
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
JOIN product_categories pc ON p.pid = pc.pid
|
||||
JOIN categories c ON pc.cat_id = c.cat_id
|
||||
JOIN category_path cp ON c.cat_id = cp.cat_id
|
||||
LEFT JOIN monthly_sales ms ON c.name = ms.name AND cp.path = ms.path
|
||||
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 60 DAY)
|
||||
GROUP BY c.name, cp.path, ms.current_month, ms.previous_month
|
||||
HAVING revenue > 0
|
||||
ORDER BY revenue DESC
|
||||
LIMIT 10
|
||||
`);
|
||||
|
||||
// Get category revenue distribution with full path
|
||||
const [distribution] = await pool.query(`
|
||||
${categoryPathCTE}
|
||||
SELECT
|
||||
c.name as category,
|
||||
cp.path as categoryPath,
|
||||
SUM(o.price * o.quantity) as value
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
JOIN product_categories pc ON p.pid = pc.pid
|
||||
JOIN categories c ON pc.cat_id = c.cat_id
|
||||
JOIN category_path cp ON c.cat_id = cp.cat_id
|
||||
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
GROUP BY c.name, cp.path
|
||||
HAVING value > 0
|
||||
ORDER BY value DESC
|
||||
LIMIT 6
|
||||
`);
|
||||
|
||||
// Get category sales trends with full path
|
||||
const [trends] = await pool.query(`
|
||||
${categoryPathCTE}
|
||||
SELECT
|
||||
c.name as category,
|
||||
cp.path as categoryPath,
|
||||
DATE_FORMAT(o.date, '%b %Y') as month,
|
||||
SUM(o.price * o.quantity) as sales
|
||||
FROM products p
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
JOIN product_categories pc ON p.pid = pc.pid
|
||||
JOIN categories c ON pc.cat_id = c.cat_id
|
||||
JOIN category_path cp ON c.cat_id = cp.cat_id
|
||||
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 6 MONTH)
|
||||
GROUP BY
|
||||
c.name,
|
||||
cp.path,
|
||||
DATE_FORMAT(o.date, '%b %Y'),
|
||||
DATE_FORMAT(o.date, '%Y-%m')
|
||||
ORDER BY
|
||||
c.name,
|
||||
DATE_FORMAT(o.date, '%Y-%m')
|
||||
`);
|
||||
|
||||
res.json({ performance, distribution, trends });
|
||||
} catch (error) {
|
||||
console.error('Error fetching category performance:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch category performance' });
|
||||
}
|
||||
});
|
||||
|
||||
// Forecast endpoint
|
||||
router.get('/forecast', async (req, res) => {
|
||||
try {
|
||||
const { brand, startDate, endDate } = req.query;
|
||||
const pool = req.app.locals.pool;
|
||||
|
||||
const [results] = await pool.query(`
|
||||
WITH RECURSIVE category_path AS (
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
c.parent_id,
|
||||
CAST(c.name AS CHAR(1000)) as path
|
||||
FROM categories c
|
||||
WHERE c.parent_id IS NULL
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
c.parent_id,
|
||||
CONCAT(cp.path, ' > ', c.name)
|
||||
FROM categories c
|
||||
JOIN category_path cp ON c.parent_id = cp.cat_id
|
||||
),
|
||||
category_metrics AS (
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name as category_name,
|
||||
cp.path,
|
||||
p.brand,
|
||||
COUNT(DISTINCT p.pid) as num_products,
|
||||
CAST(COALESCE(ROUND(SUM(o.quantity) / DATEDIFF(?, ?), 2), 0) AS DECIMAL(15,3)) as avg_daily_sales,
|
||||
COALESCE(SUM(o.quantity), 0) as total_sold,
|
||||
CAST(COALESCE(ROUND(SUM(o.quantity) / COUNT(DISTINCT p.pid), 2), 0) AS DECIMAL(15,3)) as avgTotalSold,
|
||||
CAST(COALESCE(ROUND(AVG(o.price), 2), 0) AS DECIMAL(15,3)) as avg_price
|
||||
FROM categories c
|
||||
JOIN product_categories pc ON c.cat_id = pc.cat_id
|
||||
JOIN products p ON pc.pid = p.pid
|
||||
JOIN category_path cp ON c.cat_id = cp.cat_id
|
||||
LEFT JOIN product_metrics pmet ON p.pid = pmet.pid
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
AND o.date BETWEEN ? AND ?
|
||||
AND o.canceled = false
|
||||
WHERE p.brand = ?
|
||||
AND pmet.first_received_date BETWEEN ? AND ?
|
||||
GROUP BY c.cat_id, c.name, cp.path, p.brand
|
||||
),
|
||||
product_details AS (
|
||||
SELECT
|
||||
p.pid,
|
||||
p.title,
|
||||
p.SKU,
|
||||
p.stock_quantity,
|
||||
pc.cat_id,
|
||||
pmet.first_received_date,
|
||||
COALESCE(SUM(o.quantity), 0) as total_sold,
|
||||
CAST(COALESCE(ROUND(AVG(o.price), 2), 0) AS DECIMAL(15,3)) as avg_price
|
||||
FROM products p
|
||||
JOIN product_categories pc ON p.pid = pc.pid
|
||||
JOIN product_metrics pmet ON p.pid = pmet.pid
|
||||
LEFT JOIN orders o ON p.pid = o.pid
|
||||
AND o.date BETWEEN ? AND ?
|
||||
AND o.canceled = false
|
||||
WHERE p.brand = ?
|
||||
AND pmet.first_received_date BETWEEN ? AND ?
|
||||
GROUP BY p.pid, p.title, p.SKU, p.stock_quantity, pc.cat_id, pmet.first_received_date
|
||||
)
|
||||
SELECT
|
||||
cm.*,
|
||||
JSON_ARRAYAGG(
|
||||
JSON_OBJECT(
|
||||
'pid', pd.pid,
|
||||
'title', pd.title,
|
||||
'SKU', pd.SKU,
|
||||
'stock_quantity', pd.stock_quantity,
|
||||
'total_sold', pd.total_sold,
|
||||
'avg_price', pd.avg_price,
|
||||
'first_received_date', DATE_FORMAT(pd.first_received_date, '%Y-%m-%d')
|
||||
)
|
||||
) as products
|
||||
FROM category_metrics cm
|
||||
JOIN product_details pd ON cm.cat_id = pd.cat_id
|
||||
GROUP BY cm.cat_id, cm.category_name, cm.path, cm.brand, cm.num_products, cm.avg_daily_sales, cm.total_sold, cm.avgTotalSold, cm.avg_price
|
||||
ORDER BY cm.total_sold DESC
|
||||
`, [endDate, startDate, startDate, endDate, brand, startDate, endDate, startDate, endDate, brand, startDate, endDate]);
|
||||
|
||||
res.json(results);
|
||||
} catch (error) {
|
||||
console.error('Error fetching forecast data:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch forecast data' });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
1
inventory-server/src/routes/brandsAggregate.js
Normal file
1
inventory-server/src/routes/brandsAggregate.js
Normal file
@@ -0,0 +1 @@
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
|
||||
// Get all categories
|
||||
router.get('/', async (req, res) => {
|
||||
const pool = req.app.locals.pool;
|
||||
try {
|
||||
// Get all categories with metrics and hierarchy info
|
||||
const [categories] = await pool.query(`
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
c.type,
|
||||
c.parent_id,
|
||||
c.description,
|
||||
c.status,
|
||||
p.name as parent_name,
|
||||
p.type as parent_type,
|
||||
COALESCE(cm.product_count, 0) as product_count,
|
||||
COALESCE(cm.active_products, 0) as active_products,
|
||||
CAST(COALESCE(cm.total_value, 0) AS DECIMAL(15,3)) as total_value,
|
||||
COALESCE(cm.avg_margin, 0) as avg_margin,
|
||||
COALESCE(cm.turnover_rate, 0) as turnover_rate,
|
||||
COALESCE(cm.growth_rate, 0) as growth_rate
|
||||
FROM categories c
|
||||
LEFT JOIN categories p ON c.parent_id = p.cat_id
|
||||
LEFT JOIN category_metrics cm ON c.cat_id = cm.category_id
|
||||
ORDER BY
|
||||
CASE
|
||||
WHEN c.type = 10 THEN 1 -- sections first
|
||||
WHEN c.type = 11 THEN 2 -- categories second
|
||||
WHEN c.type = 12 THEN 3 -- subcategories third
|
||||
WHEN c.type = 13 THEN 4 -- subsubcategories fourth
|
||||
WHEN c.type = 20 THEN 5 -- themes fifth
|
||||
WHEN c.type = 21 THEN 6 -- subthemes last
|
||||
ELSE 7
|
||||
END,
|
||||
c.name ASC
|
||||
`);
|
||||
|
||||
// Get overall stats
|
||||
const [stats] = await pool.query(`
|
||||
SELECT
|
||||
COUNT(DISTINCT c.cat_id) as totalCategories,
|
||||
COUNT(DISTINCT CASE WHEN c.status = 'active' THEN c.cat_id END) as activeCategories,
|
||||
CAST(COALESCE(SUM(cm.total_value), 0) AS DECIMAL(15,3)) as totalValue,
|
||||
COALESCE(ROUND(AVG(NULLIF(cm.avg_margin, 0)), 1), 0) as avgMargin,
|
||||
COALESCE(ROUND(AVG(NULLIF(cm.growth_rate, 0)), 1), 0) as avgGrowth
|
||||
FROM categories c
|
||||
LEFT JOIN category_metrics cm ON c.cat_id = cm.category_id
|
||||
`);
|
||||
|
||||
// Get type counts for filtering
|
||||
const [typeCounts] = await pool.query(`
|
||||
SELECT
|
||||
type,
|
||||
COUNT(*) as count
|
||||
FROM categories
|
||||
GROUP BY type
|
||||
ORDER BY type
|
||||
`);
|
||||
|
||||
res.json({
|
||||
categories: categories.map(cat => ({
|
||||
cat_id: cat.cat_id,
|
||||
name: cat.name,
|
||||
type: cat.type,
|
||||
parent_id: cat.parent_id,
|
||||
parent_name: cat.parent_name,
|
||||
parent_type: cat.parent_type,
|
||||
description: cat.description,
|
||||
status: cat.status,
|
||||
metrics: {
|
||||
product_count: parseInt(cat.product_count),
|
||||
active_products: parseInt(cat.active_products),
|
||||
total_value: parseFloat(cat.total_value),
|
||||
avg_margin: parseFloat(cat.avg_margin),
|
||||
turnover_rate: parseFloat(cat.turnover_rate),
|
||||
growth_rate: parseFloat(cat.growth_rate)
|
||||
}
|
||||
})),
|
||||
typeCounts: typeCounts.map(tc => ({
|
||||
type: tc.type,
|
||||
count: parseInt(tc.count)
|
||||
})),
|
||||
stats: {
|
||||
totalCategories: parseInt(stats[0].totalCategories),
|
||||
activeCategories: parseInt(stats[0].activeCategories),
|
||||
totalValue: parseFloat(stats[0].totalValue),
|
||||
avgMargin: parseFloat(stats[0].avgMargin),
|
||||
avgGrowth: parseFloat(stats[0].avgGrowth)
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error fetching categories:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch categories' });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -1,172 +0,0 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
|
||||
// Debug middleware
|
||||
router.use((req, res, next) => {
|
||||
console.log(`[Config Route] ${req.method} ${req.path}`);
|
||||
next();
|
||||
});
|
||||
|
||||
// Get all configuration values
|
||||
router.get('/', async (req, res) => {
|
||||
const pool = req.app.locals.pool;
|
||||
try {
|
||||
console.log('[Config Route] Fetching configuration values...');
|
||||
|
||||
const [stockThresholds] = await pool.query('SELECT * FROM stock_thresholds WHERE id = 1');
|
||||
console.log('[Config Route] Stock thresholds:', stockThresholds);
|
||||
|
||||
const [leadTimeThresholds] = await pool.query('SELECT * FROM lead_time_thresholds WHERE id = 1');
|
||||
console.log('[Config Route] Lead time thresholds:', leadTimeThresholds);
|
||||
|
||||
const [salesVelocityConfig] = await pool.query('SELECT * FROM sales_velocity_config WHERE id = 1');
|
||||
console.log('[Config Route] Sales velocity config:', salesVelocityConfig);
|
||||
|
||||
const [abcConfig] = await pool.query('SELECT * FROM abc_classification_config WHERE id = 1');
|
||||
console.log('[Config Route] ABC config:', abcConfig);
|
||||
|
||||
const [safetyStockConfig] = await pool.query('SELECT * FROM safety_stock_config WHERE id = 1');
|
||||
console.log('[Config Route] Safety stock config:', safetyStockConfig);
|
||||
|
||||
const [turnoverConfig] = await pool.query('SELECT * FROM turnover_config WHERE id = 1');
|
||||
console.log('[Config Route] Turnover config:', turnoverConfig);
|
||||
|
||||
const response = {
|
||||
stockThresholds: stockThresholds[0],
|
||||
leadTimeThresholds: leadTimeThresholds[0],
|
||||
salesVelocityConfig: salesVelocityConfig[0],
|
||||
abcConfig: abcConfig[0],
|
||||
safetyStockConfig: safetyStockConfig[0],
|
||||
turnoverConfig: turnoverConfig[0]
|
||||
};
|
||||
|
||||
console.log('[Config Route] Sending response:', response);
|
||||
res.json(response);
|
||||
} catch (error) {
|
||||
console.error('[Config Route] Error fetching configuration:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch configuration', details: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Update stock thresholds
|
||||
router.put('/stock-thresholds/:id', async (req, res) => {
|
||||
const pool = req.app.locals.pool;
|
||||
try {
|
||||
const { critical_days, reorder_days, overstock_days, low_stock_threshold, min_reorder_quantity } = req.body;
|
||||
const [result] = await pool.query(
|
||||
`UPDATE stock_thresholds
|
||||
SET critical_days = ?,
|
||||
reorder_days = ?,
|
||||
overstock_days = ?,
|
||||
low_stock_threshold = ?,
|
||||
min_reorder_quantity = ?
|
||||
WHERE id = ?`,
|
||||
[critical_days, reorder_days, overstock_days, low_stock_threshold, min_reorder_quantity, req.params.id]
|
||||
);
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('[Config Route] Error updating stock thresholds:', error);
|
||||
res.status(500).json({ error: 'Failed to update stock thresholds' });
|
||||
}
|
||||
});
|
||||
|
||||
// Update lead time thresholds
|
||||
router.put('/lead-time-thresholds/:id', async (req, res) => {
|
||||
const pool = req.app.locals.pool;
|
||||
try {
|
||||
const { target_days, warning_days, critical_days } = req.body;
|
||||
const [result] = await pool.query(
|
||||
`UPDATE lead_time_thresholds
|
||||
SET target_days = ?,
|
||||
warning_days = ?,
|
||||
critical_days = ?
|
||||
WHERE id = ?`,
|
||||
[target_days, warning_days, critical_days, req.params.id]
|
||||
);
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('[Config Route] Error updating lead time thresholds:', error);
|
||||
res.status(500).json({ error: 'Failed to update lead time thresholds' });
|
||||
}
|
||||
});
|
||||
|
||||
// Update sales velocity config
|
||||
router.put('/sales-velocity/:id', async (req, res) => {
|
||||
const pool = req.app.locals.pool;
|
||||
try {
|
||||
const { daily_window_days, weekly_window_days, monthly_window_days } = req.body;
|
||||
const [result] = await pool.query(
|
||||
`UPDATE sales_velocity_config
|
||||
SET daily_window_days = ?,
|
||||
weekly_window_days = ?,
|
||||
monthly_window_days = ?
|
||||
WHERE id = ?`,
|
||||
[daily_window_days, weekly_window_days, monthly_window_days, req.params.id]
|
||||
);
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('[Config Route] Error updating sales velocity config:', error);
|
||||
res.status(500).json({ error: 'Failed to update sales velocity config' });
|
||||
}
|
||||
});
|
||||
|
||||
// Update ABC classification config
|
||||
router.put('/abc-classification/:id', async (req, res) => {
|
||||
const pool = req.app.locals.pool;
|
||||
try {
|
||||
const { a_threshold, b_threshold, classification_period_days } = req.body;
|
||||
const [result] = await pool.query(
|
||||
`UPDATE abc_classification_config
|
||||
SET a_threshold = ?,
|
||||
b_threshold = ?,
|
||||
classification_period_days = ?
|
||||
WHERE id = ?`,
|
||||
[a_threshold, b_threshold, classification_period_days, req.params.id]
|
||||
);
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('[Config Route] Error updating ABC classification config:', error);
|
||||
res.status(500).json({ error: 'Failed to update ABC classification config' });
|
||||
}
|
||||
});
|
||||
|
||||
// Update safety stock config
|
||||
router.put('/safety-stock/:id', async (req, res) => {
|
||||
const pool = req.app.locals.pool;
|
||||
try {
|
||||
const { coverage_days, service_level } = req.body;
|
||||
const [result] = await pool.query(
|
||||
`UPDATE safety_stock_config
|
||||
SET coverage_days = ?,
|
||||
service_level = ?
|
||||
WHERE id = ?`,
|
||||
[coverage_days, service_level, req.params.id]
|
||||
);
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('[Config Route] Error updating safety stock config:', error);
|
||||
res.status(500).json({ error: 'Failed to update safety stock config' });
|
||||
}
|
||||
});
|
||||
|
||||
// Update turnover config
|
||||
router.put('/turnover/:id', async (req, res) => {
|
||||
const pool = req.app.locals.pool;
|
||||
try {
|
||||
const { calculation_period_days, target_rate } = req.body;
|
||||
const [result] = await pool.query(
|
||||
`UPDATE turnover_config
|
||||
SET calculation_period_days = ?,
|
||||
target_rate = ?
|
||||
WHERE id = ?`,
|
||||
[calculation_period_days, target_rate, req.params.id]
|
||||
);
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('[Config Route] Error updating turnover config:', error);
|
||||
res.status(500).json({ error: 'Failed to update turnover config' });
|
||||
}
|
||||
});
|
||||
|
||||
// Export the router
|
||||
module.exports = router;
|
||||
@@ -1,714 +0,0 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const { spawn } = require('child_process');
|
||||
const path = require('path');
|
||||
|
||||
// Debug middleware MUST be first
|
||||
router.use((req, res, next) => {
|
||||
console.log(`[CSV Route Debug] ${req.method} ${req.path}`);
|
||||
next();
|
||||
});
|
||||
|
||||
// Store active import process and its progress
|
||||
let activeImport = null;
|
||||
let importProgress = null;
|
||||
|
||||
// SSE clients for progress updates
|
||||
const updateClients = new Set();
|
||||
const importClients = new Set();
|
||||
const resetClients = new Set();
|
||||
const resetMetricsClients = new Set();
|
||||
const calculateMetricsClients = new Set();
|
||||
|
||||
// Helper to send progress to specific clients
|
||||
function sendProgressToClients(clients, progress) {
|
||||
const data = typeof progress === 'string' ? { progress } : progress;
|
||||
|
||||
// Ensure we have a status field
|
||||
if (!data.status) {
|
||||
data.status = 'running';
|
||||
}
|
||||
|
||||
const message = `data: ${JSON.stringify(data)}\n\n`;
|
||||
|
||||
clients.forEach(client => {
|
||||
try {
|
||||
client.write(message);
|
||||
// Immediately flush the response
|
||||
if (typeof client.flush === 'function') {
|
||||
client.flush();
|
||||
}
|
||||
} catch (error) {
|
||||
// Silently remove failed client
|
||||
clients.delete(client);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Progress endpoints
|
||||
router.get('/update/progress', (req, res) => {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
'Access-Control-Allow-Origin': req.headers.origin || '*',
|
||||
'Access-Control-Allow-Credentials': 'true'
|
||||
});
|
||||
|
||||
// Send an initial message to test the connection
|
||||
res.write('data: {"status":"running","operation":"Initializing connection..."}\n\n');
|
||||
|
||||
// Add this client to the update set
|
||||
updateClients.add(res);
|
||||
|
||||
// Remove client when connection closes
|
||||
req.on('close', () => {
|
||||
updateClients.delete(res);
|
||||
});
|
||||
});
|
||||
|
||||
router.get('/import/progress', (req, res) => {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
'Access-Control-Allow-Origin': req.headers.origin || '*',
|
||||
'Access-Control-Allow-Credentials': 'true'
|
||||
});
|
||||
|
||||
// Send an initial message to test the connection
|
||||
res.write('data: {"status":"running","operation":"Initializing connection..."}\n\n');
|
||||
|
||||
// Add this client to the import set
|
||||
importClients.add(res);
|
||||
|
||||
// Remove client when connection closes
|
||||
req.on('close', () => {
|
||||
importClients.delete(res);
|
||||
});
|
||||
});
|
||||
|
||||
router.get('/reset/progress', (req, res) => {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
'Access-Control-Allow-Origin': req.headers.origin || '*',
|
||||
'Access-Control-Allow-Credentials': 'true'
|
||||
});
|
||||
|
||||
// Send an initial message to test the connection
|
||||
res.write('data: {"status":"running","operation":"Initializing connection..."}\n\n');
|
||||
|
||||
// Add this client to the reset set
|
||||
resetClients.add(res);
|
||||
|
||||
// Remove client when connection closes
|
||||
req.on('close', () => {
|
||||
resetClients.delete(res);
|
||||
});
|
||||
});
|
||||
|
||||
// Add reset-metrics progress endpoint
|
||||
router.get('/reset-metrics/progress', (req, res) => {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
'Access-Control-Allow-Origin': req.headers.origin || '*',
|
||||
'Access-Control-Allow-Credentials': 'true'
|
||||
});
|
||||
|
||||
// Send an initial message to test the connection
|
||||
res.write('data: {"status":"running","operation":"Initializing connection..."}\n\n');
|
||||
|
||||
// Add this client to the reset-metrics set
|
||||
resetMetricsClients.add(res);
|
||||
|
||||
// Remove client when connection closes
|
||||
req.on('close', () => {
|
||||
resetMetricsClients.delete(res);
|
||||
});
|
||||
});
|
||||
|
||||
// Add calculate-metrics progress endpoint
|
||||
router.get('/calculate-metrics/progress', (req, res) => {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
'Access-Control-Allow-Origin': req.headers.origin || '*',
|
||||
'Access-Control-Allow-Credentials': 'true'
|
||||
});
|
||||
|
||||
// Send current progress if it exists
|
||||
if (importProgress) {
|
||||
res.write(`data: ${JSON.stringify(importProgress)}\n\n`);
|
||||
} else {
|
||||
res.write('data: {"status":"running","operation":"Initializing connection..."}\n\n');
|
||||
}
|
||||
|
||||
// Add this client to the calculate-metrics set
|
||||
calculateMetricsClients.add(res);
|
||||
|
||||
// Remove client when connection closes
|
||||
req.on('close', () => {
|
||||
calculateMetricsClients.delete(res);
|
||||
});
|
||||
});
|
||||
|
||||
// Debug endpoint to verify route registration
|
||||
router.get('/test', (req, res) => {
|
||||
console.log('CSV test endpoint hit');
|
||||
res.json({ message: 'CSV routes are working' });
|
||||
});
|
||||
|
||||
// Route to check import status
|
||||
router.get('/status', (req, res) => {
|
||||
console.log('CSV status endpoint hit');
|
||||
res.json({
|
||||
active: !!activeImport,
|
||||
progress: importProgress
|
||||
});
|
||||
});
|
||||
|
||||
// Add calculate-metrics status endpoint
|
||||
router.get('/calculate-metrics/status', (req, res) => {
|
||||
console.log('Calculate metrics status endpoint hit');
|
||||
const calculateMetrics = require('../../scripts/calculate-metrics');
|
||||
const progress = calculateMetrics.getProgress();
|
||||
|
||||
// Only consider it active if both the process is running and we have progress
|
||||
const isActive = !!activeImport && !!progress;
|
||||
|
||||
res.json({
|
||||
active: isActive,
|
||||
progress: isActive ? progress : null
|
||||
});
|
||||
});
|
||||
|
||||
// Route to update CSV files
|
||||
router.post('/update', async (req, res, next) => {
|
||||
if (activeImport) {
|
||||
return res.status(409).json({ error: 'Import already in progress' });
|
||||
}
|
||||
|
||||
try {
|
||||
const scriptPath = path.join(__dirname, '..', '..', 'scripts', 'update-csv.js');
|
||||
|
||||
if (!require('fs').existsSync(scriptPath)) {
|
||||
return res.status(500).json({ error: 'Update script not found' });
|
||||
}
|
||||
|
||||
activeImport = spawn('node', [scriptPath]);
|
||||
|
||||
activeImport.stdout.on('data', (data) => {
|
||||
const output = data.toString().trim();
|
||||
|
||||
try {
|
||||
// Try to parse as JSON
|
||||
const jsonData = JSON.parse(output);
|
||||
sendProgressToClients(updateClients, {
|
||||
status: 'running',
|
||||
...jsonData
|
||||
});
|
||||
} catch (e) {
|
||||
// If not JSON, send as plain progress
|
||||
sendProgressToClients(updateClients, {
|
||||
status: 'running',
|
||||
progress: output
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
activeImport.stderr.on('data', (data) => {
|
||||
const error = data.toString().trim();
|
||||
try {
|
||||
// Try to parse as JSON
|
||||
const jsonData = JSON.parse(error);
|
||||
sendProgressToClients(updateClients, {
|
||||
status: 'error',
|
||||
...jsonData
|
||||
});
|
||||
} catch {
|
||||
sendProgressToClients(updateClients, {
|
||||
status: 'error',
|
||||
error
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
activeImport.on('close', (code) => {
|
||||
// Don't treat cancellation (code 143/SIGTERM) as an error
|
||||
if (code === 0 || code === 143) {
|
||||
sendProgressToClients(updateClients, {
|
||||
status: 'complete',
|
||||
operation: code === 143 ? 'Operation cancelled' : 'Update complete'
|
||||
});
|
||||
resolve();
|
||||
} else {
|
||||
const errorMsg = `Update process exited with code ${code}`;
|
||||
sendProgressToClients(updateClients, {
|
||||
status: 'error',
|
||||
error: errorMsg
|
||||
});
|
||||
reject(new Error(errorMsg));
|
||||
}
|
||||
activeImport = null;
|
||||
importProgress = null;
|
||||
});
|
||||
});
|
||||
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('Error updating CSV files:', error);
|
||||
activeImport = null;
|
||||
importProgress = null;
|
||||
sendProgressToClients(updateClients, {
|
||||
status: 'error',
|
||||
error: error.message
|
||||
});
|
||||
next(error);
|
||||
}
|
||||
});
|
||||
|
||||
// Route to import CSV files
|
||||
router.post('/import', async (req, res) => {
|
||||
if (activeImport) {
|
||||
return res.status(409).json({ error: 'Import already in progress' });
|
||||
}
|
||||
|
||||
try {
|
||||
const scriptPath = path.join(__dirname, '..', '..', 'scripts', 'import-csv.js');
|
||||
|
||||
if (!require('fs').existsSync(scriptPath)) {
|
||||
return res.status(500).json({ error: 'Import script not found' });
|
||||
}
|
||||
|
||||
// Get test limits from request body
|
||||
const { products = 0, orders = 10000, purchaseOrders = 10000 } = req.body;
|
||||
|
||||
// Create environment variables for the script
|
||||
const env = {
|
||||
...process.env,
|
||||
PRODUCTS_TEST_LIMIT: products.toString(),
|
||||
ORDERS_TEST_LIMIT: orders.toString(),
|
||||
PURCHASE_ORDERS_TEST_LIMIT: purchaseOrders.toString()
|
||||
};
|
||||
|
||||
activeImport = spawn('node', [scriptPath], { env });
|
||||
|
||||
activeImport.stdout.on('data', (data) => {
|
||||
const output = data.toString().trim();
|
||||
|
||||
try {
|
||||
// Try to parse as JSON
|
||||
const jsonData = JSON.parse(output);
|
||||
sendProgressToClients(importClients, {
|
||||
status: 'running',
|
||||
...jsonData
|
||||
});
|
||||
} catch {
|
||||
// If not JSON, send as plain progress
|
||||
sendProgressToClients(importClients, {
|
||||
status: 'running',
|
||||
progress: output
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
activeImport.stderr.on('data', (data) => {
|
||||
const error = data.toString().trim();
|
||||
try {
|
||||
// Try to parse as JSON
|
||||
const jsonData = JSON.parse(error);
|
||||
sendProgressToClients(importClients, {
|
||||
status: 'error',
|
||||
...jsonData
|
||||
});
|
||||
} catch {
|
||||
sendProgressToClients(importClients, {
|
||||
status: 'error',
|
||||
error
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
activeImport.on('close', (code) => {
|
||||
// Don't treat cancellation (code 143/SIGTERM) as an error
|
||||
if (code === 0 || code === 143) {
|
||||
sendProgressToClients(importClients, {
|
||||
status: 'complete',
|
||||
operation: code === 143 ? 'Operation cancelled' : 'Import complete'
|
||||
});
|
||||
resolve();
|
||||
} else {
|
||||
sendProgressToClients(importClients, {
|
||||
status: 'error',
|
||||
error: `Process exited with code ${code}`
|
||||
});
|
||||
reject(new Error(`Import process exited with code ${code}`));
|
||||
}
|
||||
activeImport = null;
|
||||
importProgress = null;
|
||||
});
|
||||
});
|
||||
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('Error importing CSV files:', error);
|
||||
activeImport = null;
|
||||
importProgress = null;
|
||||
sendProgressToClients(importClients, {
|
||||
status: 'error',
|
||||
error: error.message
|
||||
});
|
||||
res.status(500).json({ error: 'Failed to import CSV files', details: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Route to cancel active process
|
||||
router.post('/cancel', (req, res) => {
|
||||
if (!activeImport) {
|
||||
return res.status(404).json({ error: 'No active process to cancel' });
|
||||
}
|
||||
|
||||
try {
|
||||
// If it's the prod import module, call its cancel function
|
||||
if (typeof activeImport.cancelImport === 'function') {
|
||||
activeImport.cancelImport();
|
||||
} else {
|
||||
// Otherwise it's a child process
|
||||
activeImport.kill('SIGTERM');
|
||||
}
|
||||
|
||||
// Get the operation type from the request
|
||||
const { operation } = req.query;
|
||||
|
||||
// Send cancel message only to the appropriate client set
|
||||
const cancelMessage = {
|
||||
status: 'cancelled',
|
||||
operation: 'Operation cancelled'
|
||||
};
|
||||
|
||||
switch (operation) {
|
||||
case 'update':
|
||||
sendProgressToClients(updateClients, cancelMessage);
|
||||
break;
|
||||
case 'import':
|
||||
sendProgressToClients(importClients, cancelMessage);
|
||||
break;
|
||||
case 'reset':
|
||||
sendProgressToClients(resetClients, cancelMessage);
|
||||
break;
|
||||
case 'calculate-metrics':
|
||||
sendProgressToClients(calculateMetricsClients, cancelMessage);
|
||||
break;
|
||||
}
|
||||
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
// Even if there's an error, try to clean up
|
||||
activeImport = null;
|
||||
importProgress = null;
|
||||
res.status(500).json({ error: 'Failed to cancel process' });
|
||||
}
|
||||
});
|
||||
|
||||
// Route to reset database
|
||||
router.post('/reset', async (req, res) => {
|
||||
if (activeImport) {
|
||||
return res.status(409).json({ error: 'Import already in progress' });
|
||||
}
|
||||
|
||||
try {
|
||||
const scriptPath = path.join(__dirname, '..', '..', 'scripts', 'reset-db.js');
|
||||
|
||||
if (!require('fs').existsSync(scriptPath)) {
|
||||
return res.status(500).json({ error: 'Reset script not found' });
|
||||
}
|
||||
|
||||
activeImport = spawn('node', [scriptPath]);
|
||||
|
||||
activeImport.stdout.on('data', (data) => {
|
||||
const output = data.toString().trim();
|
||||
|
||||
try {
|
||||
// Try to parse as JSON
|
||||
const jsonData = JSON.parse(output);
|
||||
sendProgressToClients(resetClients, {
|
||||
status: 'running',
|
||||
...jsonData
|
||||
});
|
||||
} catch (e) {
|
||||
// If not JSON, send as plain progress
|
||||
sendProgressToClients(resetClients, {
|
||||
status: 'running',
|
||||
progress: output
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
activeImport.stderr.on('data', (data) => {
|
||||
const error = data.toString().trim();
|
||||
try {
|
||||
// Try to parse as JSON
|
||||
const jsonData = JSON.parse(error);
|
||||
sendProgressToClients(resetClients, {
|
||||
status: 'error',
|
||||
...jsonData
|
||||
});
|
||||
} catch {
|
||||
sendProgressToClients(resetClients, {
|
||||
status: 'error',
|
||||
error
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
activeImport.on('close', (code) => {
|
||||
// Don't treat cancellation (code 143/SIGTERM) as an error
|
||||
if (code === 0 || code === 143) {
|
||||
sendProgressToClients(resetClients, {
|
||||
status: 'complete',
|
||||
operation: code === 143 ? 'Operation cancelled' : 'Reset complete'
|
||||
});
|
||||
resolve();
|
||||
} else {
|
||||
const errorMsg = `Reset process exited with code ${code}`;
|
||||
sendProgressToClients(resetClients, {
|
||||
status: 'error',
|
||||
error: errorMsg
|
||||
});
|
||||
reject(new Error(errorMsg));
|
||||
}
|
||||
activeImport = null;
|
||||
importProgress = null;
|
||||
});
|
||||
});
|
||||
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('Error resetting database:', error);
|
||||
activeImport = null;
|
||||
importProgress = null;
|
||||
sendProgressToClients(resetClients, {
|
||||
status: 'error',
|
||||
error: error.message
|
||||
});
|
||||
res.status(500).json({ error: 'Failed to reset database', details: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Add reset-metrics endpoint
|
||||
router.post('/reset-metrics', async (req, res) => {
|
||||
if (activeImport) {
|
||||
res.status(400).json({ error: 'Operation already in progress' });
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Set active import to prevent concurrent operations
|
||||
activeImport = {
|
||||
type: 'reset-metrics',
|
||||
status: 'running',
|
||||
operation: 'Starting metrics reset'
|
||||
};
|
||||
|
||||
// Send initial response
|
||||
res.status(200).json({ message: 'Reset metrics started' });
|
||||
|
||||
// Send initial progress through SSE
|
||||
sendProgressToClients(resetMetricsClients, {
|
||||
status: 'running',
|
||||
operation: 'Starting metrics reset'
|
||||
});
|
||||
|
||||
// Run the reset metrics script
|
||||
const resetMetrics = require('../../scripts/reset-metrics');
|
||||
await resetMetrics();
|
||||
|
||||
// Send completion through SSE
|
||||
sendProgressToClients(resetMetricsClients, {
|
||||
status: 'complete',
|
||||
operation: 'Metrics reset completed'
|
||||
});
|
||||
|
||||
activeImport = null;
|
||||
} catch (error) {
|
||||
console.error('Error during metrics reset:', error);
|
||||
|
||||
// Send error through SSE
|
||||
sendProgressToClients(resetMetricsClients, {
|
||||
status: 'error',
|
||||
error: error.message || 'Failed to reset metrics'
|
||||
});
|
||||
|
||||
activeImport = null;
|
||||
res.status(500).json({ error: error.message || 'Failed to reset metrics' });
|
||||
}
|
||||
});
|
||||
|
||||
// Add calculate-metrics status endpoint
|
||||
router.get('/calculate-metrics/status', (req, res) => {
|
||||
const calculateMetrics = require('../../scripts/calculate-metrics');
|
||||
const progress = calculateMetrics.getProgress();
|
||||
|
||||
// Only consider it active if both the process is running and we have progress
|
||||
const isActive = !!activeImport && !!progress;
|
||||
|
||||
res.json({
|
||||
active: isActive,
|
||||
progress: isActive ? progress : null
|
||||
});
|
||||
});
|
||||
|
||||
// Add calculate-metrics endpoint
|
||||
router.post('/calculate-metrics', async (req, res) => {
|
||||
if (activeImport) {
|
||||
return res.status(409).json({ error: 'Import already in progress' });
|
||||
}
|
||||
|
||||
try {
|
||||
const scriptPath = path.join(__dirname, '..', '..', 'scripts', 'calculate-metrics.js');
|
||||
|
||||
if (!require('fs').existsSync(scriptPath)) {
|
||||
return res.status(500).json({ error: 'Calculate metrics script not found' });
|
||||
}
|
||||
|
||||
activeImport = spawn('node', [scriptPath]);
|
||||
let wasCancelled = false;
|
||||
|
||||
activeImport.stdout.on('data', (data) => {
|
||||
const output = data.toString().trim();
|
||||
|
||||
try {
|
||||
// Try to parse as JSON
|
||||
const jsonData = JSON.parse(output);
|
||||
importProgress = {
|
||||
status: 'running',
|
||||
...jsonData.progress
|
||||
};
|
||||
sendProgressToClients(calculateMetricsClients, importProgress);
|
||||
} catch (e) {
|
||||
// If not JSON, send as plain progress
|
||||
importProgress = {
|
||||
status: 'running',
|
||||
progress: output
|
||||
};
|
||||
sendProgressToClients(calculateMetricsClients, importProgress);
|
||||
}
|
||||
});
|
||||
|
||||
activeImport.stderr.on('data', (data) => {
|
||||
if (wasCancelled) return; // Don't send errors if cancelled
|
||||
|
||||
const error = data.toString().trim();
|
||||
try {
|
||||
// Try to parse as JSON
|
||||
const jsonData = JSON.parse(error);
|
||||
importProgress = {
|
||||
status: 'error',
|
||||
...jsonData.progress
|
||||
};
|
||||
sendProgressToClients(calculateMetricsClients, importProgress);
|
||||
} catch {
|
||||
importProgress = {
|
||||
status: 'error',
|
||||
error
|
||||
};
|
||||
sendProgressToClients(calculateMetricsClients, importProgress);
|
||||
}
|
||||
});
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
activeImport.on('close', (code, signal) => {
|
||||
wasCancelled = signal === 'SIGTERM' || code === 143;
|
||||
activeImport = null;
|
||||
|
||||
if (code === 0 || wasCancelled) {
|
||||
if (wasCancelled) {
|
||||
importProgress = {
|
||||
status: 'cancelled',
|
||||
operation: 'Operation cancelled'
|
||||
};
|
||||
sendProgressToClients(calculateMetricsClients, importProgress);
|
||||
} else {
|
||||
importProgress = {
|
||||
status: 'complete',
|
||||
operation: 'Metrics calculation complete'
|
||||
};
|
||||
sendProgressToClients(calculateMetricsClients, importProgress);
|
||||
}
|
||||
resolve();
|
||||
} else {
|
||||
importProgress = null;
|
||||
reject(new Error(`Metrics calculation process exited with code ${code}`));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
res.json({ success: true });
|
||||
} catch (error) {
|
||||
console.error('Error calculating metrics:', error);
|
||||
activeImport = null;
|
||||
importProgress = null;
|
||||
|
||||
// Only send error if it wasn't a cancellation
|
||||
if (!error.message?.includes('code 143') && !error.message?.includes('SIGTERM')) {
|
||||
sendProgressToClients(calculateMetricsClients, {
|
||||
status: 'error',
|
||||
error: error.message
|
||||
});
|
||||
res.status(500).json({ error: 'Failed to calculate metrics', details: error.message });
|
||||
} else {
|
||||
res.json({ success: true });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Route to import from production database
|
||||
router.post('/import-from-prod', async (req, res) => {
|
||||
if (activeImport) {
|
||||
return res.status(409).json({ error: 'Import already in progress' });
|
||||
}
|
||||
|
||||
try {
|
||||
const importFromProd = require('../../scripts/import-from-prod');
|
||||
|
||||
// Set up progress handler
|
||||
const progressHandler = (data) => {
|
||||
importProgress = data;
|
||||
sendProgressToClients(importClients, data);
|
||||
};
|
||||
|
||||
// Start the import process
|
||||
importFromProd.outputProgress = progressHandler;
|
||||
activeImport = importFromProd; // Store the module for cancellation
|
||||
|
||||
// Run the import in the background
|
||||
importFromProd.main().catch(error => {
|
||||
console.error('Error in import process:', error);
|
||||
activeImport = null;
|
||||
importProgress = {
|
||||
status: error.message === 'Import cancelled' ? 'cancelled' : 'error',
|
||||
operation: 'Import process',
|
||||
error: error.message
|
||||
};
|
||||
sendProgressToClients(importClients, importProgress);
|
||||
}).finally(() => {
|
||||
activeImport = null;
|
||||
});
|
||||
|
||||
res.json({ message: 'Import from production started' });
|
||||
} catch (error) {
|
||||
console.error('Error starting production import:', error);
|
||||
activeImport = null;
|
||||
res.status(500).json({ error: error.message || 'Failed to start production import' });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -1,935 +1 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const db = require('../utils/db');
|
||||
|
||||
// Import status codes
|
||||
const { ReceivingStatus } = require('../types/status-codes');
|
||||
|
||||
// Helper function to execute queries using the connection pool
|
||||
async function executeQuery(sql, params = []) {
|
||||
const pool = db.getPool();
|
||||
if (!pool) {
|
||||
throw new Error('Database pool not initialized');
|
||||
}
|
||||
return pool.query(sql, params);
|
||||
}
|
||||
|
||||
// GET /dashboard/stock/metrics
|
||||
// Returns brand-level stock metrics
|
||||
router.get('/stock/metrics', async (req, res) => {
|
||||
try {
|
||||
// Get stock metrics
|
||||
const [rows] = await executeQuery(`
|
||||
SELECT
|
||||
COALESCE(COUNT(*), 0) as total_products,
|
||||
COALESCE(COUNT(CASE WHEN stock_quantity > 0 THEN 1 END), 0) as products_in_stock,
|
||||
COALESCE(SUM(CASE WHEN stock_quantity > 0 THEN stock_quantity END), 0) as total_units,
|
||||
COALESCE(SUM(CASE WHEN stock_quantity > 0 THEN stock_quantity * cost_price END), 0) as total_cost,
|
||||
COALESCE(SUM(CASE WHEN stock_quantity > 0 THEN stock_quantity * price END), 0) as total_retail
|
||||
FROM products
|
||||
`);
|
||||
const stockMetrics = rows[0];
|
||||
|
||||
console.log('Raw stockMetrics from database:', stockMetrics);
|
||||
console.log('stockMetrics.total_products:', stockMetrics.total_products);
|
||||
console.log('stockMetrics.products_in_stock:', stockMetrics.products_in_stock);
|
||||
console.log('stockMetrics.total_units:', stockMetrics.total_units);
|
||||
console.log('stockMetrics.total_cost:', stockMetrics.total_cost);
|
||||
console.log('stockMetrics.total_retail:', stockMetrics.total_retail);
|
||||
|
||||
// Get brand stock values with Other category
|
||||
const [brandValues] = await executeQuery(`
|
||||
WITH brand_totals AS (
|
||||
SELECT
|
||||
COALESCE(brand, 'Unbranded') as brand,
|
||||
COUNT(DISTINCT pid) as variant_count,
|
||||
COALESCE(SUM(stock_quantity), 0) as stock_units,
|
||||
CAST(COALESCE(SUM(stock_quantity * cost_price), 0) AS DECIMAL(15,3)) as stock_cost,
|
||||
CAST(COALESCE(SUM(stock_quantity * price), 0) AS DECIMAL(15,3)) as stock_retail
|
||||
FROM products
|
||||
WHERE stock_quantity > 0
|
||||
GROUP BY COALESCE(brand, 'Unbranded')
|
||||
HAVING stock_cost > 0
|
||||
),
|
||||
other_brands AS (
|
||||
SELECT
|
||||
'Other' as brand,
|
||||
SUM(variant_count) as variant_count,
|
||||
SUM(stock_units) as stock_units,
|
||||
CAST(SUM(stock_cost) AS DECIMAL(15,3)) as stock_cost,
|
||||
CAST(SUM(stock_retail) AS DECIMAL(15,3)) as stock_retail
|
||||
FROM brand_totals
|
||||
WHERE stock_cost <= 5000
|
||||
),
|
||||
main_brands AS (
|
||||
SELECT *
|
||||
FROM brand_totals
|
||||
WHERE stock_cost > 5000
|
||||
ORDER BY stock_cost DESC
|
||||
)
|
||||
SELECT * FROM main_brands
|
||||
UNION ALL
|
||||
SELECT * FROM other_brands
|
||||
WHERE stock_cost > 0
|
||||
ORDER BY CASE WHEN brand = 'Other' THEN 1 ELSE 0 END, stock_cost DESC
|
||||
`);
|
||||
|
||||
// Format the response with explicit type conversion
|
||||
const response = {
|
||||
totalProducts: parseInt(stockMetrics.total_products) || 0,
|
||||
productsInStock: parseInt(stockMetrics.products_in_stock) || 0,
|
||||
totalStockUnits: parseInt(stockMetrics.total_units) || 0,
|
||||
totalStockCost: parseFloat(stockMetrics.total_cost) || 0,
|
||||
totalStockRetail: parseFloat(stockMetrics.total_retail) || 0,
|
||||
brandStock: brandValues.map(v => ({
|
||||
brand: v.brand,
|
||||
variants: parseInt(v.variant_count) || 0,
|
||||
units: parseInt(v.stock_units) || 0,
|
||||
cost: parseFloat(v.stock_cost) || 0,
|
||||
retail: parseFloat(v.stock_retail) || 0
|
||||
}))
|
||||
};
|
||||
|
||||
res.json(response);
|
||||
} catch (err) {
|
||||
console.error('Error fetching stock metrics:', err);
|
||||
res.status(500).json({ error: 'Failed to fetch stock metrics' });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /dashboard/purchase/metrics
|
||||
// Returns purchase order metrics by vendor
|
||||
router.get('/purchase/metrics', async (req, res) => {
|
||||
try {
|
||||
const [rows] = await executeQuery(`
|
||||
SELECT
|
||||
COALESCE(COUNT(DISTINCT CASE
|
||||
WHEN po.receiving_status < ${ReceivingStatus.PartialReceived}
|
||||
THEN po.po_id
|
||||
END), 0) as active_pos,
|
||||
COALESCE(COUNT(DISTINCT CASE
|
||||
WHEN po.receiving_status < ${ReceivingStatus.PartialReceived}
|
||||
AND po.expected_date < CURDATE()
|
||||
THEN po.po_id
|
||||
END), 0) as overdue_pos,
|
||||
COALESCE(SUM(CASE
|
||||
WHEN po.receiving_status < ${ReceivingStatus.PartialReceived}
|
||||
THEN po.ordered
|
||||
ELSE 0
|
||||
END), 0) as total_units,
|
||||
CAST(COALESCE(SUM(CASE
|
||||
WHEN po.receiving_status < ${ReceivingStatus.PartialReceived}
|
||||
THEN po.ordered * po.cost_price
|
||||
ELSE 0
|
||||
END), 0) AS DECIMAL(15,3)) as total_cost,
|
||||
CAST(COALESCE(SUM(CASE
|
||||
WHEN po.receiving_status < ${ReceivingStatus.PartialReceived}
|
||||
THEN po.ordered * p.price
|
||||
ELSE 0
|
||||
END), 0) AS DECIMAL(15,3)) as total_retail
|
||||
FROM purchase_orders po
|
||||
JOIN products p ON po.pid = p.pid
|
||||
`);
|
||||
const poMetrics = rows[0];
|
||||
|
||||
const [vendorOrders] = await executeQuery(`
|
||||
SELECT
|
||||
po.vendor,
|
||||
COUNT(DISTINCT po.po_id) as orders,
|
||||
COALESCE(SUM(po.ordered), 0) as units,
|
||||
CAST(COALESCE(SUM(po.ordered * po.cost_price), 0) AS DECIMAL(15,3)) as cost,
|
||||
CAST(COALESCE(SUM(po.ordered * p.price), 0) AS DECIMAL(15,3)) as retail
|
||||
FROM purchase_orders po
|
||||
JOIN products p ON po.pid = p.pid
|
||||
WHERE po.receiving_status < ${ReceivingStatus.PartialReceived}
|
||||
GROUP BY po.vendor
|
||||
HAVING cost > 0
|
||||
ORDER BY cost DESC
|
||||
`);
|
||||
|
||||
// Format response to match PurchaseMetricsData interface
|
||||
const response = {
|
||||
activePurchaseOrders: parseInt(poMetrics.active_pos) || 0,
|
||||
overduePurchaseOrders: parseInt(poMetrics.overdue_pos) || 0,
|
||||
onOrderUnits: parseInt(poMetrics.total_units) || 0,
|
||||
onOrderCost: parseFloat(poMetrics.total_cost) || 0,
|
||||
onOrderRetail: parseFloat(poMetrics.total_retail) || 0,
|
||||
vendorOrders: vendorOrders.map(v => ({
|
||||
vendor: v.vendor,
|
||||
orders: parseInt(v.orders) || 0,
|
||||
units: parseInt(v.units) || 0,
|
||||
cost: parseFloat(v.cost) || 0,
|
||||
retail: parseFloat(v.retail) || 0
|
||||
}))
|
||||
};
|
||||
|
||||
res.json(response);
|
||||
} catch (err) {
|
||||
console.error('Error fetching purchase metrics:', err);
|
||||
res.status(500).json({ error: 'Failed to fetch purchase metrics' });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /dashboard/replenishment/metrics
|
||||
// Returns replenishment needs by category
|
||||
router.get('/replenishment/metrics', async (req, res) => {
|
||||
try {
|
||||
// Get summary metrics
|
||||
const [metrics] = await executeQuery(`
|
||||
SELECT
|
||||
COUNT(DISTINCT p.pid) as products_to_replenish,
|
||||
COALESCE(SUM(CASE
|
||||
WHEN p.stock_quantity < 0 THEN ABS(p.stock_quantity) + pm.reorder_qty
|
||||
ELSE pm.reorder_qty
|
||||
END), 0) as total_units_needed,
|
||||
CAST(COALESCE(SUM(CASE
|
||||
WHEN p.stock_quantity < 0 THEN (ABS(p.stock_quantity) + pm.reorder_qty) * p.cost_price
|
||||
ELSE pm.reorder_qty * p.cost_price
|
||||
END), 0) AS DECIMAL(15,3)) as total_cost,
|
||||
CAST(COALESCE(SUM(CASE
|
||||
WHEN p.stock_quantity < 0 THEN (ABS(p.stock_quantity) + pm.reorder_qty) * p.price
|
||||
ELSE pm.reorder_qty * p.price
|
||||
END), 0) AS DECIMAL(15,3)) as total_retail
|
||||
FROM products p
|
||||
JOIN product_metrics pm ON p.pid = pm.pid
|
||||
WHERE p.replenishable = true
|
||||
AND (pm.stock_status IN ('Critical', 'Reorder')
|
||||
OR p.stock_quantity < 0)
|
||||
AND pm.reorder_qty > 0
|
||||
`);
|
||||
|
||||
// Get top variants to replenish
|
||||
const [variants] = await executeQuery(`
|
||||
SELECT
|
||||
p.pid,
|
||||
p.title,
|
||||
p.stock_quantity as current_stock,
|
||||
CASE
|
||||
WHEN p.stock_quantity < 0 THEN ABS(p.stock_quantity) + pm.reorder_qty
|
||||
ELSE pm.reorder_qty
|
||||
END as replenish_qty,
|
||||
CAST(CASE
|
||||
WHEN p.stock_quantity < 0 THEN (ABS(p.stock_quantity) + pm.reorder_qty) * p.cost_price
|
||||
ELSE pm.reorder_qty * p.cost_price
|
||||
END AS DECIMAL(15,3)) as replenish_cost,
|
||||
CAST(CASE
|
||||
WHEN p.stock_quantity < 0 THEN (ABS(p.stock_quantity) + pm.reorder_qty) * p.price
|
||||
ELSE pm.reorder_qty * p.price
|
||||
END AS DECIMAL(15,3)) as replenish_retail,
|
||||
pm.stock_status
|
||||
FROM products p
|
||||
JOIN product_metrics pm ON p.pid = pm.pid
|
||||
WHERE p.replenishable = true
|
||||
AND (pm.stock_status IN ('Critical', 'Reorder')
|
||||
OR p.stock_quantity < 0)
|
||||
AND pm.reorder_qty > 0
|
||||
ORDER BY
|
||||
CASE pm.stock_status
|
||||
WHEN 'Critical' THEN 1
|
||||
WHEN 'Reorder' THEN 2
|
||||
END,
|
||||
replenish_cost DESC
|
||||
LIMIT 5
|
||||
`);
|
||||
|
||||
// Format response
|
||||
const response = {
|
||||
productsToReplenish: parseInt(metrics[0].products_to_replenish) || 0,
|
||||
unitsToReplenish: parseInt(metrics[0].total_units_needed) || 0,
|
||||
replenishmentCost: parseFloat(metrics[0].total_cost) || 0,
|
||||
replenishmentRetail: parseFloat(metrics[0].total_retail) || 0,
|
||||
topVariants: variants.map(v => ({
|
||||
id: v.pid,
|
||||
title: v.title,
|
||||
currentStock: parseInt(v.current_stock) || 0,
|
||||
replenishQty: parseInt(v.replenish_qty) || 0,
|
||||
replenishCost: parseFloat(v.replenish_cost) || 0,
|
||||
replenishRetail: parseFloat(v.replenish_retail) || 0,
|
||||
status: v.stock_status
|
||||
}))
|
||||
};
|
||||
|
||||
res.json(response);
|
||||
} catch (err) {
|
||||
console.error('Error fetching replenishment metrics:', err);
|
||||
res.status(500).json({ error: 'Failed to fetch replenishment metrics' });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /dashboard/forecast/metrics
|
||||
// Returns sales forecasts for specified period
|
||||
router.get('/forecast/metrics', async (req, res) => {
|
||||
const { startDate, endDate } = req.query;
|
||||
try {
|
||||
// Get summary metrics
|
||||
const [metrics] = await executeQuery(`
|
||||
SELECT
|
||||
COALESCE(SUM(forecast_units), 0) as total_forecast_units,
|
||||
COALESCE(SUM(forecast_revenue), 0) as total_forecast_revenue,
|
||||
COALESCE(AVG(confidence_level), 0) as overall_confidence
|
||||
FROM sales_forecasts
|
||||
WHERE forecast_date BETWEEN ? AND ?
|
||||
`, [startDate, endDate]);
|
||||
|
||||
// Get daily forecasts
|
||||
const [dailyForecasts] = await executeQuery(`
|
||||
SELECT
|
||||
DATE(forecast_date) as date,
|
||||
COALESCE(SUM(forecast_revenue), 0) as revenue,
|
||||
COALESCE(AVG(confidence_level), 0) as confidence
|
||||
FROM sales_forecasts
|
||||
WHERE forecast_date BETWEEN ? AND ?
|
||||
GROUP BY DATE(forecast_date)
|
||||
ORDER BY date
|
||||
`, [startDate, endDate]);
|
||||
|
||||
// Get category forecasts
|
||||
const [categoryForecasts] = await executeQuery(`
|
||||
SELECT
|
||||
c.name as category,
|
||||
COALESCE(SUM(cf.forecast_units), 0) as units,
|
||||
COALESCE(SUM(cf.forecast_revenue), 0) as revenue,
|
||||
COALESCE(AVG(cf.confidence_level), 0) as confidence
|
||||
FROM category_forecasts cf
|
||||
JOIN categories c ON cf.category_id = c.cat_id
|
||||
WHERE cf.forecast_date BETWEEN ? AND ?
|
||||
GROUP BY c.cat_id, c.name
|
||||
ORDER BY revenue DESC
|
||||
`, [startDate, endDate]);
|
||||
|
||||
// Format response
|
||||
const response = {
|
||||
forecastSales: parseInt(metrics[0].total_forecast_units) || 0,
|
||||
forecastRevenue: parseFloat(metrics[0].total_forecast_revenue) || 0,
|
||||
confidenceLevel: parseFloat(metrics[0].overall_confidence) || 0,
|
||||
dailyForecasts: dailyForecasts.map(d => ({
|
||||
date: d.date,
|
||||
revenue: parseFloat(d.revenue) || 0,
|
||||
confidence: parseFloat(d.confidence) || 0
|
||||
})),
|
||||
categoryForecasts: categoryForecasts.map(c => ({
|
||||
category: c.category,
|
||||
units: parseInt(c.units) || 0,
|
||||
revenue: parseFloat(c.revenue) || 0,
|
||||
confidence: parseFloat(c.confidence) || 0
|
||||
}))
|
||||
};
|
||||
|
||||
res.json(response);
|
||||
} catch (err) {
|
||||
console.error('Error fetching forecast metrics:', err);
|
||||
res.status(500).json({ error: 'Failed to fetch forecast metrics' });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /dashboard/overstock/metrics
|
||||
// Returns overstock metrics by category
|
||||
router.get('/overstock/metrics', async (req, res) => {
|
||||
try {
|
||||
const [rows] = await executeQuery(`
|
||||
WITH category_overstock AS (
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name as category_name,
|
||||
COUNT(DISTINCT CASE
|
||||
WHEN pm.stock_status = 'Overstocked'
|
||||
THEN p.pid
|
||||
END) as overstocked_products,
|
||||
SUM(CASE
|
||||
WHEN pm.stock_status = 'Overstocked'
|
||||
THEN pm.overstocked_amt
|
||||
ELSE 0
|
||||
END) as total_excess_units,
|
||||
SUM(CASE
|
||||
WHEN pm.stock_status = 'Overstocked'
|
||||
THEN pm.overstocked_amt * p.cost_price
|
||||
ELSE 0
|
||||
END) as total_excess_cost,
|
||||
SUM(CASE
|
||||
WHEN pm.stock_status = 'Overstocked'
|
||||
THEN pm.overstocked_amt * p.price
|
||||
ELSE 0
|
||||
END) as total_excess_retail
|
||||
FROM categories c
|
||||
JOIN product_categories pc ON c.cat_id = pc.cat_id
|
||||
JOIN products p ON pc.pid = p.pid
|
||||
JOIN product_metrics pm ON p.pid = pm.pid
|
||||
GROUP BY c.cat_id, c.name
|
||||
)
|
||||
SELECT
|
||||
SUM(overstocked_products) as total_overstocked,
|
||||
SUM(total_excess_units) as total_excess_units,
|
||||
SUM(total_excess_cost) as total_excess_cost,
|
||||
SUM(total_excess_retail) as total_excess_retail,
|
||||
CONCAT('[', GROUP_CONCAT(
|
||||
JSON_OBJECT(
|
||||
'category', category_name,
|
||||
'products', overstocked_products,
|
||||
'units', total_excess_units,
|
||||
'cost', total_excess_cost,
|
||||
'retail', total_excess_retail
|
||||
)
|
||||
), ']') as category_data
|
||||
FROM (
|
||||
SELECT *
|
||||
FROM category_overstock
|
||||
WHERE overstocked_products > 0
|
||||
ORDER BY total_excess_cost DESC
|
||||
LIMIT 8
|
||||
) filtered_categories
|
||||
`);
|
||||
|
||||
// Format response with explicit type conversion
|
||||
const response = {
|
||||
overstockedProducts: parseInt(rows[0].total_overstocked) || 0,
|
||||
total_excess_units: parseInt(rows[0].total_excess_units) || 0,
|
||||
total_excess_cost: parseFloat(rows[0].total_excess_cost) || 0,
|
||||
total_excess_retail: parseFloat(rows[0].total_excess_retail) || 0,
|
||||
category_data: rows[0].category_data ?
|
||||
JSON.parse(rows[0].category_data).map(obj => ({
|
||||
category: obj.category,
|
||||
products: parseInt(obj.products) || 0,
|
||||
units: parseInt(obj.units) || 0,
|
||||
cost: parseFloat(obj.cost) || 0,
|
||||
retail: parseFloat(obj.retail) || 0
|
||||
})) : []
|
||||
};
|
||||
|
||||
res.json(response);
|
||||
} catch (err) {
|
||||
console.error('Error fetching overstock metrics:', err);
|
||||
res.status(500).json({ error: 'Failed to fetch overstock metrics' });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /dashboard/overstock/products
|
||||
// Returns list of most overstocked products
|
||||
router.get('/overstock/products', async (req, res) => {
|
||||
const limit = parseInt(req.query.limit) || 50;
|
||||
try {
|
||||
const [rows] = await executeQuery(`
|
||||
SELECT
|
||||
p.pid,
|
||||
p.SKU,
|
||||
p.title,
|
||||
p.brand,
|
||||
p.vendor,
|
||||
p.stock_quantity,
|
||||
p.cost_price,
|
||||
p.price,
|
||||
pm.daily_sales_avg,
|
||||
pm.days_of_inventory,
|
||||
pm.overstocked_amt,
|
||||
(pm.overstocked_amt * p.cost_price) as excess_cost,
|
||||
(pm.overstocked_amt * p.price) as excess_retail,
|
||||
GROUP_CONCAT(c.name) as categories
|
||||
FROM products p
|
||||
JOIN product_metrics pm ON p.pid = pm.pid
|
||||
LEFT JOIN product_categories pc ON p.pid = pc.pid
|
||||
LEFT JOIN categories c ON pc.cat_id = c.cat_id
|
||||
WHERE pm.stock_status = 'Overstocked'
|
||||
GROUP BY p.pid
|
||||
ORDER BY excess_cost DESC
|
||||
LIMIT ?
|
||||
`, [limit]);
|
||||
res.json(rows);
|
||||
} catch (err) {
|
||||
console.error('Error fetching overstocked products:', err);
|
||||
res.status(500).json({ error: 'Failed to fetch overstocked products' });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /dashboard/best-sellers
|
||||
// Returns best-selling products, vendors, and categories
|
||||
router.get('/best-sellers', async (req, res) => {
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
|
||||
// Common CTE for category paths
|
||||
const categoryPathCTE = `
|
||||
WITH RECURSIVE category_path AS (
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
c.parent_id,
|
||||
CAST(c.name AS CHAR(1000)) as path
|
||||
FROM categories c
|
||||
WHERE c.parent_id IS NULL
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
c.parent_id,
|
||||
CONCAT(cp.path, ' > ', c.name)
|
||||
FROM categories c
|
||||
JOIN category_path cp ON c.parent_id = cp.cat_id
|
||||
)
|
||||
`;
|
||||
|
||||
// Get best selling products
|
||||
const [products] = await pool.query(`
|
||||
SELECT
|
||||
p.pid,
|
||||
p.SKU as sku,
|
||||
p.title,
|
||||
SUM(o.quantity) as units_sold,
|
||||
CAST(SUM(o.price * o.quantity) AS DECIMAL(15,3)) as revenue,
|
||||
CAST(SUM(o.price * o.quantity - p.cost_price * o.quantity) AS DECIMAL(15,3)) as profit
|
||||
FROM products p
|
||||
JOIN orders o ON p.pid = o.pid
|
||||
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
AND o.canceled = false
|
||||
GROUP BY p.pid
|
||||
ORDER BY units_sold DESC
|
||||
LIMIT 10
|
||||
`);
|
||||
|
||||
// Get best selling brands
|
||||
const [brands] = await pool.query(`
|
||||
SELECT
|
||||
p.brand,
|
||||
SUM(o.quantity) as units_sold,
|
||||
CAST(SUM(o.price * o.quantity) AS DECIMAL(15,3)) as revenue,
|
||||
CAST(SUM(o.price * o.quantity - p.cost_price * o.quantity) AS DECIMAL(15,3)) as profit,
|
||||
ROUND(
|
||||
((SUM(CASE
|
||||
WHEN o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
THEN o.price * o.quantity
|
||||
ELSE 0
|
||||
END) /
|
||||
NULLIF(SUM(CASE
|
||||
WHEN o.date >= DATE_SUB(CURDATE(), INTERVAL 60 DAY)
|
||||
AND o.date < DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
THEN o.price * o.quantity
|
||||
ELSE 0
|
||||
END), 0)) - 1) * 100,
|
||||
1
|
||||
) as growth_rate
|
||||
FROM products p
|
||||
JOIN orders o ON p.pid = o.pid
|
||||
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 60 DAY)
|
||||
AND o.canceled = false
|
||||
GROUP BY p.brand
|
||||
ORDER BY units_sold DESC
|
||||
LIMIT 10
|
||||
`);
|
||||
|
||||
// Get best selling categories with full path
|
||||
const [categories] = await pool.query(`
|
||||
${categoryPathCTE}
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
cp.path as categoryPath,
|
||||
SUM(o.quantity) as units_sold,
|
||||
CAST(SUM(o.price * o.quantity) AS DECIMAL(15,3)) as revenue,
|
||||
CAST(SUM(o.price * o.quantity - p.cost_price * o.quantity) AS DECIMAL(15,3)) as profit,
|
||||
ROUND(
|
||||
((SUM(CASE
|
||||
WHEN o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
THEN o.price * o.quantity
|
||||
ELSE 0
|
||||
END) /
|
||||
NULLIF(SUM(CASE
|
||||
WHEN o.date >= DATE_SUB(CURDATE(), INTERVAL 60 DAY)
|
||||
AND o.date < DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
THEN o.price * o.quantity
|
||||
ELSE 0
|
||||
END), 0)) - 1) * 100,
|
||||
1
|
||||
) as growth_rate
|
||||
FROM products p
|
||||
JOIN orders o ON p.pid = o.pid
|
||||
JOIN product_categories pc ON p.pid = pc.pid
|
||||
JOIN categories c ON pc.cat_id = c.cat_id
|
||||
JOIN category_path cp ON c.cat_id = cp.cat_id
|
||||
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 60 DAY)
|
||||
AND o.canceled = false
|
||||
GROUP BY c.cat_id, c.name, cp.path
|
||||
ORDER BY units_sold DESC
|
||||
LIMIT 10
|
||||
`);
|
||||
|
||||
res.json({ products, brands, categories });
|
||||
} catch (err) {
|
||||
console.error('Error fetching best sellers:', err);
|
||||
res.status(500).json({ error: 'Failed to fetch best sellers' });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /dashboard/sales/metrics
|
||||
// Returns sales metrics for specified period
|
||||
router.get('/sales/metrics', async (req, res) => {
|
||||
const { startDate, endDate } = req.query;
|
||||
try {
|
||||
// Get daily sales data
|
||||
const [dailyRows] = await executeQuery(`
|
||||
SELECT
|
||||
DATE(o.date) as sale_date,
|
||||
COUNT(DISTINCT o.order_number) as total_orders,
|
||||
SUM(o.quantity) as total_units,
|
||||
SUM(o.price * o.quantity) as total_revenue,
|
||||
SUM(p.cost_price * o.quantity) as total_cogs,
|
||||
SUM((o.price - p.cost_price) * o.quantity) as total_profit
|
||||
FROM orders o
|
||||
JOIN products p ON o.pid = p.pid
|
||||
WHERE o.canceled = false
|
||||
AND o.date BETWEEN ? AND ?
|
||||
GROUP BY DATE(o.date)
|
||||
ORDER BY sale_date
|
||||
`, [startDate, endDate]);
|
||||
|
||||
// Get summary metrics
|
||||
const [metrics] = await executeQuery(`
|
||||
SELECT
|
||||
COUNT(DISTINCT o.order_number) as total_orders,
|
||||
SUM(o.quantity) as total_units,
|
||||
SUM(o.price * o.quantity) as total_revenue,
|
||||
SUM(p.cost_price * o.quantity) as total_cogs,
|
||||
SUM((o.price - p.cost_price) * o.quantity) as total_profit
|
||||
FROM orders o
|
||||
JOIN products p ON o.pid = p.pid
|
||||
WHERE o.canceled = false
|
||||
AND o.date BETWEEN ? AND ?
|
||||
`, [startDate, endDate]);
|
||||
|
||||
const response = {
|
||||
totalOrders: parseInt(metrics[0]?.total_orders) || 0,
|
||||
totalUnitsSold: parseInt(metrics[0]?.total_units) || 0,
|
||||
totalCogs: parseFloat(metrics[0]?.total_cogs) || 0,
|
||||
totalRevenue: parseFloat(metrics[0]?.total_revenue) || 0,
|
||||
dailySales: dailyRows.map(day => ({
|
||||
date: day.sale_date,
|
||||
units: parseInt(day.total_units) || 0,
|
||||
revenue: parseFloat(day.total_revenue) || 0,
|
||||
cogs: parseFloat(day.total_cogs) || 0
|
||||
}))
|
||||
};
|
||||
|
||||
res.json(response);
|
||||
} catch (err) {
|
||||
console.error('Error fetching sales metrics:', err);
|
||||
res.status(500).json({ error: 'Failed to fetch sales metrics' });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /dashboard/low-stock/products
|
||||
// Returns list of products with critical or low stock levels
|
||||
router.get('/low-stock/products', async (req, res) => {
|
||||
const limit = parseInt(req.query.limit) || 50;
|
||||
try {
|
||||
const [rows] = await executeQuery(`
|
||||
SELECT
|
||||
p.pid,
|
||||
p.SKU,
|
||||
p.title,
|
||||
p.brand,
|
||||
p.vendor,
|
||||
p.stock_quantity,
|
||||
p.cost_price,
|
||||
p.price,
|
||||
pm.daily_sales_avg,
|
||||
pm.days_of_inventory,
|
||||
pm.reorder_qty,
|
||||
(pm.reorder_qty * p.cost_price) as reorder_cost,
|
||||
GROUP_CONCAT(c.name) as categories
|
||||
FROM products p
|
||||
JOIN product_metrics pm ON p.pid = pm.pid
|
||||
LEFT JOIN product_categories pc ON p.pid = pc.pid
|
||||
LEFT JOIN categories c ON pc.cat_id = c.cat_id
|
||||
WHERE pm.stock_status IN ('Critical', 'Reorder')
|
||||
AND p.replenishable = true
|
||||
GROUP BY p.pid
|
||||
ORDER BY
|
||||
CASE pm.stock_status
|
||||
WHEN 'Critical' THEN 1
|
||||
WHEN 'Reorder' THEN 2
|
||||
END,
|
||||
pm.days_of_inventory ASC
|
||||
LIMIT ?
|
||||
`, [limit]);
|
||||
res.json(rows);
|
||||
} catch (err) {
|
||||
console.error('Error fetching low stock products:', err);
|
||||
res.status(500).json({ error: 'Failed to fetch low stock products' });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /dashboard/trending/products
|
||||
// Returns list of trending products based on recent sales velocity
|
||||
router.get('/trending/products', async (req, res) => {
|
||||
const days = parseInt(req.query.days) || 30;
|
||||
const limit = parseInt(req.query.limit) || 20;
|
||||
try {
|
||||
const [rows] = await executeQuery(`
|
||||
WITH recent_sales AS (
|
||||
SELECT
|
||||
o.pid,
|
||||
COUNT(DISTINCT o.order_number) as recent_orders,
|
||||
SUM(o.quantity) as recent_units,
|
||||
SUM(o.price * o.quantity) as recent_revenue
|
||||
FROM orders o
|
||||
WHERE o.canceled = false
|
||||
AND o.date >= DATE_SUB(CURDATE(), INTERVAL ? DAY)
|
||||
GROUP BY o.pid
|
||||
)
|
||||
SELECT
|
||||
p.pid,
|
||||
p.SKU,
|
||||
p.title,
|
||||
p.brand,
|
||||
p.vendor,
|
||||
p.stock_quantity,
|
||||
rs.recent_orders,
|
||||
rs.recent_units,
|
||||
rs.recent_revenue,
|
||||
pm.daily_sales_avg,
|
||||
pm.stock_status,
|
||||
(rs.recent_units / ?) as daily_velocity,
|
||||
((rs.recent_units / ?) - pm.daily_sales_avg) / pm.daily_sales_avg * 100 as velocity_change,
|
||||
GROUP_CONCAT(c.name) as categories
|
||||
FROM recent_sales rs
|
||||
JOIN products p ON rs.pid = p.pid
|
||||
JOIN product_metrics pm ON p.pid = pm.pid
|
||||
LEFT JOIN product_categories pc ON p.pid = pc.pid
|
||||
LEFT JOIN categories c ON pc.cat_id = c.cat_id
|
||||
GROUP BY p.pid
|
||||
HAVING velocity_change > 0
|
||||
ORDER BY velocity_change DESC
|
||||
LIMIT ?
|
||||
`, [days, days, limit]);
|
||||
res.json(rows);
|
||||
} catch (err) {
|
||||
console.error('Error fetching trending products:', err);
|
||||
res.status(500).json({ error: 'Failed to fetch trending products' });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /dashboard/vendor/performance
|
||||
// Returns detailed vendor performance metrics
|
||||
router.get('/vendor/performance', async (req, res) => {
|
||||
try {
|
||||
const [rows] = await executeQuery(`
|
||||
WITH vendor_orders AS (
|
||||
SELECT
|
||||
po.vendor,
|
||||
COUNT(DISTINCT po.po_id) as total_orders,
|
||||
CAST(AVG(DATEDIFF(po.received_date, po.date)) AS DECIMAL(10,2)) as avg_lead_time,
|
||||
CAST(AVG(CASE
|
||||
WHEN po.status = 'completed'
|
||||
THEN DATEDIFF(po.received_date, po.expected_date)
|
||||
END) AS DECIMAL(10,2)) as avg_delay,
|
||||
CAST(SUM(CASE
|
||||
WHEN po.status = 'completed' AND po.received_date <= po.expected_date
|
||||
THEN 1
|
||||
ELSE 0
|
||||
END) * 100.0 / COUNT(*) AS DECIMAL(10,2)) as on_time_delivery_rate,
|
||||
CAST(AVG(CASE
|
||||
WHEN po.status = 'completed'
|
||||
THEN po.received / po.ordered * 100
|
||||
ELSE NULL
|
||||
END) AS DECIMAL(10,2)) as avg_fill_rate
|
||||
FROM purchase_orders po
|
||||
WHERE po.date >= DATE_SUB(CURDATE(), INTERVAL 180 DAY)
|
||||
GROUP BY po.vendor
|
||||
)
|
||||
SELECT
|
||||
vd.vendor,
|
||||
vd.contact_name,
|
||||
vd.status,
|
||||
CAST(vo.total_orders AS SIGNED) as total_orders,
|
||||
vo.avg_lead_time,
|
||||
vo.avg_delay,
|
||||
vo.on_time_delivery_rate,
|
||||
vo.avg_fill_rate
|
||||
FROM vendor_details vd
|
||||
JOIN vendor_orders vo ON vd.vendor = vo.vendor
|
||||
WHERE vd.status = 'active'
|
||||
ORDER BY vo.on_time_delivery_rate DESC
|
||||
`);
|
||||
|
||||
// Format response with explicit number parsing
|
||||
const formattedRows = rows.map(row => ({
|
||||
vendor: row.vendor,
|
||||
contact_name: row.contact_name,
|
||||
status: row.status,
|
||||
total_orders: parseInt(row.total_orders) || 0,
|
||||
avg_lead_time: parseFloat(row.avg_lead_time) || 0,
|
||||
avg_delay: parseFloat(row.avg_delay) || 0,
|
||||
on_time_delivery_rate: parseFloat(row.on_time_delivery_rate) || 0,
|
||||
avg_fill_rate: parseFloat(row.avg_fill_rate) || 0
|
||||
}));
|
||||
|
||||
res.json(formattedRows);
|
||||
} catch (err) {
|
||||
console.error('Error fetching vendor performance:', err);
|
||||
res.status(500).json({ error: 'Failed to fetch vendor performance' });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /dashboard/key-metrics
|
||||
// Returns key business metrics and KPIs
|
||||
router.get('/key-metrics', async (req, res) => {
|
||||
const days = Math.max(1, Math.min(365, parseInt(req.query.days) || 30));
|
||||
try {
|
||||
const [rows] = await executeQuery(`
|
||||
WITH inventory_summary AS (
|
||||
SELECT
|
||||
COUNT(*) as total_products,
|
||||
SUM(p.stock_quantity * p.cost_price) as total_inventory_value,
|
||||
AVG(pm.turnover_rate) as avg_turnover_rate,
|
||||
COUNT(CASE WHEN pm.stock_status = 'Critical' THEN 1 END) as critical_stock_count,
|
||||
COUNT(CASE WHEN pm.stock_status = 'Overstocked' THEN 1 END) as overstock_count
|
||||
FROM products p
|
||||
JOIN product_metrics pm ON p.pid = pm.pid
|
||||
),
|
||||
sales_summary AS (
|
||||
SELECT
|
||||
COUNT(DISTINCT order_number) as total_orders,
|
||||
SUM(quantity) as total_units_sold,
|
||||
SUM(price * quantity) as total_revenue,
|
||||
AVG(price * quantity) as avg_order_value
|
||||
FROM orders
|
||||
WHERE canceled = false
|
||||
AND date >= DATE_SUB(CURDATE(), INTERVAL ? DAY)
|
||||
),
|
||||
purchase_summary AS (
|
||||
SELECT
|
||||
COUNT(DISTINCT po_id) as total_pos,
|
||||
SUM(ordered * cost_price) as total_po_value,
|
||||
COUNT(CASE WHEN status = 'open' THEN 1 END) as open_pos
|
||||
FROM purchase_orders
|
||||
WHERE order_date >= DATE_SUB(CURDATE(), INTERVAL ? DAY)
|
||||
)
|
||||
SELECT
|
||||
i.*,
|
||||
s.*,
|
||||
p.*
|
||||
FROM inventory_summary i
|
||||
CROSS JOIN sales_summary s
|
||||
CROSS JOIN purchase_summary p
|
||||
`, [days, days]);
|
||||
res.json(rows[0]);
|
||||
} catch (err) {
|
||||
console.error('Error fetching key metrics:', err);
|
||||
res.status(500).json({ error: 'Failed to fetch key metrics' });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /dashboard/inventory-health
|
||||
// Returns overall inventory health metrics
|
||||
router.get('/inventory-health', async (req, res) => {
|
||||
try {
|
||||
const [rows] = await executeQuery(`
|
||||
WITH stock_distribution AS (
|
||||
SELECT
|
||||
COUNT(*) as total_products,
|
||||
SUM(CASE WHEN pm.stock_status = 'Healthy' THEN 1 ELSE 0 END) * 100.0 / COUNT(*) as healthy_stock_percent,
|
||||
SUM(CASE WHEN pm.stock_status = 'Critical' THEN 1 ELSE 0 END) * 100.0 / COUNT(*) as critical_stock_percent,
|
||||
SUM(CASE WHEN pm.stock_status = 'Reorder' THEN 1 ELSE 0 END) * 100.0 / COUNT(*) as reorder_stock_percent,
|
||||
SUM(CASE WHEN pm.stock_status = 'Overstocked' THEN 1 ELSE 0 END) * 100.0 / COUNT(*) as overstock_percent,
|
||||
AVG(pm.turnover_rate) as avg_turnover_rate,
|
||||
AVG(pm.days_of_inventory) as avg_days_inventory
|
||||
FROM products p
|
||||
JOIN product_metrics pm ON p.pid = pm.pid
|
||||
WHERE p.replenishable = true
|
||||
),
|
||||
value_distribution AS (
|
||||
SELECT
|
||||
SUM(p.stock_quantity * p.cost_price) as total_inventory_value,
|
||||
SUM(CASE
|
||||
WHEN pm.stock_status = 'Healthy'
|
||||
THEN p.stock_quantity * p.cost_price
|
||||
ELSE 0
|
||||
END) * 100.0 / SUM(p.stock_quantity * p.cost_price) as healthy_value_percent,
|
||||
SUM(CASE
|
||||
WHEN pm.stock_status = 'Critical'
|
||||
THEN p.stock_quantity * p.cost_price
|
||||
ELSE 0
|
||||
END) * 100.0 / SUM(p.stock_quantity * p.cost_price) as critical_value_percent,
|
||||
SUM(CASE
|
||||
WHEN pm.stock_status = 'Overstocked'
|
||||
THEN p.stock_quantity * p.cost_price
|
||||
ELSE 0
|
||||
END) * 100.0 / SUM(p.stock_quantity * p.cost_price) as overstock_value_percent
|
||||
FROM products p
|
||||
JOIN product_metrics pm ON p.pid = pm.pid
|
||||
),
|
||||
category_health AS (
|
||||
SELECT
|
||||
c.name as category_name,
|
||||
COUNT(*) as category_products,
|
||||
SUM(CASE WHEN pm.stock_status = 'Healthy' THEN 1 ELSE 0 END) * 100.0 / COUNT(*) as category_healthy_percent,
|
||||
AVG(pm.turnover_rate) as category_turnover_rate
|
||||
FROM categories c
|
||||
JOIN product_categories pc ON c.cat_id = pc.cat_id
|
||||
JOIN products p ON pc.pid = p.pid
|
||||
JOIN product_metrics pm ON p.pid = pm.pid
|
||||
WHERE p.replenishable = true
|
||||
GROUP BY c.cat_id, c.name
|
||||
)
|
||||
SELECT
|
||||
sd.*,
|
||||
vd.*,
|
||||
JSON_ARRAYAGG(
|
||||
JSON_OBJECT(
|
||||
'category', ch.category_name,
|
||||
'products', ch.category_products,
|
||||
'healthy_percent', ch.category_healthy_percent,
|
||||
'turnover_rate', ch.category_turnover_rate
|
||||
)
|
||||
) as category_health
|
||||
FROM stock_distribution sd
|
||||
CROSS JOIN value_distribution vd
|
||||
CROSS JOIN category_health ch
|
||||
`);
|
||||
res.json(rows[0]);
|
||||
} catch (err) {
|
||||
console.error('Error fetching inventory health:', err);
|
||||
res.status(500).json({ error: 'Failed to fetch inventory health' });
|
||||
}
|
||||
});
|
||||
|
||||
// GET /dashboard/replenish/products
|
||||
// Returns top products that need replenishment
|
||||
router.get('/replenish/products', async (req, res) => {
|
||||
const limit = Math.max(1, Math.min(100, parseInt(req.query.limit) || 50));
|
||||
try {
|
||||
const [products] = await executeQuery(`
|
||||
SELECT
|
||||
p.pid,
|
||||
p.SKU as sku,
|
||||
p.title,
|
||||
p.stock_quantity,
|
||||
pm.daily_sales_avg,
|
||||
pm.reorder_qty,
|
||||
pm.last_purchase_date
|
||||
FROM products p
|
||||
JOIN product_metrics pm ON p.pid = pm.pid
|
||||
WHERE p.replenishable = true
|
||||
AND pm.stock_status IN ('Critical', 'Reorder')
|
||||
AND pm.reorder_qty > 0
|
||||
ORDER BY
|
||||
CASE pm.stock_status
|
||||
WHEN 'Critical' THEN 1
|
||||
WHEN 'Reorder' THEN 2
|
||||
END,
|
||||
pm.reorder_qty * p.cost_price DESC
|
||||
LIMIT ?
|
||||
`, [limit]);
|
||||
|
||||
res.json(products.map(p => ({
|
||||
...p,
|
||||
stock_quantity: parseInt(p.stock_quantity) || 0,
|
||||
daily_sales_avg: parseFloat(p.daily_sales_avg) || 0,
|
||||
reorder_qty: parseInt(p.reorder_qty) || 0
|
||||
})));
|
||||
} catch (err) {
|
||||
console.error('Error fetching products to replenish:', err);
|
||||
res.status(500).json({ error: 'Failed to fetch products to replenish' });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
1592
inventory-server/src/routes/import.js
Normal file
1592
inventory-server/src/routes/import.js
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,60 +0,0 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
|
||||
// Get key metrics trends (revenue, inventory value, GMROI)
|
||||
router.get('/trends', async (req, res) => {
|
||||
const pool = req.app.locals.pool;
|
||||
try {
|
||||
const [rows] = await pool.query(`
|
||||
WITH MonthlyMetrics AS (
|
||||
SELECT
|
||||
DATE(CONCAT(pta.year, '-', LPAD(pta.month, 2, '0'), '-01')) as date,
|
||||
CAST(COALESCE(SUM(pta.total_revenue), 0) AS DECIMAL(15,3)) as revenue,
|
||||
CAST(COALESCE(SUM(pta.total_cost), 0) AS DECIMAL(15,3)) as cost,
|
||||
CAST(COALESCE(SUM(pm.inventory_value), 0) AS DECIMAL(15,3)) as inventory_value,
|
||||
CASE
|
||||
WHEN SUM(pm.inventory_value) > 0
|
||||
THEN CAST((SUM(pta.total_revenue - pta.total_cost) / SUM(pm.inventory_value)) * 100 AS DECIMAL(15,3))
|
||||
ELSE 0
|
||||
END as gmroi
|
||||
FROM product_time_aggregates pta
|
||||
JOIN product_metrics pm ON pta.pid = pm.pid
|
||||
WHERE (pta.year * 100 + pta.month) >= DATE_FORMAT(DATE_SUB(CURDATE(), INTERVAL 12 MONTH), '%Y%m')
|
||||
GROUP BY pta.year, pta.month
|
||||
ORDER BY date ASC
|
||||
)
|
||||
SELECT
|
||||
DATE_FORMAT(date, '%b %y') as date,
|
||||
revenue,
|
||||
inventory_value,
|
||||
gmroi
|
||||
FROM MonthlyMetrics
|
||||
`);
|
||||
|
||||
console.log('Raw metrics trends data:', rows);
|
||||
|
||||
// Transform the data into the format expected by the frontend
|
||||
const transformedData = {
|
||||
revenue: rows.map(row => ({
|
||||
date: row.date,
|
||||
value: parseFloat(row.revenue)
|
||||
})),
|
||||
inventory_value: rows.map(row => ({
|
||||
date: row.date,
|
||||
value: parseFloat(row.inventory_value)
|
||||
})),
|
||||
gmroi: rows.map(row => ({
|
||||
date: row.date,
|
||||
value: parseFloat(row.gmroi)
|
||||
}))
|
||||
};
|
||||
|
||||
console.log('Transformed metrics data:', transformedData);
|
||||
res.json(transformedData);
|
||||
} catch (error) {
|
||||
console.error('Error fetching metrics trends:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch metrics trends' });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -1,255 +0,0 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
|
||||
// Get all orders with pagination, filtering, and sorting
|
||||
router.get('/', async (req, res) => {
|
||||
const pool = req.app.locals.pool;
|
||||
try {
|
||||
const page = parseInt(req.query.page) || 1;
|
||||
const limit = parseInt(req.query.limit) || 50;
|
||||
const offset = (page - 1) * limit;
|
||||
const search = req.query.search || '';
|
||||
const status = req.query.status || 'all';
|
||||
const fromDate = req.query.fromDate ? new Date(req.query.fromDate) : null;
|
||||
const toDate = req.query.toDate ? new Date(req.query.toDate) : null;
|
||||
const minAmount = parseFloat(req.query.minAmount) || 0;
|
||||
const maxAmount = req.query.maxAmount ? parseFloat(req.query.maxAmount) : null;
|
||||
const sortColumn = req.query.sortColumn || 'date';
|
||||
const sortDirection = req.query.sortDirection === 'desc' ? 'DESC' : 'ASC';
|
||||
|
||||
// Build the WHERE clause
|
||||
const conditions = ['o1.canceled = false'];
|
||||
const params = [];
|
||||
|
||||
if (search) {
|
||||
conditions.push('(o1.order_number LIKE ? OR o1.customer LIKE ?)');
|
||||
params.push(`%${search}%`, `%${search}%`);
|
||||
}
|
||||
|
||||
if (status !== 'all') {
|
||||
conditions.push('o1.status = ?');
|
||||
params.push(status);
|
||||
}
|
||||
|
||||
if (fromDate) {
|
||||
conditions.push('DATE(o1.date) >= DATE(?)');
|
||||
params.push(fromDate.toISOString());
|
||||
}
|
||||
|
||||
if (toDate) {
|
||||
conditions.push('DATE(o1.date) <= DATE(?)');
|
||||
params.push(toDate.toISOString());
|
||||
}
|
||||
|
||||
if (minAmount > 0) {
|
||||
conditions.push('total_amount >= ?');
|
||||
params.push(minAmount);
|
||||
}
|
||||
|
||||
if (maxAmount) {
|
||||
conditions.push('total_amount <= ?');
|
||||
params.push(maxAmount);
|
||||
}
|
||||
|
||||
// Get total count for pagination
|
||||
const [countResult] = await pool.query(`
|
||||
SELECT COUNT(DISTINCT o1.order_number) as total
|
||||
FROM orders o1
|
||||
LEFT JOIN (
|
||||
SELECT order_number, SUM(price * quantity) as total_amount
|
||||
FROM orders
|
||||
GROUP BY order_number
|
||||
) totals ON o1.order_number = totals.order_number
|
||||
WHERE ${conditions.join(' AND ')}
|
||||
`, params);
|
||||
|
||||
const total = countResult[0].total;
|
||||
|
||||
// Get paginated results
|
||||
const query = `
|
||||
SELECT
|
||||
o1.order_number,
|
||||
o1.customer,
|
||||
o1.date,
|
||||
o1.status,
|
||||
o1.payment_method,
|
||||
o1.shipping_method,
|
||||
COUNT(o2.pid) as items_count,
|
||||
CAST(SUM(o2.price * o2.quantity) AS DECIMAL(15,3)) as total_amount
|
||||
FROM orders o1
|
||||
JOIN orders o2 ON o1.order_number = o2.order_number
|
||||
WHERE ${conditions.join(' AND ')}
|
||||
GROUP BY
|
||||
o1.order_number,
|
||||
o1.customer,
|
||||
o1.date,
|
||||
o1.status,
|
||||
o1.payment_method,
|
||||
o1.shipping_method
|
||||
ORDER BY ${
|
||||
sortColumn === 'items_count' || sortColumn === 'total_amount'
|
||||
? `${sortColumn} ${sortDirection}`
|
||||
: `o1.${sortColumn} ${sortDirection}`
|
||||
}
|
||||
LIMIT ? OFFSET ?
|
||||
`;
|
||||
|
||||
const [rows] = await pool.query(query, [...params, limit, offset]);
|
||||
|
||||
// Get order statistics
|
||||
const [stats] = await pool.query(`
|
||||
WITH CurrentStats AS (
|
||||
SELECT
|
||||
COUNT(DISTINCT order_number) as total_orders,
|
||||
CAST(SUM(price * quantity) AS DECIMAL(15,3)) as total_revenue
|
||||
FROM orders
|
||||
WHERE canceled = false
|
||||
AND DATE(date) >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
),
|
||||
PreviousStats AS (
|
||||
SELECT
|
||||
COUNT(DISTINCT order_number) as prev_orders,
|
||||
CAST(SUM(price * quantity) AS DECIMAL(15,3)) as prev_revenue
|
||||
FROM orders
|
||||
WHERE canceled = false
|
||||
AND DATE(date) BETWEEN DATE_SUB(CURDATE(), INTERVAL 60 DAY) AND DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
),
|
||||
OrderValues AS (
|
||||
SELECT
|
||||
order_number,
|
||||
CAST(SUM(price * quantity) AS DECIMAL(15,3)) as order_value
|
||||
FROM orders
|
||||
WHERE canceled = false
|
||||
AND DATE(date) >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
|
||||
GROUP BY order_number
|
||||
)
|
||||
SELECT
|
||||
cs.total_orders,
|
||||
cs.total_revenue,
|
||||
CASE
|
||||
WHEN ps.prev_orders > 0
|
||||
THEN ((cs.total_orders - ps.prev_orders) / ps.prev_orders * 100)
|
||||
ELSE 0
|
||||
END as order_growth,
|
||||
CASE
|
||||
WHEN ps.prev_revenue > 0
|
||||
THEN ((cs.total_revenue - ps.prev_revenue) / ps.prev_revenue * 100)
|
||||
ELSE 0
|
||||
END as revenue_growth,
|
||||
CASE
|
||||
WHEN cs.total_orders > 0
|
||||
THEN CAST((cs.total_revenue / cs.total_orders) AS DECIMAL(15,3))
|
||||
ELSE 0
|
||||
END as average_order_value,
|
||||
CASE
|
||||
WHEN ps.prev_orders > 0
|
||||
THEN CAST((ps.prev_revenue / ps.prev_orders) AS DECIMAL(15,3))
|
||||
ELSE 0
|
||||
END as prev_average_order_value
|
||||
FROM CurrentStats cs
|
||||
CROSS JOIN PreviousStats ps
|
||||
`);
|
||||
|
||||
const orderStats = stats[0];
|
||||
|
||||
res.json({
|
||||
orders: rows.map(row => ({
|
||||
...row,
|
||||
total_amount: parseFloat(row.total_amount) || 0,
|
||||
items_count: parseInt(row.items_count) || 0,
|
||||
date: row.date
|
||||
})),
|
||||
pagination: {
|
||||
total,
|
||||
pages: Math.ceil(total / limit),
|
||||
currentPage: page,
|
||||
limit
|
||||
},
|
||||
stats: {
|
||||
totalOrders: parseInt(orderStats.total_orders) || 0,
|
||||
totalRevenue: parseFloat(orderStats.total_revenue) || 0,
|
||||
orderGrowth: parseFloat(orderStats.order_growth) || 0,
|
||||
revenueGrowth: parseFloat(orderStats.revenue_growth) || 0,
|
||||
averageOrderValue: parseFloat(orderStats.average_order_value) || 0,
|
||||
aovGrowth: orderStats.prev_average_order_value > 0
|
||||
? ((orderStats.average_order_value - orderStats.prev_average_order_value) / orderStats.prev_average_order_value * 100)
|
||||
: 0,
|
||||
conversionRate: 2.5, // Placeholder - would need actual visitor data
|
||||
conversionGrowth: 0.5 // Placeholder - would need actual visitor data
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error fetching orders:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch orders' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get a single order with its items
|
||||
router.get('/:orderNumber', async (req, res) => {
|
||||
const pool = req.app.locals.pool;
|
||||
try {
|
||||
// Get order details
|
||||
const [orderRows] = await pool.query(`
|
||||
SELECT DISTINCT
|
||||
o1.order_number,
|
||||
o1.customer,
|
||||
o1.date,
|
||||
o1.status,
|
||||
o1.payment_method,
|
||||
o1.shipping_method,
|
||||
o1.shipping_address,
|
||||
o1.billing_address,
|
||||
COUNT(o2.pid) as items_count,
|
||||
CAST(SUM(o2.price * o2.quantity) AS DECIMAL(15,3)) as total_amount
|
||||
FROM orders o1
|
||||
JOIN orders o2 ON o1.order_number = o2.order_number
|
||||
WHERE o1.order_number = ? AND o1.canceled = false
|
||||
GROUP BY
|
||||
o1.order_number,
|
||||
o1.customer,
|
||||
o1.date,
|
||||
o1.status,
|
||||
o1.payment_method,
|
||||
o1.shipping_method,
|
||||
o1.shipping_address,
|
||||
o1.billing_address
|
||||
`, [req.params.orderNumber]);
|
||||
|
||||
if (orderRows.length === 0) {
|
||||
return res.status(404).json({ error: 'Order not found' });
|
||||
}
|
||||
|
||||
// Get order items
|
||||
const [itemRows] = await pool.query(`
|
||||
SELECT
|
||||
o.pid,
|
||||
p.title,
|
||||
p.SKU,
|
||||
o.quantity,
|
||||
o.price,
|
||||
CAST((o.price * o.quantity) AS DECIMAL(15,3)) as total
|
||||
FROM orders o
|
||||
JOIN products p ON o.pid = p.pid
|
||||
WHERE o.order_number = ? AND o.canceled = false
|
||||
`, [req.params.orderNumber]);
|
||||
|
||||
const order = {
|
||||
...orderRows[0],
|
||||
total_amount: parseFloat(orderRows[0].total_amount) || 0,
|
||||
items_count: parseInt(orderRows[0].items_count) || 0,
|
||||
items: itemRows.map(item => ({
|
||||
...item,
|
||||
price: parseFloat(item.price) || 0,
|
||||
total: parseFloat(item.total) || 0,
|
||||
quantity: parseInt(item.quantity) || 0
|
||||
}))
|
||||
};
|
||||
|
||||
res.json(order);
|
||||
} catch (error) {
|
||||
console.error('Error fetching order:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch order' });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -1,809 +0,0 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const multer = require('multer');
|
||||
const { importProductsFromCSV } = require('../utils/csvImporter');
|
||||
const { PurchaseOrderStatus, ReceivingStatus } = require('../types/status-codes');
|
||||
|
||||
// Configure multer for file uploads
|
||||
const upload = multer({ dest: 'uploads/' });
|
||||
|
||||
// Get unique brands
|
||||
router.get('/brands', async (req, res) => {
|
||||
console.log('Brands endpoint hit:', {
|
||||
url: req.url,
|
||||
method: req.method,
|
||||
headers: req.headers,
|
||||
path: req.path
|
||||
});
|
||||
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
console.log('Fetching brands from database...');
|
||||
|
||||
const [results] = await pool.query(`
|
||||
SELECT DISTINCT COALESCE(p.brand, 'Unbranded') as brand
|
||||
FROM products p
|
||||
JOIN purchase_orders po ON p.pid = po.pid
|
||||
WHERE p.visible = true
|
||||
GROUP BY COALESCE(p.brand, 'Unbranded')
|
||||
HAVING SUM(po.cost_price * po.received) >= 500
|
||||
ORDER BY COALESCE(p.brand, 'Unbranded')
|
||||
`);
|
||||
|
||||
console.log(`Found ${results.length} brands:`, results.slice(0, 3));
|
||||
res.json(results.map(r => r.brand));
|
||||
} catch (error) {
|
||||
console.error('Error fetching brands:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch brands' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get all products with pagination, filtering, and sorting
|
||||
router.get('/', async (req, res) => {
|
||||
const pool = req.app.locals.pool;
|
||||
try {
|
||||
const page = parseInt(req.query.page) || 1;
|
||||
const limit = parseInt(req.query.limit) || 50;
|
||||
const offset = (page - 1) * limit;
|
||||
const sortColumn = req.query.sort || 'title';
|
||||
const sortDirection = req.query.order === 'desc' ? 'DESC' : 'ASC';
|
||||
|
||||
const conditions = ['p.visible = true'];
|
||||
const params = [];
|
||||
|
||||
// Add default replenishable filter unless explicitly showing non-replenishable
|
||||
if (req.query.showNonReplenishable !== 'true') {
|
||||
conditions.push('p.replenishable = true');
|
||||
}
|
||||
|
||||
// Handle search filter
|
||||
if (req.query.search) {
|
||||
conditions.push('(p.title LIKE ? OR p.SKU LIKE ? OR p.barcode LIKE ?)');
|
||||
const searchTerm = `%${req.query.search}%`;
|
||||
params.push(searchTerm, searchTerm, searchTerm);
|
||||
}
|
||||
|
||||
// Handle numeric filters with operators
|
||||
const numericFields = {
|
||||
stock: 'p.stock_quantity',
|
||||
price: 'p.price',
|
||||
costPrice: 'p.cost_price',
|
||||
landingCost: 'p.landing_cost_price',
|
||||
dailySalesAvg: 'pm.daily_sales_avg',
|
||||
weeklySalesAvg: 'pm.weekly_sales_avg',
|
||||
monthlySalesAvg: 'pm.monthly_sales_avg',
|
||||
margin: 'pm.avg_margin_percent',
|
||||
gmroi: 'pm.gmroi',
|
||||
leadTime: 'pm.current_lead_time',
|
||||
stockCoverage: 'pm.days_of_inventory',
|
||||
daysOfStock: 'pm.days_of_inventory'
|
||||
};
|
||||
|
||||
Object.entries(req.query).forEach(([key, value]) => {
|
||||
const field = numericFields[key];
|
||||
if (field) {
|
||||
const operator = req.query[`${key}_operator`] || '=';
|
||||
if (operator === 'between') {
|
||||
// Handle between operator
|
||||
try {
|
||||
const [min, max] = JSON.parse(value);
|
||||
conditions.push(`${field} BETWEEN ? AND ?`);
|
||||
params.push(min, max);
|
||||
} catch (e) {
|
||||
console.error(`Invalid between value for ${key}:`, value);
|
||||
}
|
||||
} else {
|
||||
// Handle other operators
|
||||
conditions.push(`${field} ${operator} ?`);
|
||||
params.push(parseFloat(value));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Handle select filters
|
||||
if (req.query.vendor) {
|
||||
conditions.push('p.vendor = ?');
|
||||
params.push(req.query.vendor);
|
||||
}
|
||||
|
||||
if (req.query.brand) {
|
||||
conditions.push('p.brand = ?');
|
||||
params.push(req.query.brand);
|
||||
}
|
||||
|
||||
if (req.query.category) {
|
||||
conditions.push('p.categories LIKE ?');
|
||||
params.push(`%${req.query.category}%`);
|
||||
}
|
||||
|
||||
if (req.query.stockStatus && req.query.stockStatus !== 'all') {
|
||||
conditions.push('pm.stock_status = ?');
|
||||
params.push(req.query.stockStatus);
|
||||
}
|
||||
|
||||
if (req.query.abcClass) {
|
||||
conditions.push('pm.abc_class = ?');
|
||||
params.push(req.query.abcClass);
|
||||
}
|
||||
|
||||
if (req.query.leadTimeStatus) {
|
||||
conditions.push('pm.lead_time_status = ?');
|
||||
params.push(req.query.leadTimeStatus);
|
||||
}
|
||||
|
||||
if (req.query.replenishable !== undefined) {
|
||||
conditions.push('p.replenishable = ?');
|
||||
params.push(req.query.replenishable === 'true' ? 1 : 0);
|
||||
}
|
||||
|
||||
if (req.query.managingStock !== undefined) {
|
||||
conditions.push('p.managing_stock = ?');
|
||||
params.push(req.query.managingStock === 'true' ? 1 : 0);
|
||||
}
|
||||
|
||||
// Combine all conditions with AND
|
||||
const whereClause = conditions.length > 0 ? 'WHERE ' + conditions.join(' AND ') : '';
|
||||
|
||||
// Get total count for pagination
|
||||
const countQuery = `
|
||||
SELECT COUNT(DISTINCT p.pid) as total
|
||||
FROM products p
|
||||
LEFT JOIN product_metrics pm ON p.pid = pm.pid
|
||||
${whereClause}
|
||||
`;
|
||||
const [countResult] = await pool.query(countQuery, params);
|
||||
const total = countResult[0].total;
|
||||
|
||||
// Get available filters
|
||||
const [categories] = await pool.query(
|
||||
'SELECT name FROM categories ORDER BY name'
|
||||
);
|
||||
const [vendors] = await pool.query(
|
||||
'SELECT DISTINCT vendor FROM products WHERE visible = true AND vendor IS NOT NULL AND vendor != "" ORDER BY vendor'
|
||||
);
|
||||
const [brands] = await pool.query(
|
||||
'SELECT DISTINCT COALESCE(brand, \'Unbranded\') as brand FROM products WHERE visible = true ORDER BY brand'
|
||||
);
|
||||
|
||||
// Main query with all fields
|
||||
const query = `
|
||||
WITH RECURSIVE
|
||||
category_path AS (
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
c.parent_id,
|
||||
CAST(c.name AS CHAR(1000)) as path
|
||||
FROM categories c
|
||||
WHERE c.parent_id IS NULL
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
c.parent_id,
|
||||
CONCAT(cp.path, ' > ', c.name)
|
||||
FROM categories c
|
||||
JOIN category_path cp ON c.parent_id = cp.cat_id
|
||||
),
|
||||
product_thresholds AS (
|
||||
SELECT
|
||||
p.pid,
|
||||
COALESCE(
|
||||
(SELECT overstock_days FROM stock_thresholds st
|
||||
WHERE st.category_id IN (
|
||||
SELECT pc.cat_id
|
||||
FROM product_categories pc
|
||||
WHERE pc.pid = p.pid
|
||||
)
|
||||
AND (st.vendor = p.vendor OR st.vendor IS NULL)
|
||||
ORDER BY st.vendor IS NULL
|
||||
LIMIT 1),
|
||||
(SELECT overstock_days FROM stock_thresholds st
|
||||
WHERE st.category_id IS NULL
|
||||
AND (st.vendor = p.vendor OR st.vendor IS NULL)
|
||||
ORDER BY st.vendor IS NULL
|
||||
LIMIT 1),
|
||||
90
|
||||
) as target_days
|
||||
FROM products p
|
||||
),
|
||||
product_leaf_categories AS (
|
||||
-- Find categories that aren't parents to other categories for this product
|
||||
SELECT DISTINCT pc.cat_id
|
||||
FROM product_categories pc
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1
|
||||
FROM categories child
|
||||
JOIN product_categories child_pc ON child.cat_id = child_pc.cat_id
|
||||
WHERE child.parent_id = pc.cat_id
|
||||
AND child_pc.pid = pc.pid
|
||||
)
|
||||
)
|
||||
SELECT
|
||||
p.*,
|
||||
COALESCE(p.brand, 'Unbranded') as brand,
|
||||
GROUP_CONCAT(DISTINCT CONCAT(c.cat_id, ':', c.name)) as categories,
|
||||
pm.daily_sales_avg,
|
||||
pm.weekly_sales_avg,
|
||||
pm.monthly_sales_avg,
|
||||
pm.avg_quantity_per_order,
|
||||
pm.number_of_orders,
|
||||
pm.first_sale_date,
|
||||
pm.last_sale_date,
|
||||
pm.days_of_inventory,
|
||||
pm.weeks_of_inventory,
|
||||
pm.reorder_point,
|
||||
pm.safety_stock,
|
||||
pm.avg_margin_percent,
|
||||
CAST(pm.total_revenue AS DECIMAL(15,3)) as total_revenue,
|
||||
CAST(pm.inventory_value AS DECIMAL(15,3)) as inventory_value,
|
||||
CAST(pm.cost_of_goods_sold AS DECIMAL(15,3)) as cost_of_goods_sold,
|
||||
CAST(pm.gross_profit AS DECIMAL(15,3)) as gross_profit,
|
||||
pm.gmroi,
|
||||
pm.avg_lead_time_days,
|
||||
pm.last_purchase_date,
|
||||
pm.last_received_date,
|
||||
pm.abc_class,
|
||||
pm.stock_status,
|
||||
pm.turnover_rate,
|
||||
pm.current_lead_time,
|
||||
pm.target_lead_time,
|
||||
pm.lead_time_status,
|
||||
pm.reorder_qty,
|
||||
pm.overstocked_amt,
|
||||
COALESCE(pm.days_of_inventory / NULLIF(pt.target_days, 0), 0) as stock_coverage_ratio
|
||||
FROM products p
|
||||
LEFT JOIN product_metrics pm ON p.pid = pm.pid
|
||||
LEFT JOIN product_categories pc ON p.pid = pc.pid
|
||||
LEFT JOIN categories c ON pc.cat_id = c.cat_id
|
||||
LEFT JOIN product_thresholds pt ON p.pid = pt.pid
|
||||
JOIN product_leaf_categories plc ON c.cat_id = plc.cat_id
|
||||
${whereClause ? 'WHERE ' + whereClause.substring(6) : ''}
|
||||
GROUP BY p.pid
|
||||
ORDER BY ${sortColumn} ${sortDirection}
|
||||
LIMIT ? OFFSET ?
|
||||
`;
|
||||
|
||||
// Add pagination params to the main query params
|
||||
const queryParams = [...params, limit, offset];
|
||||
console.log('Query:', query.replace(/\s+/g, ' '));
|
||||
console.log('Params:', queryParams);
|
||||
|
||||
const [rows] = await pool.query(query, queryParams);
|
||||
|
||||
// Transform the results
|
||||
const products = rows.map(row => ({
|
||||
...row,
|
||||
categories: row.categories ? row.categories.split(',') : [],
|
||||
price: parseFloat(row.price),
|
||||
cost_price: parseFloat(row.cost_price),
|
||||
landing_cost_price: row.landing_cost_price ? parseFloat(row.landing_cost_price) : null,
|
||||
stock_quantity: parseInt(row.stock_quantity),
|
||||
daily_sales_avg: parseFloat(row.daily_sales_avg) || 0,
|
||||
weekly_sales_avg: parseFloat(row.weekly_sales_avg) || 0,
|
||||
monthly_sales_avg: parseFloat(row.monthly_sales_avg) || 0,
|
||||
avg_quantity_per_order: parseFloat(row.avg_quantity_per_order) || 0,
|
||||
number_of_orders: parseInt(row.number_of_orders) || 0,
|
||||
first_sale_date: row.first_sale_date || null,
|
||||
last_sale_date: row.last_sale_date || null,
|
||||
days_of_inventory: parseFloat(row.days_of_inventory) || 0,
|
||||
weeks_of_inventory: parseFloat(row.weeks_of_inventory) || 0,
|
||||
reorder_point: parseFloat(row.reorder_point) || 0,
|
||||
safety_stock: parseFloat(row.safety_stock) || 0,
|
||||
avg_margin_percent: parseFloat(row.avg_margin_percent) || 0,
|
||||
total_revenue: parseFloat(row.total_revenue) || 0,
|
||||
inventory_value: parseFloat(row.inventory_value) || 0,
|
||||
cost_of_goods_sold: parseFloat(row.cost_of_goods_sold) || 0,
|
||||
gross_profit: parseFloat(row.gross_profit) || 0,
|
||||
gmroi: parseFloat(row.gmroi) || 0,
|
||||
avg_lead_time_days: parseFloat(row.avg_lead_time_days) || 0,
|
||||
last_purchase_date: row.last_purchase_date || null,
|
||||
last_received_date: row.last_received_date || null,
|
||||
abc_class: row.abc_class || null,
|
||||
stock_status: row.stock_status || null,
|
||||
turnover_rate: parseFloat(row.turnover_rate) || 0,
|
||||
current_lead_time: parseFloat(row.current_lead_time) || 0,
|
||||
target_lead_time: parseFloat(row.target_lead_time) || 0,
|
||||
lead_time_status: row.lead_time_status || null,
|
||||
stock_coverage_ratio: parseFloat(row.stock_coverage_ratio) || 0,
|
||||
reorder_qty: parseInt(row.reorder_qty) || 0,
|
||||
overstocked_amt: parseInt(row.overstocked_amt) || 0
|
||||
}));
|
||||
|
||||
res.json({
|
||||
products,
|
||||
pagination: {
|
||||
total,
|
||||
currentPage: page,
|
||||
pages: Math.ceil(total / limit),
|
||||
limit
|
||||
},
|
||||
filters: {
|
||||
categories: categories.map(category => category.name),
|
||||
vendors: vendors.map(vendor => vendor.vendor),
|
||||
brands: brands.map(brand => brand.brand)
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error fetching products:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch products' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get trending products
|
||||
router.get('/trending', async (req, res) => {
|
||||
const pool = req.app.locals.pool;
|
||||
try {
|
||||
// First check if we have any data
|
||||
const [checkData] = await pool.query(`
|
||||
SELECT COUNT(*) as count,
|
||||
MAX(total_revenue) as max_revenue,
|
||||
MAX(daily_sales_avg) as max_daily_sales,
|
||||
COUNT(DISTINCT pid) as products_with_metrics
|
||||
FROM product_metrics
|
||||
WHERE total_revenue > 0 OR daily_sales_avg > 0
|
||||
`);
|
||||
console.log('Product metrics stats:', checkData[0]);
|
||||
|
||||
if (checkData[0].count === 0) {
|
||||
console.log('No products with metrics found');
|
||||
return res.json([]);
|
||||
}
|
||||
|
||||
// Get trending products
|
||||
const [rows] = await pool.query(`
|
||||
SELECT
|
||||
p.pid,
|
||||
p.sku,
|
||||
p.title,
|
||||
COALESCE(pm.daily_sales_avg, 0) as daily_sales_avg,
|
||||
COALESCE(pm.weekly_sales_avg, 0) as weekly_sales_avg,
|
||||
CASE
|
||||
WHEN pm.weekly_sales_avg > 0 AND pm.daily_sales_avg > 0
|
||||
THEN ((pm.daily_sales_avg - pm.weekly_sales_avg) / pm.weekly_sales_avg) * 100
|
||||
ELSE 0
|
||||
END as growth_rate,
|
||||
COALESCE(pm.total_revenue, 0) as total_revenue
|
||||
FROM products p
|
||||
INNER JOIN product_metrics pm ON p.pid = pm.pid
|
||||
WHERE (pm.total_revenue > 0 OR pm.daily_sales_avg > 0)
|
||||
AND p.visible = true
|
||||
ORDER BY growth_rate DESC
|
||||
LIMIT 50
|
||||
`);
|
||||
|
||||
console.log('Trending products:', rows);
|
||||
res.json(rows);
|
||||
} catch (error) {
|
||||
console.error('Error fetching trending products:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch trending products' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get a single product
|
||||
router.get('/:id', async (req, res) => {
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
const id = parseInt(req.params.id);
|
||||
|
||||
// Common CTE for category paths
|
||||
const categoryPathCTE = `
|
||||
WITH RECURSIVE category_path AS (
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
c.parent_id,
|
||||
CAST(c.name AS CHAR(1000)) as path
|
||||
FROM categories c
|
||||
WHERE c.parent_id IS NULL
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
c.parent_id,
|
||||
CONCAT(cp.path, ' > ', c.name)
|
||||
FROM categories c
|
||||
JOIN category_path cp ON c.parent_id = cp.cat_id
|
||||
)
|
||||
`;
|
||||
|
||||
// Get product details with category paths
|
||||
const [productRows] = await pool.query(`
|
||||
SELECT
|
||||
p.*,
|
||||
pm.daily_sales_avg,
|
||||
pm.weekly_sales_avg,
|
||||
pm.monthly_sales_avg,
|
||||
pm.days_of_inventory,
|
||||
pm.reorder_point,
|
||||
pm.safety_stock,
|
||||
pm.stock_status,
|
||||
pm.abc_class,
|
||||
pm.avg_margin_percent,
|
||||
pm.total_revenue,
|
||||
pm.inventory_value,
|
||||
pm.turnover_rate,
|
||||
pm.gmroi,
|
||||
pm.cost_of_goods_sold,
|
||||
pm.gross_profit,
|
||||
pm.avg_lead_time_days,
|
||||
pm.current_lead_time,
|
||||
pm.target_lead_time,
|
||||
pm.lead_time_status,
|
||||
pm.reorder_qty,
|
||||
pm.overstocked_amt
|
||||
FROM products p
|
||||
LEFT JOIN product_metrics pm ON p.pid = pm.pid
|
||||
WHERE p.pid = ?
|
||||
`, [id]);
|
||||
|
||||
if (!productRows.length) {
|
||||
return res.status(404).json({ error: 'Product not found' });
|
||||
}
|
||||
|
||||
// Get categories and their paths separately to avoid GROUP BY issues
|
||||
const [categoryRows] = await pool.query(`
|
||||
WITH RECURSIVE
|
||||
category_path AS (
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
c.parent_id,
|
||||
CAST(c.name AS CHAR(1000)) as path
|
||||
FROM categories c
|
||||
WHERE c.parent_id IS NULL
|
||||
|
||||
UNION ALL
|
||||
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name,
|
||||
c.parent_id,
|
||||
CONCAT(cp.path, ' > ', c.name)
|
||||
FROM categories c
|
||||
JOIN category_path cp ON c.parent_id = cp.cat_id
|
||||
),
|
||||
product_leaf_categories AS (
|
||||
-- Find categories assigned to this product that aren't parents
|
||||
-- of other categories assigned to this product
|
||||
SELECT pc.cat_id
|
||||
FROM product_categories pc
|
||||
WHERE pc.pid = ?
|
||||
AND NOT EXISTS (
|
||||
-- Check if there are any child categories also assigned to this product
|
||||
SELECT 1
|
||||
FROM categories child
|
||||
JOIN product_categories child_pc ON child.cat_id = child_pc.cat_id
|
||||
WHERE child.parent_id = pc.cat_id
|
||||
AND child_pc.pid = pc.pid
|
||||
)
|
||||
)
|
||||
SELECT
|
||||
c.cat_id,
|
||||
c.name as category_name,
|
||||
cp.path as full_path
|
||||
FROM product_categories pc
|
||||
JOIN categories c ON pc.cat_id = c.cat_id
|
||||
JOIN category_path cp ON c.cat_id = cp.cat_id
|
||||
JOIN product_leaf_categories plc ON c.cat_id = plc.cat_id
|
||||
WHERE pc.pid = ?
|
||||
ORDER BY cp.path
|
||||
`, [id, id]);
|
||||
|
||||
// Transform the results
|
||||
const categoryPathMap = categoryRows.reduce((acc, row) => {
|
||||
// Use cat_id in the key to differentiate categories with the same name
|
||||
acc[`${row.cat_id}:${row.category_name}`] = row.full_path;
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
const product = {
|
||||
...productRows[0],
|
||||
// Include cat_id in categories array to match the keys in categoryPathMap
|
||||
categories: categoryRows.map(row => `${row.cat_id}:${row.category_name}`),
|
||||
category_paths: categoryPathMap,
|
||||
price: parseFloat(productRows[0].price),
|
||||
regular_price: parseFloat(productRows[0].regular_price),
|
||||
cost_price: parseFloat(productRows[0].cost_price),
|
||||
landing_cost_price: parseFloat(productRows[0].landing_cost_price),
|
||||
stock_quantity: parseInt(productRows[0].stock_quantity),
|
||||
moq: parseInt(productRows[0].moq),
|
||||
uom: parseInt(productRows[0].uom),
|
||||
managing_stock: Boolean(productRows[0].managing_stock),
|
||||
replenishable: Boolean(productRows[0].replenishable),
|
||||
daily_sales_avg: parseFloat(productRows[0].daily_sales_avg) || 0,
|
||||
weekly_sales_avg: parseFloat(productRows[0].weekly_sales_avg) || 0,
|
||||
monthly_sales_avg: parseFloat(productRows[0].monthly_sales_avg) || 0,
|
||||
avg_quantity_per_order: parseFloat(productRows[0].avg_quantity_per_order) || 0,
|
||||
number_of_orders: parseInt(productRows[0].number_of_orders) || 0,
|
||||
first_sale_date: productRows[0].first_sale_date || null,
|
||||
last_sale_date: productRows[0].last_sale_date || null,
|
||||
days_of_inventory: parseFloat(productRows[0].days_of_inventory) || 0,
|
||||
weeks_of_inventory: parseFloat(productRows[0].weeks_of_inventory) || 0,
|
||||
reorder_point: parseFloat(productRows[0].reorder_point) || 0,
|
||||
safety_stock: parseFloat(productRows[0].safety_stock) || 0,
|
||||
avg_margin_percent: parseFloat(productRows[0].avg_margin_percent) || 0,
|
||||
total_revenue: parseFloat(productRows[0].total_revenue) || 0,
|
||||
inventory_value: parseFloat(productRows[0].inventory_value) || 0,
|
||||
cost_of_goods_sold: parseFloat(productRows[0].cost_of_goods_sold) || 0,
|
||||
gross_profit: parseFloat(productRows[0].gross_profit) || 0,
|
||||
gmroi: parseFloat(productRows[0].gmroi) || 0,
|
||||
avg_lead_time_days: parseFloat(productRows[0].avg_lead_time_days) || 0,
|
||||
current_lead_time: parseFloat(productRows[0].current_lead_time) || 0,
|
||||
target_lead_time: parseFloat(productRows[0].target_lead_time) || 0,
|
||||
lead_time_status: productRows[0].lead_time_status || null,
|
||||
reorder_qty: parseInt(productRows[0].reorder_qty) || 0,
|
||||
overstocked_amt: parseInt(productRows[0].overstocked_amt) || 0
|
||||
};
|
||||
|
||||
res.json(product);
|
||||
} catch (error) {
|
||||
console.error('Error fetching product:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch product' });
|
||||
}
|
||||
});
|
||||
|
||||
// Import products from CSV
|
||||
router.post('/import', upload.single('file'), async (req, res) => {
|
||||
if (!req.file) {
|
||||
return res.status(400).json({ error: 'No file uploaded' });
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await importProductsFromCSV(req.file.path, req.app.locals.pool);
|
||||
// Clean up the uploaded file
|
||||
require('fs').unlinkSync(req.file.path);
|
||||
res.json(result);
|
||||
} catch (error) {
|
||||
console.error('Error importing products:', error);
|
||||
res.status(500).json({ error: 'Failed to import products' });
|
||||
}
|
||||
});
|
||||
|
||||
// Update a product
|
||||
router.put('/:id', async (req, res) => {
|
||||
const pool = req.app.locals.pool;
|
||||
try {
|
||||
const {
|
||||
title,
|
||||
sku,
|
||||
stock_quantity,
|
||||
price,
|
||||
regular_price,
|
||||
cost_price,
|
||||
vendor,
|
||||
brand,
|
||||
categories,
|
||||
visible,
|
||||
managing_stock
|
||||
} = req.body;
|
||||
|
||||
const [result] = await pool.query(
|
||||
`UPDATE products
|
||||
SET title = ?,
|
||||
sku = ?,
|
||||
stock_quantity = ?,
|
||||
price = ?,
|
||||
regular_price = ?,
|
||||
cost_price = ?,
|
||||
vendor = ?,
|
||||
brand = ?,
|
||||
categories = ?,
|
||||
visible = ?,
|
||||
managing_stock = ?
|
||||
WHERE pid = ?`,
|
||||
[
|
||||
title,
|
||||
sku,
|
||||
stock_quantity,
|
||||
price,
|
||||
regular_price,
|
||||
cost_price,
|
||||
vendor,
|
||||
brand,
|
||||
categories,
|
||||
visible,
|
||||
managing_stock,
|
||||
req.params.id
|
||||
]
|
||||
);
|
||||
|
||||
if (result.affectedRows === 0) {
|
||||
return res.status(404).json({ error: 'Product not found' });
|
||||
}
|
||||
|
||||
res.json({ message: 'Product updated successfully' });
|
||||
} catch (error) {
|
||||
console.error('Error updating product:', error);
|
||||
res.status(500).json({ error: 'Failed to update product' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get product metrics
|
||||
router.get('/:id/metrics', async (req, res) => {
|
||||
const pool = req.app.locals.pool;
|
||||
try {
|
||||
const { id } = req.params;
|
||||
|
||||
// Get metrics from product_metrics table with inventory health data
|
||||
const [metrics] = await pool.query(`
|
||||
WITH inventory_status AS (
|
||||
SELECT
|
||||
p.pid,
|
||||
CASE
|
||||
WHEN pm.daily_sales_avg = 0 THEN 'New'
|
||||
WHEN p.stock_quantity <= CEIL(pm.daily_sales_avg * 7) THEN 'Critical'
|
||||
WHEN p.stock_quantity <= CEIL(pm.daily_sales_avg * 14) THEN 'Reorder'
|
||||
WHEN p.stock_quantity > (pm.daily_sales_avg * 90) THEN 'Overstocked'
|
||||
ELSE 'Healthy'
|
||||
END as calculated_status
|
||||
FROM products p
|
||||
LEFT JOIN product_metrics pm ON p.pid = pm.pid
|
||||
WHERE p.pid = ?
|
||||
)
|
||||
SELECT
|
||||
COALESCE(pm.daily_sales_avg, 0) as daily_sales_avg,
|
||||
COALESCE(pm.weekly_sales_avg, 0) as weekly_sales_avg,
|
||||
COALESCE(pm.monthly_sales_avg, 0) as monthly_sales_avg,
|
||||
COALESCE(pm.days_of_inventory, 0) as days_of_inventory,
|
||||
COALESCE(pm.reorder_point, CEIL(COALESCE(pm.daily_sales_avg, 0) * 14)) as reorder_point,
|
||||
COALESCE(pm.safety_stock, CEIL(COALESCE(pm.daily_sales_avg, 0) * 7)) as safety_stock,
|
||||
COALESCE(pm.avg_margin_percent,
|
||||
((p.price - COALESCE(p.cost_price, 0)) / NULLIF(p.price, 0)) * 100
|
||||
) as avg_margin_percent,
|
||||
COALESCE(pm.total_revenue, 0) as total_revenue,
|
||||
COALESCE(pm.inventory_value, p.stock_quantity * COALESCE(p.cost_price, 0)) as inventory_value,
|
||||
COALESCE(pm.turnover_rate, 0) as turnover_rate,
|
||||
COALESCE(pm.abc_class, 'C') as abc_class,
|
||||
COALESCE(pm.stock_status, is.calculated_status) as stock_status,
|
||||
COALESCE(pm.avg_lead_time_days, 0) as avg_lead_time_days,
|
||||
COALESCE(pm.current_lead_time, 0) as current_lead_time,
|
||||
COALESCE(pm.target_lead_time, 14) as target_lead_time,
|
||||
COALESCE(pm.lead_time_status, 'Unknown') as lead_time_status,
|
||||
COALESCE(pm.reorder_qty, 0) as reorder_qty,
|
||||
COALESCE(pm.overstocked_amt, 0) as overstocked_amt
|
||||
FROM products p
|
||||
LEFT JOIN product_metrics pm ON p.pid = pm.pid
|
||||
LEFT JOIN inventory_status is ON p.pid = is.pid
|
||||
WHERE p.pid = ?
|
||||
`, [id]);
|
||||
|
||||
if (!metrics.length) {
|
||||
// Return default metrics structure if no data found
|
||||
res.json({
|
||||
daily_sales_avg: 0,
|
||||
weekly_sales_avg: 0,
|
||||
monthly_sales_avg: 0,
|
||||
days_of_inventory: 0,
|
||||
reorder_point: 0,
|
||||
safety_stock: 0,
|
||||
avg_margin_percent: 0,
|
||||
total_revenue: 0,
|
||||
inventory_value: 0,
|
||||
turnover_rate: 0,
|
||||
abc_class: 'C',
|
||||
stock_status: 'New',
|
||||
avg_lead_time_days: 0,
|
||||
current_lead_time: 0,
|
||||
target_lead_time: 14,
|
||||
lead_time_status: 'Unknown',
|
||||
reorder_qty: 0,
|
||||
overstocked_amt: 0
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
res.json(metrics[0]);
|
||||
} catch (error) {
|
||||
console.error('Error fetching product metrics:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch product metrics' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get product time series data
|
||||
router.get('/:id/time-series', async (req, res) => {
|
||||
const { id } = req.params;
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
|
||||
// Get monthly sales data
|
||||
const [monthlySales] = await pool.query(`
|
||||
SELECT
|
||||
DATE_FORMAT(date, '%Y-%m') as month,
|
||||
COUNT(DISTINCT order_number) as order_count,
|
||||
SUM(quantity) as units_sold,
|
||||
CAST(SUM(price * quantity) AS DECIMAL(15,3)) as revenue
|
||||
FROM orders
|
||||
WHERE pid = ?
|
||||
AND canceled = false
|
||||
GROUP BY DATE_FORMAT(date, '%Y-%m')
|
||||
ORDER BY month DESC
|
||||
LIMIT 12
|
||||
`, [id]);
|
||||
|
||||
// Format monthly sales data
|
||||
const formattedMonthlySales = monthlySales.map(month => ({
|
||||
month: month.month,
|
||||
order_count: parseInt(month.order_count),
|
||||
units_sold: parseInt(month.units_sold),
|
||||
revenue: parseFloat(month.revenue),
|
||||
profit: 0 // Set to 0 since we don't have cost data in orders table
|
||||
}));
|
||||
|
||||
// Get recent orders
|
||||
const [recentOrders] = await pool.query(`
|
||||
SELECT
|
||||
DATE_FORMAT(date, '%Y-%m-%d') as date,
|
||||
order_number,
|
||||
quantity,
|
||||
price,
|
||||
discount,
|
||||
tax,
|
||||
shipping,
|
||||
customer_name as customer,
|
||||
status
|
||||
FROM orders
|
||||
WHERE pid = ?
|
||||
AND canceled = false
|
||||
ORDER BY date DESC
|
||||
LIMIT 10
|
||||
`, [id]);
|
||||
|
||||
// Get recent purchase orders with detailed status
|
||||
const [recentPurchases] = await pool.query(`
|
||||
SELECT
|
||||
DATE_FORMAT(date, '%Y-%m-%d') as date,
|
||||
DATE_FORMAT(expected_date, '%Y-%m-%d') as expected_date,
|
||||
DATE_FORMAT(received_date, '%Y-%m-%d') as received_date,
|
||||
po_id,
|
||||
ordered,
|
||||
received,
|
||||
status,
|
||||
receiving_status,
|
||||
cost_price,
|
||||
notes,
|
||||
CASE
|
||||
WHEN received_date IS NOT NULL THEN
|
||||
DATEDIFF(received_date, date)
|
||||
WHEN expected_date < CURDATE() AND status < ${PurchaseOrderStatus.ReceivingStarted} THEN
|
||||
DATEDIFF(CURDATE(), expected_date)
|
||||
ELSE NULL
|
||||
END as lead_time_days
|
||||
FROM purchase_orders
|
||||
WHERE pid = ?
|
||||
AND status != ${PurchaseOrderStatus.Canceled}
|
||||
ORDER BY date DESC
|
||||
LIMIT 10
|
||||
`, [id]);
|
||||
|
||||
res.json({
|
||||
monthly_sales: formattedMonthlySales,
|
||||
recent_orders: recentOrders.map(order => ({
|
||||
...order,
|
||||
price: parseFloat(order.price),
|
||||
discount: parseFloat(order.discount),
|
||||
tax: parseFloat(order.tax),
|
||||
shipping: parseFloat(order.shipping),
|
||||
quantity: parseInt(order.quantity)
|
||||
})),
|
||||
recent_purchases: recentPurchases.map(po => ({
|
||||
...po,
|
||||
ordered: parseInt(po.ordered),
|
||||
received: parseInt(po.received),
|
||||
status: parseInt(po.status),
|
||||
receiving_status: parseInt(po.receiving_status),
|
||||
cost_price: parseFloat(po.cost_price),
|
||||
lead_time_days: po.lead_time_days ? parseInt(po.lead_time_days) : null
|
||||
}))
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error fetching product time series:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch product time series' });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -1,424 +0,0 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
|
||||
// Status code constants
|
||||
const STATUS = {
|
||||
CANCELED: 0,
|
||||
CREATED: 1,
|
||||
ELECTRONICALLY_READY_SEND: 10,
|
||||
ORDERED: 11,
|
||||
PREORDERED: 12,
|
||||
ELECTRONICALLY_SENT: 13,
|
||||
RECEIVING_STARTED: 15,
|
||||
DONE: 50
|
||||
};
|
||||
|
||||
const RECEIVING_STATUS = {
|
||||
CANCELED: 0,
|
||||
CREATED: 1,
|
||||
PARTIAL_RECEIVED: 30,
|
||||
FULL_RECEIVED: 40,
|
||||
PAID: 50
|
||||
};
|
||||
|
||||
// Get all purchase orders with summary metrics
|
||||
router.get('/', async (req, res) => {
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
const { search, status, vendor, startDate, endDate, page = 1, limit = 100, sortColumn = 'date', sortDirection = 'desc' } = req.query;
|
||||
|
||||
let whereClause = '1=1';
|
||||
const params = [];
|
||||
|
||||
if (search) {
|
||||
whereClause += ' AND (po.po_id LIKE ? OR po.vendor LIKE ?)';
|
||||
params.push(`%${search}%`, `%${search}%`);
|
||||
}
|
||||
|
||||
if (status && status !== 'all') {
|
||||
whereClause += ' AND po.status = ?';
|
||||
params.push(Number(status));
|
||||
}
|
||||
|
||||
if (vendor && vendor !== 'all') {
|
||||
whereClause += ' AND po.vendor = ?';
|
||||
params.push(vendor);
|
||||
}
|
||||
|
||||
if (startDate) {
|
||||
whereClause += ' AND po.date >= ?';
|
||||
params.push(startDate);
|
||||
}
|
||||
|
||||
if (endDate) {
|
||||
whereClause += ' AND po.date <= ?';
|
||||
params.push(endDate);
|
||||
}
|
||||
|
||||
// Get filtered summary metrics
|
||||
const [summary] = await pool.query(`
|
||||
WITH po_totals AS (
|
||||
SELECT
|
||||
po_id,
|
||||
SUM(ordered) as total_ordered,
|
||||
SUM(received) as total_received,
|
||||
CAST(SUM(ordered * cost_price) AS DECIMAL(15,3)) as total_cost
|
||||
FROM purchase_orders po
|
||||
WHERE ${whereClause}
|
||||
GROUP BY po_id
|
||||
)
|
||||
SELECT
|
||||
COUNT(DISTINCT po_id) as order_count,
|
||||
SUM(total_ordered) as total_ordered,
|
||||
SUM(total_received) as total_received,
|
||||
ROUND(
|
||||
SUM(total_received) / NULLIF(SUM(total_ordered), 0), 3
|
||||
) as fulfillment_rate,
|
||||
CAST(SUM(total_cost) AS DECIMAL(15,3)) as total_value,
|
||||
CAST(AVG(total_cost) AS DECIMAL(15,3)) as avg_cost
|
||||
FROM po_totals
|
||||
`, params);
|
||||
|
||||
// Get total count for pagination
|
||||
const [countResult] = await pool.query(`
|
||||
SELECT COUNT(DISTINCT po_id) as total
|
||||
FROM purchase_orders po
|
||||
WHERE ${whereClause}
|
||||
`, params);
|
||||
|
||||
const total = countResult[0].total;
|
||||
const offset = (page - 1) * limit;
|
||||
const pages = Math.ceil(total / limit);
|
||||
|
||||
// Get recent purchase orders
|
||||
const [orders] = await pool.query(`
|
||||
WITH po_totals AS (
|
||||
SELECT
|
||||
po_id,
|
||||
vendor,
|
||||
date,
|
||||
status,
|
||||
receiving_status,
|
||||
COUNT(DISTINCT pid) as total_items,
|
||||
SUM(ordered) as total_quantity,
|
||||
CAST(SUM(ordered * cost_price) AS DECIMAL(15,3)) as total_cost,
|
||||
SUM(received) as total_received,
|
||||
ROUND(
|
||||
SUM(received) / NULLIF(SUM(ordered), 0), 3
|
||||
) as fulfillment_rate
|
||||
FROM purchase_orders po
|
||||
WHERE ${whereClause}
|
||||
GROUP BY po_id, vendor, date, status, receiving_status
|
||||
)
|
||||
SELECT
|
||||
po_id as id,
|
||||
vendor as vendor_name,
|
||||
DATE_FORMAT(date, '%Y-%m-%d') as order_date,
|
||||
status,
|
||||
receiving_status,
|
||||
total_items,
|
||||
total_quantity,
|
||||
total_cost,
|
||||
total_received,
|
||||
fulfillment_rate
|
||||
FROM po_totals
|
||||
ORDER BY
|
||||
CASE
|
||||
WHEN ? = 'order_date' THEN date
|
||||
WHEN ? = 'vendor_name' THEN vendor
|
||||
WHEN ? = 'total_cost' THEN CAST(total_cost AS DECIMAL(15,3))
|
||||
WHEN ? = 'total_received' THEN CAST(total_received AS DECIMAL(15,3))
|
||||
WHEN ? = 'total_items' THEN CAST(total_items AS SIGNED)
|
||||
WHEN ? = 'total_quantity' THEN CAST(total_quantity AS SIGNED)
|
||||
WHEN ? = 'fulfillment_rate' THEN CAST(fulfillment_rate AS DECIMAL(5,3))
|
||||
WHEN ? = 'status' THEN status
|
||||
ELSE date
|
||||
END ${sortDirection === 'desc' ? 'DESC' : 'ASC'}
|
||||
LIMIT ? OFFSET ?
|
||||
`, [...params, sortColumn, sortColumn, sortColumn, sortColumn, sortColumn, sortColumn, sortColumn, sortColumn, Number(limit), offset]);
|
||||
|
||||
// Get unique vendors for filter options
|
||||
const [vendors] = await pool.query(`
|
||||
SELECT DISTINCT vendor
|
||||
FROM purchase_orders
|
||||
WHERE vendor IS NOT NULL AND vendor != ''
|
||||
ORDER BY vendor
|
||||
`);
|
||||
|
||||
// Get unique statuses for filter options
|
||||
const [statuses] = await pool.query(`
|
||||
SELECT DISTINCT status
|
||||
FROM purchase_orders
|
||||
WHERE status IS NOT NULL
|
||||
ORDER BY status
|
||||
`);
|
||||
|
||||
// Parse numeric values
|
||||
const parsedOrders = orders.map(order => ({
|
||||
id: order.id,
|
||||
vendor_name: order.vendor_name,
|
||||
order_date: order.order_date,
|
||||
status: Number(order.status),
|
||||
receiving_status: Number(order.receiving_status),
|
||||
total_items: Number(order.total_items) || 0,
|
||||
total_quantity: Number(order.total_quantity) || 0,
|
||||
total_cost: Number(order.total_cost) || 0,
|
||||
total_received: Number(order.total_received) || 0,
|
||||
fulfillment_rate: Number(order.fulfillment_rate) || 0
|
||||
}));
|
||||
|
||||
// Parse summary metrics
|
||||
const parsedSummary = {
|
||||
order_count: Number(summary[0].order_count) || 0,
|
||||
total_ordered: Number(summary[0].total_ordered) || 0,
|
||||
total_received: Number(summary[0].total_received) || 0,
|
||||
fulfillment_rate: Number(summary[0].fulfillment_rate) || 0,
|
||||
total_value: Number(summary[0].total_value) || 0,
|
||||
avg_cost: Number(summary[0].avg_cost) || 0
|
||||
};
|
||||
|
||||
res.json({
|
||||
orders: parsedOrders,
|
||||
summary: parsedSummary,
|
||||
pagination: {
|
||||
total,
|
||||
pages,
|
||||
page: Number(page),
|
||||
limit: Number(limit)
|
||||
},
|
||||
filters: {
|
||||
vendors: vendors.map(v => v.vendor),
|
||||
statuses: statuses.map(s => Number(s.status))
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error fetching purchase orders:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch purchase orders' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get vendor performance metrics
|
||||
router.get('/vendor-metrics', async (req, res) => {
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
|
||||
const [metrics] = await pool.query(`
|
||||
WITH delivery_metrics AS (
|
||||
SELECT
|
||||
vendor,
|
||||
po_id,
|
||||
ordered,
|
||||
received,
|
||||
cost_price,
|
||||
CASE
|
||||
WHEN status >= ${STATUS.RECEIVING_STARTED} AND receiving_status >= ${RECEIVING_STATUS.PARTIAL_RECEIVED}
|
||||
AND received_date IS NOT NULL AND date IS NOT NULL
|
||||
THEN DATEDIFF(received_date, date)
|
||||
ELSE NULL
|
||||
END as delivery_days
|
||||
FROM purchase_orders
|
||||
WHERE vendor IS NOT NULL AND vendor != ''
|
||||
AND status != ${STATUS.CANCELED} -- Exclude canceled orders
|
||||
)
|
||||
SELECT
|
||||
vendor as vendor_name,
|
||||
COUNT(DISTINCT po_id) as total_orders,
|
||||
SUM(ordered) as total_ordered,
|
||||
SUM(received) as total_received,
|
||||
ROUND(
|
||||
SUM(received) / NULLIF(SUM(ordered), 0), 3
|
||||
) as fulfillment_rate,
|
||||
CAST(ROUND(
|
||||
SUM(ordered * cost_price) / NULLIF(SUM(ordered), 0), 2
|
||||
) AS DECIMAL(15,3)) as avg_unit_cost,
|
||||
CAST(SUM(ordered * cost_price) AS DECIMAL(15,3)) as total_spend,
|
||||
ROUND(
|
||||
AVG(NULLIF(delivery_days, 0)), 1
|
||||
) as avg_delivery_days
|
||||
FROM delivery_metrics
|
||||
GROUP BY vendor
|
||||
HAVING total_orders > 0
|
||||
ORDER BY total_spend DESC
|
||||
`);
|
||||
|
||||
// Parse numeric values
|
||||
const parsedMetrics = metrics.map(vendor => ({
|
||||
id: vendor.vendor_name,
|
||||
vendor_name: vendor.vendor_name,
|
||||
total_orders: Number(vendor.total_orders) || 0,
|
||||
total_ordered: Number(vendor.total_ordered) || 0,
|
||||
total_received: Number(vendor.total_received) || 0,
|
||||
fulfillment_rate: Number(vendor.fulfillment_rate) || 0,
|
||||
avg_unit_cost: Number(vendor.avg_unit_cost) || 0,
|
||||
total_spend: Number(vendor.total_spend) || 0,
|
||||
avg_delivery_days: vendor.avg_delivery_days === null ? null : Number(vendor.avg_delivery_days)
|
||||
}));
|
||||
|
||||
res.json(parsedMetrics);
|
||||
} catch (error) {
|
||||
console.error('Error fetching vendor metrics:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch vendor metrics' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get cost analysis
|
||||
router.get('/cost-analysis', async (req, res) => {
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
|
||||
const [analysis] = await pool.query(`
|
||||
WITH category_costs AS (
|
||||
SELECT
|
||||
c.name as category,
|
||||
po.pid,
|
||||
po.cost_price,
|
||||
po.ordered,
|
||||
po.received,
|
||||
po.status,
|
||||
po.receiving_status
|
||||
FROM purchase_orders po
|
||||
JOIN product_categories pc ON po.pid = pc.pid
|
||||
JOIN categories c ON pc.cat_id = c.cat_id
|
||||
WHERE po.status != ${STATUS.CANCELED} -- Exclude canceled orders
|
||||
)
|
||||
SELECT
|
||||
category,
|
||||
COUNT(DISTINCT pid) as unique_products,
|
||||
CAST(AVG(cost_price) AS DECIMAL(15,3)) as avg_cost,
|
||||
CAST(MIN(cost_price) AS DECIMAL(15,3)) as min_cost,
|
||||
CAST(MAX(cost_price) AS DECIMAL(15,3)) as max_cost,
|
||||
CAST(STDDEV(cost_price) AS DECIMAL(15,3)) as cost_variance,
|
||||
CAST(SUM(ordered * cost_price) AS DECIMAL(15,3)) as total_spend
|
||||
FROM category_costs
|
||||
GROUP BY category
|
||||
ORDER BY total_spend DESC
|
||||
`);
|
||||
|
||||
// Parse numeric values
|
||||
const parsedAnalysis = {
|
||||
categories: analysis.map(cat => ({
|
||||
category: cat.category,
|
||||
unique_products: Number(cat.unique_products) || 0,
|
||||
avg_cost: Number(cat.avg_cost) || 0,
|
||||
min_cost: Number(cat.min_cost) || 0,
|
||||
max_cost: Number(cat.max_cost) || 0,
|
||||
cost_variance: Number(cat.cost_variance) || 0,
|
||||
total_spend: Number(cat.total_spend) || 0
|
||||
}))
|
||||
};
|
||||
|
||||
res.json(parsedAnalysis);
|
||||
} catch (error) {
|
||||
console.error('Error fetching cost analysis:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch cost analysis' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get receiving status metrics
|
||||
router.get('/receiving-status', async (req, res) => {
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
|
||||
const [status] = await pool.query(`
|
||||
WITH po_totals AS (
|
||||
SELECT
|
||||
po_id,
|
||||
status,
|
||||
receiving_status,
|
||||
SUM(ordered) as total_ordered,
|
||||
SUM(received) as total_received,
|
||||
CAST(SUM(ordered * cost_price) AS DECIMAL(15,3)) as total_cost
|
||||
FROM purchase_orders
|
||||
WHERE status != ${STATUS.CANCELED}
|
||||
GROUP BY po_id, status, receiving_status
|
||||
)
|
||||
SELECT
|
||||
COUNT(DISTINCT po_id) as order_count,
|
||||
SUM(total_ordered) as total_ordered,
|
||||
SUM(total_received) as total_received,
|
||||
ROUND(
|
||||
SUM(total_received) / NULLIF(SUM(total_ordered), 0), 3
|
||||
) as fulfillment_rate,
|
||||
CAST(SUM(total_cost) AS DECIMAL(15,3)) as total_value,
|
||||
CAST(AVG(total_cost) AS DECIMAL(15,3)) as avg_cost,
|
||||
COUNT(DISTINCT CASE
|
||||
WHEN receiving_status = ${RECEIVING_STATUS.CREATED} THEN po_id
|
||||
END) as pending_count,
|
||||
COUNT(DISTINCT CASE
|
||||
WHEN receiving_status = ${RECEIVING_STATUS.PARTIAL_RECEIVED} THEN po_id
|
||||
END) as partial_count,
|
||||
COUNT(DISTINCT CASE
|
||||
WHEN receiving_status >= ${RECEIVING_STATUS.FULL_RECEIVED} THEN po_id
|
||||
END) as completed_count,
|
||||
COUNT(DISTINCT CASE
|
||||
WHEN receiving_status = ${RECEIVING_STATUS.CANCELED} THEN po_id
|
||||
END) as canceled_count
|
||||
FROM po_totals
|
||||
`);
|
||||
|
||||
// Parse numeric values
|
||||
const parsedStatus = {
|
||||
order_count: Number(status[0].order_count) || 0,
|
||||
total_ordered: Number(status[0].total_ordered) || 0,
|
||||
total_received: Number(status[0].total_received) || 0,
|
||||
fulfillment_rate: Number(status[0].fulfillment_rate) || 0,
|
||||
total_value: Number(status[0].total_value) || 0,
|
||||
avg_cost: Number(status[0].avg_cost) || 0,
|
||||
status_breakdown: {
|
||||
pending: Number(status[0].pending_count) || 0,
|
||||
partial: Number(status[0].partial_count) || 0,
|
||||
completed: Number(status[0].completed_count) || 0,
|
||||
canceled: Number(status[0].canceled_count) || 0
|
||||
}
|
||||
};
|
||||
|
||||
res.json(parsedStatus);
|
||||
} catch (error) {
|
||||
console.error('Error fetching receiving status:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch receiving status' });
|
||||
}
|
||||
});
|
||||
|
||||
// Get order vs received quantities by product
|
||||
router.get('/order-vs-received', async (req, res) => {
|
||||
try {
|
||||
const pool = req.app.locals.pool;
|
||||
|
||||
const [quantities] = await pool.query(`
|
||||
SELECT
|
||||
p.product_id,
|
||||
p.title as product,
|
||||
p.SKU as sku,
|
||||
SUM(po.ordered) as ordered_quantity,
|
||||
SUM(po.received) as received_quantity,
|
||||
ROUND(
|
||||
SUM(po.received) / NULLIF(SUM(po.ordered), 0) * 100, 1
|
||||
) as fulfillment_rate,
|
||||
COUNT(DISTINCT po.po_id) as order_count
|
||||
FROM products p
|
||||
JOIN purchase_orders po ON p.product_id = po.product_id
|
||||
WHERE po.date >= DATE_SUB(CURDATE(), INTERVAL 90 DAY)
|
||||
GROUP BY p.product_id, p.title, p.SKU
|
||||
HAVING order_count > 0
|
||||
ORDER BY ordered_quantity DESC
|
||||
LIMIT 20
|
||||
`);
|
||||
|
||||
// Parse numeric values and add id for React keys
|
||||
const parsedQuantities = quantities.map(q => ({
|
||||
id: q.product_id,
|
||||
...q,
|
||||
ordered_quantity: Number(q.ordered_quantity),
|
||||
received_quantity: Number(q.received_quantity),
|
||||
fulfillment_rate: Number(q.fulfillment_rate),
|
||||
order_count: Number(q.order_count)
|
||||
}));
|
||||
|
||||
res.json(parsedQuantities);
|
||||
} catch (error) {
|
||||
console.error('Error fetching order vs received quantities:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch order vs received quantities' });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -1,22 +0,0 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const { testConnection } = require('../../scripts/test-prod-connection');
|
||||
|
||||
router.get('/test-prod-connection', async (req, res) => {
|
||||
try {
|
||||
const productCount = await testConnection();
|
||||
res.json({
|
||||
success: true,
|
||||
message: 'Successfully connected to production database',
|
||||
productCount
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Production connection test failed:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message || 'Failed to connect to production database'
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -1,108 +0,0 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
|
||||
// Get vendors with pagination, filtering, and sorting
|
||||
router.get('/', async (req, res) => {
|
||||
const pool = req.app.locals.pool;
|
||||
try {
|
||||
// Get all vendors with metrics
|
||||
const [vendors] = await pool.query(`
|
||||
SELECT DISTINCT
|
||||
p.vendor as name,
|
||||
COALESCE(vm.active_products, 0) as active_products,
|
||||
COALESCE(vm.total_orders, 0) as total_orders,
|
||||
COALESCE(vm.avg_lead_time_days, 0) as avg_lead_time_days,
|
||||
COALESCE(vm.on_time_delivery_rate, 0) as on_time_delivery_rate,
|
||||
COALESCE(vm.order_fill_rate, 0) as order_fill_rate,
|
||||
CASE
|
||||
WHEN COALESCE(vm.total_orders, 0) > 0 AND COALESCE(vm.order_fill_rate, 0) >= 75 THEN 'active'
|
||||
WHEN COALESCE(vm.total_orders, 0) > 0 THEN 'inactive'
|
||||
ELSE 'pending'
|
||||
END as status
|
||||
FROM products p
|
||||
LEFT JOIN vendor_metrics vm ON p.vendor = vm.vendor
|
||||
WHERE p.vendor IS NOT NULL AND p.vendor != ''
|
||||
`);
|
||||
|
||||
// Get cost metrics for all vendors
|
||||
const vendorNames = vendors.map(v => v.name);
|
||||
const [costMetrics] = await pool.query(`
|
||||
SELECT
|
||||
vendor,
|
||||
CAST(ROUND(SUM(ordered * cost_price) / NULLIF(SUM(ordered), 0), 2) AS DECIMAL(15,3)) as avg_unit_cost,
|
||||
CAST(SUM(ordered * cost_price) AS DECIMAL(15,3)) as total_spend
|
||||
FROM purchase_orders
|
||||
WHERE status = 'closed'
|
||||
AND cost_price IS NOT NULL
|
||||
AND ordered > 0
|
||||
AND vendor IN (?)
|
||||
GROUP BY vendor
|
||||
`, [vendorNames]);
|
||||
|
||||
// Create a map of cost metrics by vendor
|
||||
const costMetricsMap = costMetrics.reduce((acc, curr) => {
|
||||
acc[curr.vendor] = {
|
||||
avg_unit_cost: curr.avg_unit_cost,
|
||||
total_spend: curr.total_spend
|
||||
};
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
// Get overall stats
|
||||
const [stats] = await pool.query(`
|
||||
SELECT
|
||||
COUNT(DISTINCT p.vendor) as totalVendors,
|
||||
COUNT(DISTINCT CASE
|
||||
WHEN COALESCE(vm.total_orders, 0) > 0 AND COALESCE(vm.order_fill_rate, 0) >= 75
|
||||
THEN p.vendor
|
||||
END) as activeVendors,
|
||||
COALESCE(ROUND(AVG(NULLIF(vm.avg_lead_time_days, 0)), 1), 0) as avgLeadTime,
|
||||
COALESCE(ROUND(AVG(NULLIF(vm.order_fill_rate, 0)), 1), 0) as avgFillRate,
|
||||
COALESCE(ROUND(AVG(NULLIF(vm.on_time_delivery_rate, 0)), 1), 0) as avgOnTimeDelivery
|
||||
FROM products p
|
||||
LEFT JOIN vendor_metrics vm ON p.vendor = vm.vendor
|
||||
WHERE p.vendor IS NOT NULL AND p.vendor != ''
|
||||
`);
|
||||
|
||||
// Get overall cost metrics
|
||||
const [overallCostMetrics] = await pool.query(`
|
||||
SELECT
|
||||
CAST(ROUND(SUM(ordered * cost_price) / NULLIF(SUM(ordered), 0), 2) AS DECIMAL(15,3)) as avg_unit_cost,
|
||||
CAST(SUM(ordered * cost_price) AS DECIMAL(15,3)) as total_spend
|
||||
FROM purchase_orders
|
||||
WHERE status = 'closed'
|
||||
AND cost_price IS NOT NULL
|
||||
AND ordered > 0
|
||||
AND vendor IS NOT NULL AND vendor != ''
|
||||
`);
|
||||
|
||||
res.json({
|
||||
vendors: vendors.map(vendor => ({
|
||||
vendor_id: vendor.name,
|
||||
name: vendor.name,
|
||||
status: vendor.status,
|
||||
avg_lead_time_days: parseFloat(vendor.avg_lead_time_days),
|
||||
on_time_delivery_rate: parseFloat(vendor.on_time_delivery_rate),
|
||||
order_fill_rate: parseFloat(vendor.order_fill_rate),
|
||||
total_orders: parseInt(vendor.total_orders),
|
||||
active_products: parseInt(vendor.active_products),
|
||||
avg_unit_cost: parseFloat(costMetricsMap[vendor.name]?.avg_unit_cost || 0),
|
||||
total_spend: parseFloat(costMetricsMap[vendor.name]?.total_spend || 0)
|
||||
})),
|
||||
stats: {
|
||||
totalVendors: parseInt(stats[0].totalVendors),
|
||||
activeVendors: parseInt(stats[0].activeVendors),
|
||||
avgLeadTime: parseFloat(stats[0].avgLeadTime),
|
||||
avgFillRate: parseFloat(stats[0].avgFillRate),
|
||||
avgOnTimeDelivery: parseFloat(stats[0].avgOnTimeDelivery),
|
||||
avgUnitCost: parseFloat(overallCostMetrics[0].avg_unit_cost),
|
||||
totalSpend: parseFloat(overallCostMetrics[0].total_spend)
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error fetching vendors:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch vendors' });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user