Compare commits
322 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 matt
|
cf71cc4dec | ||
|
matt
|
4be0f877fa | ||
|
matt
|
82e568d455 | ||
|
matt
|
1ab14ba45f | ||
|
matt
|
36f23b527e | ||
|
matt
|
c0f4f1de0d | ||
|
matt
|
38f4db3d15 | ||
|
matt
|
edfa86608c | ||
|
matt
|
8721ba67df | ||
|
matt
|
123946c159 | ||
|
matt
|
9ab5d4300a | ||
|
matt
|
338f829eb6 | ||
|
matt
|
c276f165f4 | ||
|
matt
|
4b2b3d5a9f | ||
|
matt
|
e43abdafd0 | ||
|
matt
|
54f8cc2706 | ||
|
matt
|
b95bd4a4a0 | ||
|
matt
|
407731e17d | ||
|
matt
|
e4f5e2c4dd | ||
|
matt
|
23b94d1c48 | ||
|
matt
|
9643cf191f | ||
|
matt
|
76a8836769 | ||
|
matt
|
884bcbad78 | ||
|
matt
|
f8b81d2111 | ||
|
matt
|
1b836567cd | ||
|
matt
|
39b8faa208 | ||
|
matt
|
177f7778b9 | ||
|
matt
|
f887dc6af1 | ||
|
matt
|
c344fdc3b8 | ||
|
matt
|
ebef903f3b | ||
|
matt
|
16d2399de8 | ||
|
matt
|
c3e09d5fd1 | ||
|
matt
|
bae8c575bc | ||
|
matt
|
45ded53530 | ||
|
matt
|
f41b5ab0f6 | ||
|
matt
|
6834a77a80 | ||
|
matt
|
38b12c188f | ||
|
matt
|
6aefc1b40d | ||
|
matt
|
7c41a7f799 | ||
|
matt
|
12cc7a4639 | ||
|
matt
|
9b2f9016f6 | ||
|
matt
|
8044771301 | ||
|
matt
|
b5469440bf | ||
|
matt
|
fd14af0f9e | ||
|
matt
|
a703019b0b | ||
|
matt
|
2744e82264 | ||
|
matt
|
450fd96e19 | ||
|
matt
|
4372dc5e26 | ||
|
matt
|
dd0e989669 | ||
|
matt
|
89d518b57f | ||
|
matt
|
ac39257a51 | ||
|
matt
|
003e1ddd61 | ||
|
matt
|
2dc8152b53 | ||
|
matt
|
01d4097030 | ||
|
matt
|
f9e8c9265e | ||
|
matt
|
ee2f314775 | ||
|
matt
|
11d0555eeb | ||
|
matt
|
ec8ab17d3f | ||
|
matt
|
100e398aae | ||
|
matt
|
aec02e490a | ||
|
matt
|
3831cef234 | ||
|
matt
|
1866cbae7e | ||
|
matt
|
3d1e8862f9 | ||
|
matt
|
1dcb47cfc5 | ||
|
matt
|
167c13c572 | ||
|
matt
|
7218e7cc3f | ||
|
matt
|
43d76e011d | ||
|
matt
|
9ce84fe5b9 | ||
|
matt
|
d15360a7d4 | ||
|
matt
|
630945e901 | ||
|
matt
|
54ddaa0492 | ||
|
matt
|
262890a7be | ||
|
matt
|
ef50aec33c | ||
|
matt
|
0ffd02e22e | ||
|
matt
|
738ed94ad5 | ||
|
matt
|
f5b2b4e421 | ||
|
matt
|
b81dfb9649 | ||
|
matt
|
9be0f34f07 | ||
|
matt
|
ad5b797ce6 | ||
|
matt
|
78932360d1 | ||
|
matt
|
217abd41af | ||
|
matt
|
d56beb5143 | ||
|
matt
|
0b5f3162c7 | ||
|
matt
|
72930bbc73 | ||
|
matt
|
0ceef144d7 | ||
|
matt
|
f0e2023803 | ||
|
matt
|
0a20d74bb6 | ||
|
matt
|
9761c29934 | ||
|
matt
|
e84c7e568f | ||
|
matt
|
4953355b91 | ||
|
matt
|
dadcf3b6c6 | ||
|
matt
|
920c33d119 | ||
|
matt
|
451d5f0b3b | ||
|
matt
|
dd79298b94 | ||
|
matt
|
7b7274f72c | ||
|
matt
|
60875c25a6 | ||
|
matt
|
e10df632d8 | ||
|
matt
|
945e4a8cc3 | ||
|
matt
|
c6e4fc9cff | ||
|
matt
|
ff17b290aa | ||
|
matt
|
6bffcfb0a4 | ||
|
matt
|
2c5255cd13 | ||
|
matt
|
1696ecf591 | ||
|
matt
|
dc774862a7 | ||
|
matt
|
d3e3cba087 | ||
|
matt
|
4ea3a4aec3 | ||
|
matt
|
a161f4533d | ||
|
matt
|
6e30ba60ff | ||
|
matt
|
138251cf86 | ||
|
matt
|
24aee1db90 | ||
|
matt
|
2fe7fd5b2f | ||
|
matt
|
d8b39979cd | ||
|
matt
|
4776a112b6 | ||
|
matt
|
2ff325a132 | ||
|
matt
|
5d46a2a7e5 | ||
|
matt
|
512b351429 | ||
|
matt
|
3991341376 | ||
|
matt
|
5833779c10 | ||
|
matt
|
c61115f665 | ||
|
matt
|
7da2b304b4 | ||
|
matt
|
4ccda8ad49 | ||
|
matt
|
88f703ec70 | ||
|
matt
|
ab998fb7c4 | ||
|
matt
|
faaa8cc47a | ||
|
matt
|
459c5092d2 | ||
|
matt
|
6c9fd062e9 | ||
|
matt
|
5d7d7a8671 | ||
|
matt
|
54f55b06a1 | ||
|
matt
|
4935cfe3bb | ||
|
matt
|
5e2ee73e2d | ||
|
matt
|
4dfe85231a | ||
|
matt
|
9e7aac836e | ||
|
matt
|
d35c7dd6cf | ||
|
matt
|
ad1ebeefe1 | ||
|
matt
|
a0c442d1af | ||
|
matt
|
7938c50762 | ||
|
matt
|
5dcd19e7f3 | ||
|
matt
|
075e7253a0 | ||
|
matt
|
763aa4f74b | ||
|
matt
|
520ff5bd74 | ||
|
matt
|
8496bbc4ee | ||
|
matt
|
38f6688f10 | ||
|
matt
|
fcfe7e2fab | ||
|
matt
|
2e3e81a02b | ||
|
matt
|
8606a90e34 | ||
|
matt
|
a97819f4a6 | ||
|
matt
|
dd82c624d8 | ||
|
matt
|
7999e1e64a | ||
|
matt
|
12a0f540b3 | ||
|
matt
|
e793cb0cc5 | ||
|
matt
|
b2330dee22 | ||
|
matt
|
00501704df | ||
|
matt
|
4cb41a7e4c | ||
|
matt
|
d05d27494d | ||
|
matt
|
4ed734e5c0 | ||
|
matt
|
1e3be5d4cb | ||
|
matt
|
8dd852dd6a | ||
|
matt
|
eeff5817ea | ||
|
matt
|
1b19feb172 | ||
|
matt
|
80ff8124ec | ||
|
matt
|
8508bfac93 | ||
|
matt
|
ac14179bd2 | ||
|
matt
|
00249f7c33 | ||
|
matt
|
f271f3aae4 | ||
|
matt
|
43f76e4ac0 | ||
|
matt
|
92ff80fba2 | ||
|
matt
|
a4c1a19d2e | ||
|
matt
|
c9b656d34b | ||
|
matt
|
d081a60662 | ||
|
matt
|
4021fe487d | ||
|
matt
|
4552fa4862 | ||
|
matt
|
2601a04211 | ||
|
matt
|
6051b849d6 | ||
|
matt
|
dbd0232285 | ||
|
matt
|
1b9f01d101 | ||
|
matt
|
a9dbbbf824 | ||
|
matt
|
97296946f1 | ||
|
matt
|
5035dda733 | ||
|
matt
|
796a2e5d1f | ||
|
matt
|
047122a620 | ||
|
matt
|
4c4359908c | ||
|
matt
|
54cc4be1e3 | ||
|
matt
|
f4854423ab | ||
|
matt
|
0796518e26 | ||
|
matt
|
7aa494aaad | ||
|
matt
|
1e0be3f86e | ||
|
matt
|
a068a253cd | ||
|
matt
|
087ec710f6 | ||
|
matt
|
957c7b5eb1 | ||
|
matt
|
8b8845b423 | ||
|
matt
|
e5c4f617c5 | ||
|
matt
|
8e19e6cd74 | ||
|
matt
|
749907bd30 | ||
|
matt
|
108181c63d | ||
|
matt
|
5dd779cb4a | ||
|
matt
|
7b0e792d03 | ||
|
matt
|
517bbe72f4 | ||
|
matt
|
87d4b9e804 | ||
|
matt
|
75da2c6772 | ||
|
matt
|
00a02aa788 | ||
|
matt
|
114018080a | ||
|
matt
|
228ae8b2a9 | ||
|
matt
|
dd4b3f7145 | ||
|
matt
|
7eb4077224 | ||
|
matt
|
d60a8cbc6e | ||
|
matt
|
1fcbf54989 | ||
|
matt
|
ce75496770 | ||
|
matt
|
7eae4a0b29 | ||
|
matt
|
f421154c1d | ||
|
matt
|
03dc119a15 | ||
|
matt
|
1963bee00c | ||
|
matt
|
387e7e5e73 | ||
|
matt
|
a51a48ce89 | ||
|
matt
|
aacb3a2fd0 | ||
|
matt
|
35d2f0df7c | ||
|
matt
|
7d46ebd6ba | ||
|
matt
|
1496aa57b1 | ||
|
matt
|
fc9ef2f0d7 | ||
|
matt
|
af067f7360 | ||
|
matt
|
949b543d1f | ||
|
matt
|
8fdb68fb19 | ||
|
matt
|
136f767309 | ||
|
matt
|
aa9664c459 | ||
|
matt
|
f60f0b1b5c | ||
|
matt
|
676cd44d9d | ||
|
matt
|
1d081bb218 | ||
|
matt
|
52ae7e10aa | ||
|
matt
|
153bbecc44 | ||
|
matt
|
cb46970808 | ||
|
matt
|
97fa7f3495 | ||
|
matt
|
a88dbb8486 | ||
|
matt
|
d0a83c04ca | ||
|
matt
|
f95c1f2d43 | ||
|
matt
|
0ef27a3229 | ||
|
matt
|
0f89373d11 | ||
|
matt
|
f55d35e301 | ||
|
matt
|
1aee18a025 | ||
|
matt
|
0068d77ad9 | ||
|
matt
|
b69182e2c7 | ||
|
matt
|
1c8709f520 | ||
|
matt
|
de1408bd58 | ||
|
matt
|
c295c330ff | ||
|
matt
|
7cc723ce83 | ||
|
matt
|
c3c48669ad | ||
|
matt
|
78a0018940 | ||
|
matt
|
851cc3c4cc | ||
|
matt
|
74454cdc7f | ||
|
matt
|
31c838197a | ||
|
matt
|
45fa583ce8 | ||
|
matt
|
c96f514bcd | ||
|
matt
|
6a5e6d2bfb | ||
|
matt
|
875d0b8f55 | ||
|
matt
|
b15387041b | ||
|
matt
|
60cdb1cee3 | ||
|
matt
|
52fd47a921 | ||
|
matt
|
b723ec3c0f | ||
|
matt
|
68ca7e93a1 | ||
|
matt
|
bc5607f48c | ||
|
matt
|
36a5186c17 | ||
|
matt
|
05bac73c45 | ||
|
matt
|
7a43428e76 | ||
|
matt
|
e21da8330e | ||
|
matt
|
56c3f0534d | ||
|
matt
|
98e3b89d46 | ||
|
matt
|
8271c9f95a | ||
|
matt
|
f7bdefb0a3 | ||
|
matt
|
e0a7787139 | ||
|
matt
|
c1159f518c | ||
|
matt
|
a19a8ba412 | ||
|
matt
|
bb455b3c37 | ||
|
matt
|
ca35a67e9f | ||
|
matt
|
88f1853b09 | ||
|
matt
|
3ca72674af | ||
|
matt
|
c185d4e3ca | ||
|
matt
|
2d62cac5f7 | ||
|
matt
|
e3361cf098 | ||
|
matt
|
41f7f33746 | ||
|
matt
|
8141fafb34 | ||
|
matt
|
42af434bd7 | ||
|
matt
|
fbb200c4ee | ||
|
matt
|
b96a9f412a | ||
|
matt
|
6b101a91f6 | ||
|
matt
|
2df5428712 | ||
|
matt
|
5d7e05172d | ||
|
matt
|
41058ff5c6 | ||
|
matt
|
54a87ca3dc | ||
|
matt
|
6bf93d33ea | ||
|
matt
|
441a2c74ad | ||
|
matt
|
f628774267 | ||
|
matt
|
3f16413769 | ||
|
matt
|
959a64aebc | ||
|
matt
|
694014934c | ||
|
matt
|
cff176e7a3 | ||
|
matt
|
7f7e6fdd1f | ||
|
matt
|
45a52cbc33 | ||
|
matt
|
bba7362641 | ||
|
matt
|
468f85c45d | ||
|
matt
|
24e2d01ccc | ||
|
matt
|
43d7775d08 | ||
|
matt
|
527dec4d49 | ||
|
matt
|
fe70b56d24 | ||
|
matt
|
ed62f03ba0 | ||
|
matt
|
e034e83198 | ||
|
matt
|
110f4ec332 | ||
|
matt
|
5bf265ed46 | ||
|
matt
|
528fe7c024 | ||
|
matt
|
08be0658cb | ||
|
matt
|
f823841b15 | ||
|
matt
|
9ce3793067 | ||
|
matt
|
89d4605577 | ||
|
matt
|
675a0fc374 | ||
|
matt
|
ca2653ea1a | ||
|
matt
|
a8d3fd8033 | ||
|
matt
|
702b956ff1 | ||
|
matt
|
9b8577f258 | ||
|
matt
|
9623681a15 | ||
|
matt
|
cc22fd8c35 | ||
|
matt
|
0ef1b6100e | ||
|
matt
|
a519746ccb | ||
|
matt
|
f29dd8ef8b | ||
|
matt
|
f2a5c06005 | ||
|
matt
|
fb9f959fe5 |
@@ -0,0 +1,41 @@
|
||||
# Details
|
||||
|
||||
Date : 2025-03-17 16:24:17
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 26 files, 6193 codes, 1008 comments, 1017 blanks, all 8218 lines
|
||||
|
||||
[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
|
||||
## Files
|
||||
| filename | language | code | comment | blank | total |
|
||||
| :--- | :--- | ---: | ---: | ---: | ---: |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md) | Markdown | 39 | 0 | 19 | 58 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx) | TypeScript JSX | 230 | 10 | 8 | 248 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx) | TypeScript JSX | 18 | 0 | 3 | 21 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx) | TypeScript JSX | 273 | 19 | 37 | 329 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx) | TypeScript JSX | 374 | 42 | 44 | 460 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx) | TypeScript JSX | 730 | 126 | 106 | 962 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx) | TypeScript JSX | 499 | 48 | 54 | 601 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx) | TypeScript JSX | 112 | 12 | 21 | 145 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx) | TypeScript JSX | 232 | 31 | 32 | 295 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx) | TypeScript JSX | 407 | 56 | 52 | 515 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx) | TypeScript JSX | 193 | 23 | 22 | 238 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx) | TypeScript JSX | 289 | 36 | 31 | 356 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx) | TypeScript JSX | 500 | 75 | 89 | 664 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx) | TypeScript JSX | 248 | 69 | 74 | 391 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx) | TypeScript JSX | 204 | 26 | 33 | 263 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx) | TypeScript JSX | 209 | 49 | 50 | 308 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx) | TypeScript JSX | 219 | 39 | 47 | 305 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx) | TypeScript JSX | 1,060 | 228 | 229 | 1,517 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx) | TypeScript JSX | 20 | 6 | 2 | 28 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts) | TypeScript | 4 | 0 | 1 | 5 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts) | TypeScript | 16 | 4 | 4 | 24 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts) | TypeScript | 124 | 4 | 14 | 142 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts) | TypeScript | 21 | 15 | 5 | 41 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts) | TypeScript | 43 | 24 | 7 | 74 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js) | JavaScript | 28 | 7 | 9 | 44 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts) | TypeScript | 101 | 59 | 24 | 184 |
|
||||
|
||||
[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
@@ -0,0 +1,20 @@
|
||||
# Diff Details
|
||||
|
||||
Date : 2025-03-17 16:24:17
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 5 files, -358 codes, -15 comments, -33 blanks, all -406 lines
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details
|
||||
|
||||
## Files
|
||||
| filename | language | code | comment | blank | total |
|
||||
| :--- | :--- | ---: | ---: | ---: | ---: |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SaveTemplateDialog.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SaveTemplateDialog.tsx) | TypeScript JSX | -83 | 0 | -4 | -87 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/TemplateManager.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/TemplateManager.tsx) | TypeScript JSX | -193 | -4 | -15 | -212 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx) | TypeScript JSX | -241 | -68 | -72 | -381 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useFilters.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useFilters.tsx) | TypeScript JSX | -89 | -12 | -16 | -117 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx) | TypeScript JSX | 248 | 69 | 74 | 391 |
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details
|
||||
@@ -0,0 +1,23 @@
|
||||
# Diff Summary
|
||||
|
||||
Date : 2025-03-17 16:24:17
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 5 files, -358 codes, -15 comments, -33 blanks, all -406 lines
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md)
|
||||
|
||||
## Languages
|
||||
| language | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| TypeScript JSX | 5 | -358 | -15 | -33 | -406 |
|
||||
|
||||
## Directories
|
||||
| path | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| . | 5 | -358 | -15 | -33 | -406 |
|
||||
| components | 3 | -517 | -72 | -91 | -680 |
|
||||
| hooks | 2 | 159 | 57 | 58 | 274 |
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md)
|
||||
@@ -0,0 +1,31 @@
|
||||
Date : 2025-03-17 16:24:17
|
||||
Directory : /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
Total : 5 files, -358 codes, -15 comments, -33 blanks, all -406 lines
|
||||
|
||||
Languages
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| language | files | code | comment | blank | total |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| TypeScript JSX | 5 | -358 | -15 | -33 | -406 |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Directories
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| path | files | code | comment | blank | total |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| . | 5 | -358 | -15 | -33 | -406 |
|
||||
| components | 3 | -517 | -72 | -91 | -680 |
|
||||
| hooks | 2 | 159 | 57 | 58 | 274 |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Files
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| filename | language | code | comment | blank | total |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SaveTemplateDialog.tsx | TypeScript JSX | -83 | 0 | -4 | -87 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/TemplateManager.tsx | TypeScript JSX | -193 | -4 | -15 | -212 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx | TypeScript JSX | -241 | -68 | -72 | -381 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useFilters.tsx | TypeScript JSX | -89 | -12 | -16 | -117 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx | TypeScript JSX | 248 | 69 | 74 | 391 |
|
||||
| Total | | -358 | -15 | -33 | -406 |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,31 @@
|
||||
# Summary
|
||||
|
||||
Date : 2025-03-17 16:24:17
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 26 files, 6193 codes, 1008 comments, 1017 blanks, all 8218 lines
|
||||
|
||||
Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
|
||||
## Languages
|
||||
| language | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| TypeScript JSX | 18 | 5,817 | 895 | 934 | 7,646 |
|
||||
| TypeScript | 6 | 309 | 106 | 55 | 470 |
|
||||
| Markdown | 1 | 39 | 0 | 19 | 58 |
|
||||
| JavaScript | 1 | 28 | 7 | 9 | 44 |
|
||||
|
||||
## Directories
|
||||
| path | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| . | 26 | 6,193 | 1,008 | 1,017 | 8,218 |
|
||||
| . (Files) | 3 | 63 | 6 | 22 | 91 |
|
||||
| components | 11 | 3,357 | 403 | 410 | 4,170 |
|
||||
| components (Files) | 6 | 2,124 | 245 | 252 | 2,621 |
|
||||
| components/cells | 5 | 1,233 | 158 | 158 | 1,549 |
|
||||
| hooks | 6 | 2,440 | 486 | 522 | 3,448 |
|
||||
| types | 1 | 16 | 4 | 4 | 24 |
|
||||
| utils | 5 | 317 | 109 | 59 | 485 |
|
||||
|
||||
Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
@@ -0,0 +1,60 @@
|
||||
Date : 2025-03-17 16:24:17
|
||||
Directory : /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
Total : 26 files, 6193 codes, 1008 comments, 1017 blanks, all 8218 lines
|
||||
|
||||
Languages
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| language | files | code | comment | blank | total |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| TypeScript JSX | 18 | 5,817 | 895 | 934 | 7,646 |
|
||||
| TypeScript | 6 | 309 | 106 | 55 | 470 |
|
||||
| Markdown | 1 | 39 | 0 | 19 | 58 |
|
||||
| JavaScript | 1 | 28 | 7 | 9 | 44 |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Directories
|
||||
+------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| path | files | code | comment | blank | total |
|
||||
+------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| . | 26 | 6,193 | 1,008 | 1,017 | 8,218 |
|
||||
| . (Files) | 3 | 63 | 6 | 22 | 91 |
|
||||
| components | 11 | 3,357 | 403 | 410 | 4,170 |
|
||||
| components (Files) | 6 | 2,124 | 245 | 252 | 2,621 |
|
||||
| components/cells | 5 | 1,233 | 158 | 158 | 1,549 |
|
||||
| hooks | 6 | 2,440 | 486 | 522 | 3,448 |
|
||||
| types | 1 | 16 | 4 | 4 | 24 |
|
||||
| utils | 5 | 317 | 109 | 59 | 485 |
|
||||
+------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Files
|
||||
+------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| filename | language | code | comment | blank | total |
|
||||
+------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md | Markdown | 39 | 0 | 19 | 58 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx | TypeScript JSX | 230 | 10 | 8 | 248 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx | TypeScript JSX | 18 | 0 | 3 | 21 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx | TypeScript JSX | 273 | 19 | 37 | 329 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx | TypeScript JSX | 374 | 42 | 44 | 460 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx | TypeScript JSX | 730 | 126 | 106 | 962 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx | TypeScript JSX | 499 | 48 | 54 | 601 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx | TypeScript JSX | 112 | 12 | 21 | 145 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx | TypeScript JSX | 232 | 31 | 32 | 295 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx | TypeScript JSX | 407 | 56 | 52 | 515 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx | TypeScript JSX | 193 | 23 | 22 | 238 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx | TypeScript JSX | 289 | 36 | 31 | 356 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx | TypeScript JSX | 500 | 75 | 89 | 664 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx | TypeScript JSX | 248 | 69 | 74 | 391 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx | TypeScript JSX | 204 | 26 | 33 | 263 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx | TypeScript JSX | 209 | 49 | 50 | 308 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx | TypeScript JSX | 219 | 39 | 47 | 305 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx | TypeScript JSX | 1,060 | 228 | 229 | 1,517 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx | TypeScript JSX | 20 | 6 | 2 | 28 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts | TypeScript | 4 | 0 | 1 | 5 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts | TypeScript | 16 | 4 | 4 | 24 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts | TypeScript | 124 | 4 | 14 | 142 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts | TypeScript | 21 | 15 | 5 | 41 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts | TypeScript | 43 | 24 | 7 | 74 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js | JavaScript | 28 | 7 | 9 | 44 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts | TypeScript | 101 | 59 | 24 | 184 |
|
||||
| Total | | 6,193 | 1,008 | 1,017 | 8,218 |
|
||||
+------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
@@ -0,0 +1,42 @@
|
||||
# Details
|
||||
|
||||
Date : 2025-03-18 12:39:04
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 27 files, 6925 codes, 1247 comments, 1248 blanks, all 9420 lines
|
||||
|
||||
[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
|
||||
## Files
|
||||
| filename | language | code | comment | blank | total |
|
||||
| :--- | :--- | ---: | ---: | ---: | ---: |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md) | Markdown | 39 | 0 | 19 | 58 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx) | TypeScript JSX | 230 | 10 | 8 | 248 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx) | TypeScript JSX | 18 | 0 | 3 | 21 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx) | TypeScript JSX | 273 | 19 | 37 | 329 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx) | TypeScript JSX | 113 | 17 | 10 | 140 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx) | TypeScript JSX | 377 | 49 | 54 | 480 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx) | TypeScript JSX | 969 | 182 | 158 | 1,309 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx) | TypeScript JSX | 509 | 50 | 57 | 616 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx) | TypeScript JSX | 112 | 12 | 21 | 145 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx) | TypeScript JSX | 233 | 34 | 33 | 300 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx) | TypeScript JSX | 420 | 66 | 59 | 545 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx) | TypeScript JSX | 193 | 23 | 22 | 238 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx) | TypeScript JSX | 227 | 36 | 32 | 295 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx) | TypeScript JSX | 500 | 75 | 89 | 664 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx) | TypeScript JSX | 264 | 75 | 81 | 420 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx) | TypeScript JSX | 204 | 26 | 33 | 263 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx) | TypeScript JSX | 337 | 88 | 92 | 517 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx) | TypeScript JSX | 360 | 78 | 85 | 523 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx) | TypeScript JSX | 1,190 | 288 | 289 | 1,767 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx) | TypeScript JSX | 20 | 6 | 2 | 28 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts) | TypeScript | 4 | 0 | 1 | 5 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts) | TypeScript | 16 | 4 | 4 | 24 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts) | TypeScript | 124 | 4 | 14 | 142 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts) | TypeScript | 21 | 15 | 5 | 41 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts) | TypeScript | 43 | 24 | 7 | 74 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js) | JavaScript | 28 | 7 | 9 | 44 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts) | TypeScript | 101 | 59 | 24 | 184 |
|
||||
|
||||
[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
@@ -0,0 +1,26 @@
|
||||
# Diff Details
|
||||
|
||||
Date : 2025-03-18 12:39:04
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 11 files, 732 codes, 239 comments, 231 blanks, all 1202 lines
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details
|
||||
|
||||
## Files
|
||||
| filename | language | code | comment | blank | total |
|
||||
| :--- | :--- | ---: | ---: | ---: | ---: |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx) | TypeScript JSX | 113 | 17 | 10 | 140 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx) | TypeScript JSX | 3 | 7 | 10 | 20 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx) | TypeScript JSX | 239 | 56 | 52 | 347 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx) | TypeScript JSX | 10 | 2 | 3 | 15 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx) | TypeScript JSX | 1 | 3 | 1 | 5 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx) | TypeScript JSX | 13 | 10 | 7 | 30 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx) | TypeScript JSX | -62 | 0 | 1 | -61 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx) | TypeScript JSX | 16 | 6 | 7 | 29 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx) | TypeScript JSX | 128 | 39 | 42 | 209 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx) | TypeScript JSX | 141 | 39 | 38 | 218 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx) | TypeScript JSX | 130 | 60 | 60 | 250 |
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details
|
||||
@@ -0,0 +1,25 @@
|
||||
# Diff Summary
|
||||
|
||||
Date : 2025-03-18 12:39:04
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 11 files, 732 codes, 239 comments, 231 blanks, all 1202 lines
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md)
|
||||
|
||||
## Languages
|
||||
| language | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| TypeScript JSX | 11 | 732 | 239 | 231 | 1,202 |
|
||||
|
||||
## Directories
|
||||
| path | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| . | 11 | 732 | 239 | 231 | 1,202 |
|
||||
| components | 7 | 317 | 95 | 84 | 496 |
|
||||
| components (Files) | 4 | 365 | 82 | 75 | 522 |
|
||||
| components/cells | 3 | -48 | 13 | 9 | -26 |
|
||||
| hooks | 4 | 415 | 144 | 147 | 706 |
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md)
|
||||
@@ -0,0 +1,39 @@
|
||||
Date : 2025-03-18 12:39:04
|
||||
Directory : /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
Total : 11 files, 732 codes, 239 comments, 231 blanks, all 1202 lines
|
||||
|
||||
Languages
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| language | files | code | comment | blank | total |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| TypeScript JSX | 11 | 732 | 239 | 231 | 1,202 |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Directories
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| path | files | code | comment | blank | total |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| . | 11 | 732 | 239 | 231 | 1,202 |
|
||||
| components | 7 | 317 | 95 | 84 | 496 |
|
||||
| components (Files) | 4 | 365 | 82 | 75 | 522 |
|
||||
| components/cells | 3 | -48 | 13 | 9 | -26 |
|
||||
| hooks | 4 | 415 | 144 | 147 | 706 |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Files
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| filename | language | code | comment | blank | total |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx | TypeScript JSX | 113 | 17 | 10 | 140 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx | TypeScript JSX | 3 | 7 | 10 | 20 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx | TypeScript JSX | 239 | 56 | 52 | 347 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx | TypeScript JSX | 10 | 2 | 3 | 15 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx | TypeScript JSX | 1 | 3 | 1 | 5 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx | TypeScript JSX | 13 | 10 | 7 | 30 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx | TypeScript JSX | -62 | 0 | 1 | -61 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx | TypeScript JSX | 16 | 6 | 7 | 29 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx | TypeScript JSX | 128 | 39 | 42 | 209 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx | TypeScript JSX | 141 | 39 | 38 | 218 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx | TypeScript JSX | 130 | 60 | 60 | 250 |
|
||||
| Total | | 732 | 239 | 231 | 1,202 |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,31 @@
|
||||
# Summary
|
||||
|
||||
Date : 2025-03-18 12:39:04
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 27 files, 6925 codes, 1247 comments, 1248 blanks, all 9420 lines
|
||||
|
||||
Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
|
||||
## Languages
|
||||
| language | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| TypeScript JSX | 19 | 6,549 | 1,134 | 1,165 | 8,848 |
|
||||
| TypeScript | 6 | 309 | 106 | 55 | 470 |
|
||||
| Markdown | 1 | 39 | 0 | 19 | 58 |
|
||||
| JavaScript | 1 | 28 | 7 | 9 | 44 |
|
||||
|
||||
## Directories
|
||||
| path | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| . | 27 | 6,925 | 1,247 | 1,248 | 9,420 |
|
||||
| . (Files) | 3 | 63 | 6 | 22 | 91 |
|
||||
| components | 12 | 3,674 | 498 | 494 | 4,666 |
|
||||
| components (Files) | 7 | 2,489 | 327 | 327 | 3,143 |
|
||||
| components/cells | 5 | 1,185 | 171 | 167 | 1,523 |
|
||||
| hooks | 6 | 2,855 | 630 | 669 | 4,154 |
|
||||
| types | 1 | 16 | 4 | 4 | 24 |
|
||||
| utils | 5 | 317 | 109 | 59 | 485 |
|
||||
|
||||
Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
@@ -0,0 +1,61 @@
|
||||
Date : 2025-03-18 12:39:04
|
||||
Directory : /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
Total : 27 files, 6925 codes, 1247 comments, 1248 blanks, all 9420 lines
|
||||
|
||||
Languages
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| language | files | code | comment | blank | total |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| TypeScript JSX | 19 | 6,549 | 1,134 | 1,165 | 8,848 |
|
||||
| TypeScript | 6 | 309 | 106 | 55 | 470 |
|
||||
| Markdown | 1 | 39 | 0 | 19 | 58 |
|
||||
| JavaScript | 1 | 28 | 7 | 9 | 44 |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Directories
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| path | files | code | comment | blank | total |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| . | 27 | 6,925 | 1,247 | 1,248 | 9,420 |
|
||||
| . (Files) | 3 | 63 | 6 | 22 | 91 |
|
||||
| components | 12 | 3,674 | 498 | 494 | 4,666 |
|
||||
| components (Files) | 7 | 2,489 | 327 | 327 | 3,143 |
|
||||
| components/cells | 5 | 1,185 | 171 | 167 | 1,523 |
|
||||
| hooks | 6 | 2,855 | 630 | 669 | 4,154 |
|
||||
| types | 1 | 16 | 4 | 4 | 24 |
|
||||
| utils | 5 | 317 | 109 | 59 | 485 |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Files
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| filename | language | code | comment | blank | total |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md | Markdown | 39 | 0 | 19 | 58 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx | TypeScript JSX | 230 | 10 | 8 | 248 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx | TypeScript JSX | 18 | 0 | 3 | 21 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx | TypeScript JSX | 273 | 19 | 37 | 329 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx | TypeScript JSX | 113 | 17 | 10 | 140 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx | TypeScript JSX | 377 | 49 | 54 | 480 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx | TypeScript JSX | 969 | 182 | 158 | 1,309 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx | TypeScript JSX | 509 | 50 | 57 | 616 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx | TypeScript JSX | 112 | 12 | 21 | 145 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx | TypeScript JSX | 233 | 34 | 33 | 300 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx | TypeScript JSX | 420 | 66 | 59 | 545 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx | TypeScript JSX | 193 | 23 | 22 | 238 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx | TypeScript JSX | 227 | 36 | 32 | 295 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx | TypeScript JSX | 500 | 75 | 89 | 664 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx | TypeScript JSX | 264 | 75 | 81 | 420 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx | TypeScript JSX | 204 | 26 | 33 | 263 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx | TypeScript JSX | 337 | 88 | 92 | 517 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx | TypeScript JSX | 360 | 78 | 85 | 523 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx | TypeScript JSX | 1,190 | 288 | 289 | 1,767 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx | TypeScript JSX | 20 | 6 | 2 | 28 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts | TypeScript | 4 | 0 | 1 | 5 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts | TypeScript | 16 | 4 | 4 | 24 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts | TypeScript | 124 | 4 | 14 | 142 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts | TypeScript | 21 | 15 | 5 | 41 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts | TypeScript | 43 | 24 | 7 | 74 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js | JavaScript | 28 | 7 | 9 | 44 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts | TypeScript | 101 | 59 | 24 | 184 |
|
||||
| Total | | 6,925 | 1,247 | 1,248 | 9,420 |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
@@ -0,0 +1,42 @@
|
||||
# Details
|
||||
|
||||
Date : 2025-03-18 13:49:23
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 27 files, 6961 codes, 1254 comments, 1252 blanks, all 9467 lines
|
||||
|
||||
[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
|
||||
## Files
|
||||
| filename | language | code | comment | blank | total |
|
||||
| :--- | :--- | ---: | ---: | ---: | ---: |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md) | Markdown | 39 | 0 | 19 | 58 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx) | TypeScript JSX | 230 | 10 | 8 | 248 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx) | TypeScript JSX | 18 | 0 | 3 | 21 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx) | TypeScript JSX | 273 | 19 | 37 | 329 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx) | TypeScript JSX | 113 | 17 | 10 | 140 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx) | TypeScript JSX | 395 | 51 | 55 | 501 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx) | TypeScript JSX | 969 | 182 | 158 | 1,309 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx) | TypeScript JSX | 527 | 55 | 60 | 642 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx) | TypeScript JSX | 112 | 12 | 21 | 145 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx) | TypeScript JSX | 233 | 34 | 33 | 300 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx) | TypeScript JSX | 420 | 66 | 59 | 545 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx) | TypeScript JSX | 193 | 23 | 22 | 238 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx) | TypeScript JSX | 227 | 36 | 32 | 295 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx) | TypeScript JSX | 500 | 75 | 89 | 664 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx) | TypeScript JSX | 264 | 75 | 81 | 420 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx) | TypeScript JSX | 204 | 26 | 33 | 263 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx) | TypeScript JSX | 337 | 88 | 92 | 517 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx) | TypeScript JSX | 360 | 78 | 85 | 523 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx) | TypeScript JSX | 1,190 | 288 | 289 | 1,767 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx) | TypeScript JSX | 20 | 6 | 2 | 28 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts) | TypeScript | 4 | 0 | 1 | 5 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts) | TypeScript | 16 | 4 | 4 | 24 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts) | TypeScript | 124 | 4 | 14 | 142 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts) | TypeScript | 21 | 15 | 5 | 41 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts) | TypeScript | 43 | 24 | 7 | 74 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js) | JavaScript | 28 | 7 | 9 | 44 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts) | TypeScript | 101 | 59 | 24 | 184 |
|
||||
|
||||
[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
@@ -0,0 +1,17 @@
|
||||
# Diff Details
|
||||
|
||||
Date : 2025-03-18 13:49:23
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 2 files, 36 codes, 7 comments, 4 blanks, all 47 lines
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details
|
||||
|
||||
## Files
|
||||
| filename | language | code | comment | blank | total |
|
||||
| :--- | :--- | ---: | ---: | ---: | ---: |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx) | TypeScript JSX | 18 | 2 | 1 | 21 |
|
||||
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx) | TypeScript JSX | 18 | 5 | 3 | 26 |
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details
|
||||
@@ -0,0 +1,22 @@
|
||||
# Diff Summary
|
||||
|
||||
Date : 2025-03-18 13:49:23
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 2 files, 36 codes, 7 comments, 4 blanks, all 47 lines
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md)
|
||||
|
||||
## Languages
|
||||
| language | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| TypeScript JSX | 2 | 36 | 7 | 4 | 47 |
|
||||
|
||||
## Directories
|
||||
| path | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| . | 2 | 36 | 7 | 4 | 47 |
|
||||
| components | 2 | 36 | 7 | 4 | 47 |
|
||||
|
||||
[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md)
|
||||
@@ -0,0 +1,27 @@
|
||||
Date : 2025-03-18 13:49:23
|
||||
Directory : /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
Total : 2 files, 36 codes, 7 comments, 4 blanks, all 47 lines
|
||||
|
||||
Languages
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| language | files | code | comment | blank | total |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| TypeScript JSX | 2 | 36 | 7 | 4 | 47 |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Directories
|
||||
+---------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| path | files | code | comment | blank | total |
|
||||
+---------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| . | 2 | 36 | 7 | 4 | 47 |
|
||||
| components | 2 | 36 | 7 | 4 | 47 |
|
||||
+---------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Files
|
||||
+---------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| filename | language | code | comment | blank | total |
|
||||
+---------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx | TypeScript JSX | 18 | 2 | 1 | 21 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx | TypeScript JSX | 18 | 5 | 3 | 26 |
|
||||
| Total | | 36 | 7 | 4 | 47 |
|
||||
+---------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,31 @@
|
||||
# Summary
|
||||
|
||||
Date : 2025-03-18 13:49:23
|
||||
|
||||
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
|
||||
Total : 27 files, 6961 codes, 1254 comments, 1252 blanks, all 9467 lines
|
||||
|
||||
Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
|
||||
## Languages
|
||||
| language | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| TypeScript JSX | 19 | 6,585 | 1,141 | 1,169 | 8,895 |
|
||||
| TypeScript | 6 | 309 | 106 | 55 | 470 |
|
||||
| Markdown | 1 | 39 | 0 | 19 | 58 |
|
||||
| JavaScript | 1 | 28 | 7 | 9 | 44 |
|
||||
|
||||
## Directories
|
||||
| path | files | code | comment | blank | total |
|
||||
| :--- | ---: | ---: | ---: | ---: | ---: |
|
||||
| . | 27 | 6,961 | 1,254 | 1,252 | 9,467 |
|
||||
| . (Files) | 3 | 63 | 6 | 22 | 91 |
|
||||
| components | 12 | 3,710 | 505 | 498 | 4,713 |
|
||||
| components (Files) | 7 | 2,525 | 334 | 331 | 3,190 |
|
||||
| components/cells | 5 | 1,185 | 171 | 167 | 1,523 |
|
||||
| hooks | 6 | 2,855 | 630 | 669 | 4,154 |
|
||||
| types | 1 | 16 | 4 | 4 | 24 |
|
||||
| utils | 5 | 317 | 109 | 59 | 485 |
|
||||
|
||||
Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
|
||||
@@ -0,0 +1,61 @@
|
||||
Date : 2025-03-18 13:49:23
|
||||
Directory : /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
|
||||
Total : 27 files, 6961 codes, 1254 comments, 1252 blanks, all 9467 lines
|
||||
|
||||
Languages
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| language | files | code | comment | blank | total |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
| TypeScript JSX | 19 | 6,585 | 1,141 | 1,169 | 8,895 |
|
||||
| TypeScript | 6 | 309 | 106 | 55 | 470 |
|
||||
| Markdown | 1 | 39 | 0 | 19 | 58 |
|
||||
| JavaScript | 1 | 28 | 7 | 9 | 44 |
|
||||
+----------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Directories
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| path | files | code | comment | blank | total |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
| . | 27 | 6,961 | 1,254 | 1,252 | 9,467 |
|
||||
| . (Files) | 3 | 63 | 6 | 22 | 91 |
|
||||
| components | 12 | 3,710 | 505 | 498 | 4,713 |
|
||||
| components (Files) | 7 | 2,525 | 334 | 331 | 3,190 |
|
||||
| components/cells | 5 | 1,185 | 171 | 167 | 1,523 |
|
||||
| hooks | 6 | 2,855 | 630 | 669 | 4,154 |
|
||||
| types | 1 | 16 | 4 | 4 | 24 |
|
||||
| utils | 5 | 317 | 109 | 59 | 485 |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
|
||||
|
||||
Files
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| filename | language | code | comment | blank | total |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md | Markdown | 39 | 0 | 19 | 58 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx | TypeScript JSX | 230 | 10 | 8 | 248 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx | TypeScript JSX | 18 | 0 | 3 | 21 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx | TypeScript JSX | 273 | 19 | 37 | 329 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx | TypeScript JSX | 113 | 17 | 10 | 140 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx | TypeScript JSX | 395 | 51 | 55 | 501 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx | TypeScript JSX | 969 | 182 | 158 | 1,309 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx | TypeScript JSX | 527 | 55 | 60 | 642 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx | TypeScript JSX | 112 | 12 | 21 | 145 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx | TypeScript JSX | 233 | 34 | 33 | 300 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx | TypeScript JSX | 420 | 66 | 59 | 545 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx | TypeScript JSX | 193 | 23 | 22 | 238 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx | TypeScript JSX | 227 | 36 | 32 | 295 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx | TypeScript JSX | 500 | 75 | 89 | 664 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx | TypeScript JSX | 264 | 75 | 81 | 420 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx | TypeScript JSX | 204 | 26 | 33 | 263 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx | TypeScript JSX | 337 | 88 | 92 | 517 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx | TypeScript JSX | 360 | 78 | 85 | 523 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx | TypeScript JSX | 1,190 | 288 | 289 | 1,767 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx | TypeScript JSX | 20 | 6 | 2 | 28 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts | TypeScript | 4 | 0 | 1 | 5 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts | TypeScript | 16 | 4 | 4 | 24 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts | TypeScript | 124 | 4 | 14 | 142 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts | TypeScript | 21 | 15 | 5 | 41 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts | TypeScript | 43 | 24 | 7 | 74 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js | JavaScript | 28 | 7 | 9 | 44 |
|
||||
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts | TypeScript | 101 | 59 | 24 | 184 |
|
||||
| Total | | 6,961 | 1,254 | 1,252 | 9,467 |
|
||||
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
|
||||
@@ -0,0 +1,172 @@
|
||||
# CLAUDE.md
|
||||
|
||||
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
|
||||
|
||||
## Project Overview
|
||||
|
||||
This is a full-stack inventory management system with a React + TypeScript frontend and Node.js/Express backend using PostgreSQL. The system includes product management, analytics, forecasting, purchase orders, and a comprehensive dashboard for business metrics.
|
||||
|
||||
**Monorepo Structure:**
|
||||
- `inventory/` - Vite-based React frontend with TypeScript
|
||||
- `inventory-server/` - Express backend API server
|
||||
- Root `package.json` contains shared dependencies
|
||||
|
||||
## Development Commands
|
||||
|
||||
### Frontend (inventory/)
|
||||
```bash
|
||||
cd inventory
|
||||
npm run dev # Start dev server on port 5175
|
||||
npm run build # Build for production (outputs to build/ then copies to ../inventory-server/frontend/build)
|
||||
npm run lint # Run ESLint
|
||||
npm run preview # Preview production build
|
||||
```
|
||||
|
||||
### Backend (inventory-server/)
|
||||
```bash
|
||||
cd inventory-server
|
||||
npm run dev # Start with nodemon (auto-reload)
|
||||
npm start # Start server (production)
|
||||
npm run prod # Start with PM2 for production
|
||||
npm run prod:stop # Stop PM2 instance
|
||||
npm run prod:restart # Restart PM2 instance
|
||||
npm run prod:logs # View PM2 logs
|
||||
npm run setup # Create required directories (logs, uploads)
|
||||
```
|
||||
|
||||
## Architecture
|
||||
|
||||
### Frontend Architecture
|
||||
|
||||
**Router Structure:** React Router with lazy loading for code splitting:
|
||||
- Main chunks: Core inventory, Dashboard, Product Import, Chat Archive
|
||||
- Authentication flow uses `RequireAuth` and `Protected` components with permission-based access
|
||||
- All routes except `/login` and `/small` require authentication
|
||||
|
||||
**Key Directories:**
|
||||
- `src/pages/` - Top-level page components (Overview, Products, Analytics, Dashboard, etc.)
|
||||
- `src/components/` - Organized by feature (dashboard/, products/, analytics/, etc.)
|
||||
- `src/components/ui/` - shadcn/ui components
|
||||
- `src/types/` - TypeScript type definitions
|
||||
- `src/contexts/` - React contexts (AuthContext, DashboardScrollContext)
|
||||
- `src/hooks/` - Custom React hooks (use-toast, useDebounce, use-mobile)
|
||||
- `src/utils/` - Utility functions (emojiUtils, productUtils, naturalLanguagePeriod)
|
||||
- `src/services/` - API service layer
|
||||
- `src/config/` - Configuration files
|
||||
|
||||
**State Management:**
|
||||
- React Context for auth and global state
|
||||
- @tanstack/react-query for server state management
|
||||
- zustand for client state management
|
||||
- Local storage for auth tokens, session storage for login state
|
||||
|
||||
**Key Dependencies:**
|
||||
- UI: Radix UI primitives, shadcn/ui, Tailwind CSS, Framer Motion
|
||||
- Data: @tanstack/react-table, react-data-grid, @tanstack/react-virtual
|
||||
- Forms: react-hook-form, zod
|
||||
- Charts: recharts, chart.js, react-chartjs-2
|
||||
- File handling: xlsx for Excel export, react-dropzone for uploads
|
||||
- Other: axios for HTTP, date-fns/luxon for dates
|
||||
|
||||
**Path Alias:** `@/` maps to `./src/`
|
||||
|
||||
### Backend Architecture
|
||||
|
||||
**Entry Point:** `inventory-server/src/server.js`
|
||||
|
||||
**Key Directories:**
|
||||
- `src/routes/` - Express route handlers (products, dashboard, analytics, import, etc.)
|
||||
- `src/middleware/` - Express middleware (CORS, auth, etc.)
|
||||
- `src/utils/` - Utility functions (database connection, API helpers)
|
||||
- `src/types/` - Type definitions (e.g., status-codes)
|
||||
|
||||
**Database:**
|
||||
- PostgreSQL with connection pooling (pg library)
|
||||
- Pool initialized in `utils/db.js` via `initPool()`
|
||||
- Pool attached to `app.locals.pool` for route access
|
||||
- Environment variables loaded from `/var/www/inventory/.env` (production path)
|
||||
|
||||
**API Routes:** All prefixed with `/api/`
|
||||
- `/api/products` - Product CRUD operations
|
||||
- `/api/dashboard` - Dashboard metrics and data
|
||||
- `/api/analytics` - Analytics and reporting
|
||||
- `/api/orders` - Order management
|
||||
- `/api/purchase-orders` - Purchase order management
|
||||
- `/api/csv` - CSV import/export (data management)
|
||||
- `/api/import` - Product import workflows
|
||||
- `/api/config` - Configuration management
|
||||
- `/api/metrics` - System metrics
|
||||
- `/api/ai-validation` - AI-powered validation
|
||||
- `/api/ai-prompts` - AI prompt management
|
||||
- `/api/templates` - Template management
|
||||
- `/api/reusable-images` - Image management
|
||||
- `/api/categoriesAggregate`, `/api/vendorsAggregate`, `/api/brandsAggregate` - Aggregate data endpoints
|
||||
|
||||
**Authentication:**
|
||||
- External auth service at `/auth-inv` endpoint
|
||||
- Token-based authentication (Bearer tokens)
|
||||
- Frontend stores tokens in localStorage
|
||||
- Protected routes verify tokens via auth service `/me` endpoint
|
||||
|
||||
**File Uploads:**
|
||||
- Multer middleware for file handling
|
||||
- Uploads directory: `inventory-server/uploads/`
|
||||
|
||||
### Development Proxy Setup
|
||||
|
||||
The Vite dev server (port 5175) proxies API requests to `https://inventory.kent.pw`:
|
||||
- `/api/*` → production API
|
||||
- `/auth-inv/*` → authentication service
|
||||
- `/chat-api/*` → chat service
|
||||
- `/uploads/*` → uploaded files
|
||||
- Various third-party services (Aircall, Klaviyo, Meta, Gorgias, Typeform, ACOT, Clarity)
|
||||
|
||||
### Build Process
|
||||
|
||||
When building the frontend:
|
||||
1. TypeScript compilation (`tsc -b`)
|
||||
2. Vite build (outputs to `inventory/build/`)
|
||||
3. Custom Vite plugin copies build to `inventory-server/frontend/build/`
|
||||
4. Manual chunks for vendor splitting (react-vendor, ui-vendor, query-vendor)
|
||||
|
||||
## Testing
|
||||
|
||||
Run tests for individual components or features:
|
||||
```bash
|
||||
# No test suite currently configured
|
||||
# Tests would typically use Jest or Vitest with React Testing Library
|
||||
```
|
||||
|
||||
## Common Development Workflows
|
||||
|
||||
### Adding a New Page
|
||||
1. Create page component in `inventory/src/pages/YourPage.tsx`
|
||||
2. Add lazy import in `inventory/src/App.tsx`
|
||||
3. Add route with `<Protected>` wrapper and permission check
|
||||
4. Add corresponding backend route in `inventory-server/src/routes/`
|
||||
5. Update permission system if needed
|
||||
|
||||
### Adding a New API Endpoint
|
||||
1. Create or update route file in `inventory-server/src/routes/`
|
||||
2. Use `executeQuery()` helper for database queries
|
||||
3. Register router in `inventory-server/src/server.js`
|
||||
4. Frontend can access at `/api/{route-name}`
|
||||
|
||||
### Working with Database
|
||||
- Use parameterized queries: `executeQuery(sql, [param1, param2])`
|
||||
- Pool is accessed via `db.getPool()` or `app.locals.pool`
|
||||
- Connection helper: `db.getConnection()` returns a client for transactions
|
||||
|
||||
### Permissions System
|
||||
- User permissions stored in `user.permissions` array (permission codes)
|
||||
- Check permissions in `<Protected page="permission_code">` component
|
||||
- Admin users (`is_admin: true`) have access to all pages
|
||||
|
||||
## Important Notes
|
||||
|
||||
- Environment variables must be configured in `/var/www/inventory/.env` for production
|
||||
- The frontend expects the backend at `/api` (proxied in dev, served together in production)
|
||||
- PM2 is used for production process management
|
||||
- Database uses PostgreSQL with SSL support (configurable via `DB_SSL` env var)
|
||||
- File uploads stored in `inventory-server/uploads/` directory
|
||||
- Build artifacts in `inventory/build/` are copied to `inventory-server/frontend/build/`
|
||||
+28
@@ -50,6 +50,11 @@ dashboard-server/meta-server/._package-lock.json
|
||||
dashboard-server/meta-server/._services
|
||||
*.tsbuildinfo
|
||||
|
||||
uploads/*
|
||||
uploads/**/*
|
||||
**/uploads/*
|
||||
**/uploads/**/*
|
||||
|
||||
# CSV data files
|
||||
*.csv
|
||||
csv/*
|
||||
@@ -59,3 +64,26 @@ csv/**/*
|
||||
!csv/.gitkeep
|
||||
inventory/tsconfig.tsbuildinfo
|
||||
inventory-server/scripts/.fuse_hidden00000fa20000000a
|
||||
|
||||
.VSCodeCounter/
|
||||
.VSCodeCounter/*
|
||||
.VSCodeCounter/**/*
|
||||
|
||||
*/chat/db-convert/db/*
|
||||
*/chat/db-convert/mongo_converter_env/*
|
||||
|
||||
# Ignore compiled Vite config to avoid duplication
|
||||
vite.config.js
|
||||
inventory-server/inventory_backup.sql
|
||||
chat-files.tar.gz
|
||||
chat-migration*/
|
||||
**/chat-migration*/
|
||||
chat-migration*/**
|
||||
**/chat-migration*/**
|
||||
|
||||
venv/
|
||||
venv/**
|
||||
**/venv/*
|
||||
**/venv/**
|
||||
|
||||
inventory-server/data/taxonomy-embeddings.json
|
||||
@@ -0,0 +1,4 @@
|
||||
* Avoid using glob tool for search as it may not work properly on this codebase. Search using bash instead.
|
||||
* If you use the task tool to have an agent investigate something, make sure to let it know to avoid using glob
|
||||
* Prefer solving tasks in a single session. Only spawn subagents for genuinely independent workstreams.
|
||||
* The postgres/query tool is not working and not connected to the current version of the database. If you need to query the database for any reason you can use "ssh netcup" and use psql on the server with inventory_readonly 6D3GUkxuFgi2UghwgnUd
|
||||
@@ -0,0 +1,899 @@
|
||||
# Server Consolidation & Security Hardening Plan
|
||||
|
||||
Audit-driven plan to (a) reduce 12 PM2 processes to 3 application servers + 1 auth server, (b) put every API endpoint behind real authentication, and (c) standardize on ESM across all Node services. Approach is "do it properly the first time" — no half-finished pieces, no deferred cleanup.
|
||||
|
||||
---
|
||||
|
||||
## Status (2026-05-24)
|
||||
|
||||
| Phase | Status | Notes |
|
||||
|---|---|---|
|
||||
| 1 — Decommission dead services | **Complete** | aircall/gorgias/clarity/legacy-auth-server deleted from repo + PM2 + Caddyfile + ecosystem.cjs |
|
||||
| 2 — Build shared `lib/` | **Complete** | Lives at `inventory-server/shared/` (see Deviations). `/verify` endpoint live on auth-server |
|
||||
| 3 — Convert auth-server + inventory-server to ESM | **Complete** | All 58 server-side files ESM; both services live under the ESM build for >24h. See Deviations #10–13 |
|
||||
| 4 — Build `dashboard-server` (the merge) | **Complete (live) — 2026-05-24** | Merged service running on :3015 under PM2; Caddy routes for klaviyo/meta/dashboard-analytics/typeform all reverse-proxy to it. Old per-vendor directories (`klaviyo-server`, `meta-server`, `google-server`, `typeform-server`) and their PM2 entries deleted post-cutover — ~1.27 GB reclaimed (largely duplicated `node_modules`). Phase 6.2 gates wired (meta_write, klaviyo_admin). See Deviations #16–19 |
|
||||
| 5 — Convert `acot-server` to ESM | Not started | |
|
||||
| 6 — Auth hardening | **Complete** | All in-process items live: rate-limit, JWT precondition, CORS lockdown, request-log, upload allowlist, `requirePermission` on sensitive routes, permissions seed migration. `authenticate()` is live on `/api/*`. 6.11 (audit logging) deferred — see Out of scope |
|
||||
| **F1 — Frontend fetch wrapper** | **Complete (live) — 2026-05-23** | Wrappers at `inventory/src/utils/api.ts` (`apiFetch`) and `inventory/src/utils/apiClient.ts` (axios instance). 170 `fetch()` sites across 76 files migrated to `apiFetch`; 32 `axios.*` sites across 11 files migrated to `apiClient`. AuthContext `/login`+`/me`, App.tsx `/me`, and `services/apiv2.ts` (external PHP backend) intentionally left as raw `fetch`. Shipped alongside the Phase 3+6 pm2 reload |
|
||||
| 7 — Caddyfile final form | **Complete — applied 2026-05-24** | Final Caddyfile live at `/etc/caddy/Caddyfile` (forward_auth gate + per-vendor reverse_proxy to :3015). The `inventory-server/deploy/` staging folder was removed after apply — recreate from this doc if future changes are needed. Backup convention: `/etc/caddy/Caddyfile.bak.YYYY-MM-DD` |
|
||||
| 8 — ecosystem.config.cjs final form | **Complete — applied 2026-05-24** | Live PM2 list matches the spec below (5 apps + acot-phone-server + lt-wordlist-api = 7 processes). Includes Phase 6.4 JWT_SECRET shadow-override fix and 6.10 lt-wordlist token move. `inventory-server/deploy/` removed post-apply |
|
||||
|
||||
**Live PM2 process count: 7** (5 application apps — auth-server, inventory-server, chat-server, dashboard-server, acot-server — plus acot-phone-server + lt-wordlist-api). Down from 13 pre-refactor.
|
||||
|
||||
**All apply steps complete (2026-05-24).** The original sequencing (npm install → F1 ship → pm2 reload → env consolidation → vendor PM2 delete → ecosystem apply → Caddyfile apply) was executed in order. Remaining work is Phase 5 (acot-server ESM conversion) only.
|
||||
|
||||
---
|
||||
|
||||
## Goals
|
||||
|
||||
- Every public-facing endpoint requires a valid auth token (Caddy gate + per-server middleware + per-route permission checks for sensitive operations).
|
||||
- Reduce service count from 12 PM2 processes to 4: `inventory-server`, `acot-server`, `dashboard-server`, `auth-server`.
|
||||
- Standardize on ESM (`"type": "module"`) across all Node services.
|
||||
- Decommission `aircall-server`, `gorgias-server`, `clarity-server`, and the legacy `auth-server` (port 3003).
|
||||
- Eliminate dependency duplication: one Redis client, one Postgres pool helper, one logger, one auth middleware — shared across services.
|
||||
|
||||
## Non-goals
|
||||
|
||||
- Rewriting business logic. Route handlers move as-is unless they break under ESM or shared middleware.
|
||||
- Switching auth providers (we keep JWT + bcrypt + Postgres).
|
||||
- Replacing PM2 or Caddy.
|
||||
- Migrating Klaviyo/Meta/Google/Typeform's external API contracts.
|
||||
|
||||
---
|
||||
|
||||
## Target architecture
|
||||
|
||||
```
|
||||
┌──────────────────────────┐
|
||||
│ tools.acherryontop.com │
|
||||
│ (Caddy) │
|
||||
│ forward_auth gate ─────┼──► auth-server:3011
|
||||
└────────────┬─────────────┘ /verify endpoint
|
||||
│
|
||||
┌────────────────────────────────┼────────────────────────────────┐
|
||||
▼ ▼ ▼
|
||||
┌─────────────────────┐ ┌──────────────────────┐ ┌─────────────────────┐
|
||||
│ inventory-server │ │ dashboard-server │ │ acot-server │
|
||||
│ :3010 (ESM) │ │ :3015 (ESM) │ │ :3012 (ESM) │
|
||||
│ │ │ │ │ │
|
||||
│ /api/products │ │ /api/klaviyo/* │ │ /api/acot/* │
|
||||
│ /api/orders │ │ /api/meta/* │ │ (MySQL via SSH) │
|
||||
│ /api/analytics │ │ /api/google-*/* │ │ │
|
||||
│ /api/dashboard │ │ /api/typeform/* │ │ │
|
||||
│ ... (~25 routers) │ │ │ │ │
|
||||
└─────────────────────┘ └──────────────────────┘ └─────────────────────┘
|
||||
│ │ │
|
||||
├── Postgres (inventory_db) ├── Postgres (klaviyo) └── MySQL (workpi, via ssh2 tunnel)
|
||||
├── shared lib/ ◄────────────────┤
|
||||
│ - auth middleware ├── Redis (shared client)
|
||||
│ - permission helper └── shared lib/ ◄─────────────────┐
|
||||
│ - logger │
|
||||
│ - pg pool factory │
|
||||
│ - error formatter │
|
||||
└─────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
┌──────────────────┴───┐
|
||||
│ auth-server │
|
||||
│ :3011 (ESM) │
|
||||
│ /login, /me, │
|
||||
│ /verify, user mgmt │
|
||||
└──────────────────────┘
|
||||
```
|
||||
|
||||
PM2 process count: **12 → 4** (plus `acot-phone-server` and `lt-wordlist-api`, which stay as-is — out of scope).
|
||||
|
||||
---
|
||||
|
||||
## Phase 1 — Decommission dead/leaving services
|
||||
|
||||
Status: **Complete (2026-05-23)**. All four services removed from repo, PM2, Caddyfile, and ecosystem.config.cjs. Frontend widgets (`AircallDashboard.jsx`, `GorgiasOverview.jsx`) and their dashboard.ts/Navigation.jsx/vite.config.ts wiring also removed. Verification: smoke-tested `https://tools.acherryontop.com/api/{aircall,gorgias,clarity}/*` → 404. Backups left at `/home/matt/{ecosystem.config.cjs,Caddyfile}.bak.2026-05-23`.
|
||||
|
||||
### To remove
|
||||
|
||||
| Service | Reason | Steps |
|
||||
|---|---|---|
|
||||
| `aircall-server` (3002) | Migrating off Aircall | `pm2 delete aircall-server`; remove from `ecosystem.config.cjs`; remove `/api/aircall/*` from Caddyfile; drop `inventory/dashboard/aircall-server/` directory; remove MongoDB connection from any frontend code; cancel Mongo if it was only feeding Aircall |
|
||||
| `gorgias-server` (3006) | Migrating off Gorgias | same pattern; check frontend for `/api/gorgias/*` callers and delete the dashboards/widgets that use them |
|
||||
| `clarity-server` (3009) | Already dead (no `.js` files, not in ecosystem) | remove `/api/clarity/*` from Caddyfile; delete `inventory/dashboard/clarity-server/` directory |
|
||||
| `auth-server` (3003, legacy) | Replaced by `new-auth-server` on 3011 | grep entire codebase for `dashboard-auth` and `localhost:3003`; redirect or remove callers; `pm2 delete auth-server`; remove from ecosystem; remove `/dashboard-auth/*` from Caddyfile; delete `inventory/dashboard/auth-server/` directory |
|
||||
|
||||
### Verification before deletion
|
||||
|
||||
```bash
|
||||
# from inventory/ root — find any references before removing
|
||||
grep -rn "aircall\|/api/aircall" inventory/src/ inventory-server/src/
|
||||
grep -rn "gorgias\|/api/gorgias" inventory/src/ inventory-server/src/
|
||||
grep -rn "/dashboard-auth\|localhost:3003" inventory/src/ inventory-server/src/
|
||||
grep -rn "/api/clarity" inventory/src/ inventory-server/src/
|
||||
```
|
||||
|
||||
Any remaining callers must be deleted or repointed before the server is removed. Do **not** leave a 502 response in production.
|
||||
|
||||
### Database/secret cleanup
|
||||
|
||||
- Drop the MongoDB instance feeding Aircall (after confirming no other consumers).
|
||||
- Rotate any Gorgias/Aircall API keys still in `.env` files (defense in depth — they'll be useless soon anyway, but commit hygiene matters).
|
||||
- Remove `MONGODB_URI`, `AIRCALL_*`, `GORGIAS_*` from any `.env` files.
|
||||
|
||||
---
|
||||
|
||||
## Phase 2 — Build the shared `lib/`
|
||||
|
||||
Status: **Complete (2026-05-23)**. All 11 modules written under `inventory-server/shared/` (NOT repo root — see Deviations). `/verify` endpoint added to auth-server in CJS form (will move to shared/auth/verify.js usage during Phase 3 ESM conversion). Smoke-tested with no-token / bad-token / expired-token / valid-token cases. No service consumes shared/ yet; that happens in Phases 3–5.
|
||||
|
||||
### Location
|
||||
|
||||
A single shared directory at the repo root: `shared/` (sibling of `inventory/` and `acot-phone/`). Each service imports from it via a relative path. We do **not** introduce npm workspaces yet — relative imports are fine for three consumers and avoid the npm-link / hoisting headaches.
|
||||
|
||||
### Modules to create
|
||||
|
||||
```
|
||||
shared/
|
||||
├── package.json # "type": "module"
|
||||
├── auth/
|
||||
│ ├── middleware.js # authenticate(), requirePermission(), requireAdmin()
|
||||
│ └── verify.js # verifyToken() — pure function, no Express dependency
|
||||
├── db/
|
||||
│ ├── pg.js # createPool(envPrefix) — returns configured Pool
|
||||
│ └── redis.js # createRedis() — single client, lazy-connect
|
||||
├── logging/
|
||||
│ ├── logger.js # pino-based, redacts Authorization/Cookie
|
||||
│ └── request-log.js # Express middleware, structured access log
|
||||
├── errors/
|
||||
│ └── handler.js # consistent error envelope, no leak in prod
|
||||
├── cors/
|
||||
│ └── policy.js # single allowed-origins list, exported as cors() options
|
||||
└── rate-limit/
|
||||
└── login.js # express-rate-limit config for /login
|
||||
```
|
||||
|
||||
### Auth middleware spec (`shared/auth/middleware.js`)
|
||||
|
||||
```js
|
||||
// Pseudocode — final implementation matches the existing pattern in
|
||||
// inventory/auth/routes.js authenticate() but factored out.
|
||||
|
||||
export function authenticate({ pool }) {
|
||||
return async (req, res, next) => {
|
||||
const header = req.headers.authorization;
|
||||
if (!header?.startsWith('Bearer ')) {
|
||||
return res.status(401).json({ error: 'Authentication required' });
|
||||
}
|
||||
try {
|
||||
const decoded = jwt.verify(header.slice(7), process.env.JWT_SECRET);
|
||||
// Short-circuit DB hit with an in-memory cache, 60s TTL keyed by token jti
|
||||
const user = await loadUserCached(pool, decoded.userId);
|
||||
if (!user.is_active) return res.status(403).json({ error: 'Account inactive' });
|
||||
req.user = user;
|
||||
next();
|
||||
} catch {
|
||||
res.status(401).json({ error: 'Invalid token' });
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function requirePermission(code) {
|
||||
return (req, res, next) => {
|
||||
if (req.user.is_admin) return next();
|
||||
if (req.user.permissions?.includes(code)) return next();
|
||||
res.status(403).json({ error: 'Insufficient permissions' });
|
||||
};
|
||||
}
|
||||
|
||||
export const requireAdmin = (req, res, next) =>
|
||||
req.user.is_admin ? next() : res.status(403).json({ error: 'Admin only' });
|
||||
```
|
||||
|
||||
### Why a 60s in-memory user cache
|
||||
|
||||
`forward_auth` in Caddy will call `auth-server` on every request. Each per-server `authenticate()` middleware also has a DB lookup to load permissions. Without caching, every API request becomes 1 SQL query for the user row + 1 for permissions. 60s TTL is short enough that deactivating a user takes effect within a minute, long enough that Klaviyo dashboards (which fire dozens of requests on load) don't hammer Postgres.
|
||||
|
||||
### Add to `auth-server`: a `/verify` endpoint
|
||||
|
||||
Caddy's `forward_auth` only needs "is this token valid? give me a user-id." Today's `/me` does that but with a full permissions join. Add a lightweight `/verify` that:
|
||||
|
||||
- Verifies JWT signature only (no DB hit).
|
||||
- Returns `200` with `X-User-Id` and `X-User-Is-Admin` response headers (which Caddy `copy_headers` will pass to upstream).
|
||||
- Returns `401` on bad token.
|
||||
|
||||
**Decision: each service re-verifies the JWT independently.** Caddy's `forward_auth` is a fast first-pass reject for obviously bad tokens, but the security boundary is the per-server `authenticate()` middleware. Cost is negligible (one HMAC-SHA256 per request); the upside is that a misconfigured Caddyfile can never let an unauthenticated request reach a backend. Upstream services do **not** trust any `X-User-*` headers from Caddy — they parse the `Authorization` header themselves.
|
||||
|
||||
---
|
||||
|
||||
## Phase 3 — Convert `auth-server` and `inventory-server` to ESM
|
||||
|
||||
Status: **Complete (live) — 2026-05-24.** Both servers + all sub-trees converted to ESM and running under PM2. 58 importable .js files. Two latent bugs surfaced and fixed during the conversion: `??`/`||` precedence in `shared/db/{pg,redis}.js`, and CJS named-import of `Pool` from `pg` in both auth files (now uses `import pg from 'pg'; const { Pool } = pg`).
|
||||
|
||||
Scripts under `inventory-server/scripts/` (one-shot maintenance / orchestrators) kept CommonJS via a sibling `scripts/package.json` declaring `"type": "commonjs"` — Node's package-type resolution walks up directory by directory, so this overrides the parent's `"type": "module"` without renaming any file or touching any `spawn()` callsite. Convert individual scripts to ESM if/when touched.
|
||||
|
||||
Went live 2026-05-24 after `npm install` on netcup (new deps: `pino`, `pino-http`, `ioredis`, `express-rate-limit`, `jsonwebtoken`) + `pm2 reload`. Phase F1 (frontend fetch wrapper) shipped in the same window so the SPA continued to send `Authorization: Bearer` on every request as `authenticate()` came online.
|
||||
|
||||
### Mechanical conversion
|
||||
|
||||
Per service:
|
||||
|
||||
1. Add `"type": "module"` to `package.json`.
|
||||
2. Convert `require()` → `import`. `module.exports` → `export` / `export default`.
|
||||
3. Fix `__dirname`/`__filename` (use `import.meta.url` + `fileURLToPath`).
|
||||
4. Convert any dynamic require (e.g., conditional plugin loading) to `await import()`.
|
||||
5. Update any sub-imports that don't include the file extension — ESM requires `./foo.js`, not `./foo`.
|
||||
6. Update `ecosystem.config.cjs` if any service entry depended on CJS semantics. The ecosystem file itself can stay `.cjs` — PM2 reads it as config, doesn't matter what the apps it spawns are.
|
||||
7. Update nodemon config / scripts.
|
||||
|
||||
### Risk areas in inventory-server
|
||||
|
||||
- `routes/ai.js` does a lazy init (`aiRouter.initInBackground()` called from `server.js`) — confirm the export shape still works as a default export of an Express router with a sidecar function. May need to split into `export default router; export function initInBackground() {}`.
|
||||
- Multer setup in `routes/import.js` — straightforward, no ESM-specific concerns.
|
||||
- SSE setup in `server.js` — moves over cleanly, no module-system entanglement.
|
||||
- The `child_process.spawn` calls for metrics calculation: ESM doesn't change `child_process` behavior, but if any spawned script uses `require()` of a sibling, that sibling must also be ESM (or stay CJS with a `.cjs` extension).
|
||||
|
||||
### Test strategy
|
||||
|
||||
- After conversion, `pm2 start ecosystem.config.cjs --only inventory-server` on the server, watch logs for require/import errors at startup.
|
||||
- Hit `/health`, then the most exercised endpoints (`/api/products`, `/api/dashboard/overview`, `/api/analytics/...`). If startup is clean and three smoke endpoints work, ESM conversion is done. Functional correctness is preserved because no logic changed.
|
||||
|
||||
### Auth-server
|
||||
|
||||
Already small (~200 LOC server.js + ~few hundred in routes.js + permissions.js). 1-day conversion. Add the new `/verify` endpoint as part of this work.
|
||||
|
||||
---
|
||||
|
||||
## Phase 4 — Build `dashboard-server` (the merge)
|
||||
|
||||
Status: **Complete (live) — 2026-05-24.** Klaviyo + Meta + Google + Typeform merged into a single ESM service at `inventory-server/dashboard/server.js`. Shared Pool + ioredis client injected through router factories. Phase 6.2 permission gates wired (`meta_write` on Meta budget/status mutations; `klaviyo_admin` on Klaviyo `/events/clearCache`). Post-cutover cleanup (2026-05-24) deleted the four old per-vendor directories (`klaviyo-server`, `meta-server`, `google-server`, `typeform-server`) along with their PM2 entries — ~1.27 GB reclaimed, largely duplicated `node_modules` across vendors. Original boot test on netcup: `/health` 200; unauthenticated `/api/klaviyo/*` returns `{"error":"No token provided"}` HTTP 401 via shared `authenticate()`.
|
||||
|
||||
### Layout
|
||||
|
||||
```
|
||||
inventory/dashboard/
|
||||
├── server.js # entry: load env, init Postgres+Redis, mount routes, listen
|
||||
├── package.json # "type": "module", deps from all 4 source servers (deduped)
|
||||
├── .env # KLAVIYO_*, META_*, GOOGLE_*, TYPEFORM_*, shared DB_*, REDIS_URL
|
||||
├── routes/
|
||||
│ ├── klaviyo/ # absorbed from dashboard/klaviyo-server/src/
|
||||
│ ├── meta/ # absorbed from dashboard/meta-server/
|
||||
│ ├── google/ # absorbed from dashboard/google-server/
|
||||
│ └── typeform/ # absorbed from dashboard/typeform-server/
|
||||
├── services/ # per-vendor API clients (Klaviyo SDK calls, etc.)
|
||||
├── scripts/
|
||||
│ └── import-campaign-products.js # one-shot, moved from klaviyo-server/scripts/
|
||||
└── logs/
|
||||
```
|
||||
|
||||
### Mount points
|
||||
|
||||
```js
|
||||
// server.js (sketch)
|
||||
import { authenticate, requirePermission } from '../../shared/auth/middleware.js';
|
||||
import { createPool } from '../../shared/db/pg.js';
|
||||
import { createRedis } from '../../shared/db/redis.js';
|
||||
import { logger, requestLog } from '../../shared/logging/index.js';
|
||||
import corsPolicy from '../../shared/cors/policy.js';
|
||||
import errorHandler from '../../shared/errors/handler.js';
|
||||
|
||||
import klaviyoRouter from './routes/klaviyo/index.js';
|
||||
import metaRouter from './routes/meta/index.js';
|
||||
import googleRouter from './routes/google/index.js';
|
||||
import typeformRouter from './routes/typeform/index.js';
|
||||
|
||||
const app = express();
|
||||
const pool = await createPool('KLAVIYO_DB'); // klaviyo has its own DB; others can share or have none
|
||||
const redis = await createRedis();
|
||||
|
||||
app.use(requestLog);
|
||||
app.use(cors(corsPolicy));
|
||||
app.use(express.json({ limit: '10mb' }));
|
||||
|
||||
// Everything below this line requires a valid token.
|
||||
app.use('/api', authenticate({ pool }));
|
||||
|
||||
app.use('/api/klaviyo', klaviyoRouter({ pool, redis }));
|
||||
app.use('/api/meta', metaRouter({ redis }));
|
||||
app.use('/api/google-analytics', googleRouter({ redis })); // matches Caddy /api/dashboard-analytics rewrite
|
||||
app.use('/api/typeform', typeformRouter({ redis }));
|
||||
|
||||
app.get('/health', (req, res) => res.json({ ok: true }));
|
||||
app.use(errorHandler);
|
||||
|
||||
app.listen(process.env.DASHBOARD_PORT || 3015);
|
||||
```
|
||||
|
||||
### Per-vendor routers
|
||||
|
||||
Each vendor's existing route file becomes a factory that takes the shared `pool`/`redis` and returns an Express router. Replace each server's per-instance pool/redis with the injected one.
|
||||
|
||||
### Permission gates (sensitive routes only)
|
||||
|
||||
Authenticated-only is the default after `app.use('/api', authenticate(...))`. For sensitive operations, add `requirePermission` per route:
|
||||
|
||||
- Anything that mutates Klaviyo lists/segments → `requirePermission('klaviyo_write')`
|
||||
- Triggering a campaign sync → `requirePermission('klaviyo_admin')`
|
||||
- Read-only dashboards → no extra check beyond authenticate.
|
||||
|
||||
Define the new permission codes in the `permissions` table via a migration in Phase 6.
|
||||
|
||||
### Dependency dedup
|
||||
|
||||
**Decision: standardize on `ioredis`.** Klaviyo's larger codebase already uses it, and `ioredis` has better cluster/sentinel support if we ever need it. Update `meta`/`google`/`typeform` call sites — each is a handful of `get`/`set` calls, mechanical conversion. Remove the `redis` package from `dashboard-server`'s `package.json`.
|
||||
|
||||
### Env consolidation
|
||||
|
||||
Single `.env` at `inventory/dashboard/.env`, prefixed keys:
|
||||
|
||||
```
|
||||
DASHBOARD_PORT=3015
|
||||
KLAVIYO_API_KEY=...
|
||||
KLAVIYO_DB_HOST=...
|
||||
KLAVIYO_DB_NAME=...
|
||||
META_ACCESS_TOKEN=...
|
||||
GOOGLE_SERVICE_ACCOUNT_KEY=...
|
||||
TYPEFORM_TOKEN=...
|
||||
REDIS_URL=...
|
||||
JWT_SECRET=... # shared with auth-server; same secret means same tokens valid here
|
||||
```
|
||||
|
||||
### Klaviyo's `scripts/import-campaign-products.js`
|
||||
|
||||
One-shot script — keep it, but run it from the merged dashboard-server's directory. Update the script's imports to ESM. If it's run via cron, update the cron entry to the new path.
|
||||
|
||||
### Risk: shared error states
|
||||
|
||||
When all four vendors share a Redis client, a Redis hiccup affects all four. Make sure the connection has retry config (`ioredis` defaults are reasonable but verify) and that vendor routes degrade gracefully when Redis is unavailable (most use it as a cache, so cache-miss → fall through to upstream API is the right behavior).
|
||||
|
||||
---
|
||||
|
||||
## Phase 5 — Convert `acot-server` to ESM (stays standalone)
|
||||
|
||||
Status: **Not started.** Largest single conversion (~5K LOC), but no merge involved.
|
||||
|
||||
### Special concern: ssh2 tunnel
|
||||
|
||||
`acot-server` opens an SSH tunnel via `ssh2` to access the production MySQL at `192.168.1.5:3309`. The tunnel must be:
|
||||
|
||||
- Established before the HTTP listener starts (so no requests fail with "no DB connection").
|
||||
- Re-established on disconnect (`ssh2` connection's `close` event → recreate).
|
||||
- Cleanly torn down on `SIGTERM`/`SIGINT` so PM2 restarts don't leak file descriptors.
|
||||
|
||||
Verify (or add) this lifecycle handling as part of the conversion. If it's already correct, conversion is mechanical; if not, this is a good moment to fix it.
|
||||
|
||||
### Test strategy
|
||||
|
||||
Same as inventory-server: start with PM2, smoke-test the most-used `/api/acot/*` endpoints, watch logs for unhandled rejection or tunnel-close events.
|
||||
|
||||
---
|
||||
|
||||
## Phase 6 — Auth hardening
|
||||
|
||||
Status: **Complete (live) — 2026-05-24.** All hardening (in-process + edge) is live in production. The Phase 3 ESM conversion + Phase 6 middleware shipped together, with Phase F1 (frontend fetch wrapper) flipping immediately ahead of the `pm2 reload` so the SPA continued to carry `Authorization: Bearer` on every API call. Caddy `forward_auth` gate and the JWT_SECRET ecosystem fix went live with the Phase 7/8 apply on 2026-05-24.
|
||||
|
||||
Per-item status:
|
||||
|
||||
| # | Item | Status | Where |
|
||||
|---|---|---|---|
|
||||
| 6.1 | Caddy `forward_auth` gate | **Live — 2026-05-24** | Applied via Caddy admin API + `sudo cp` to `/etc/caddy/Caddyfile`. `@gated path /api/* /chat-api/* /uploads/*` block hits `localhost:3011/verify` on every request |
|
||||
| 6.2 | `requirePermission` on sensitive routes + permissions migration | **Done** | inline in `config.js`, `data-management.js`, `import.js`, `ai-prompts.js`, `ai-validation.js`, `templates.js`, `reusable-images.js`; codes seeded by `migrations/005_phase6_permission_codes.sql`. **Phase 4 follow-on (2026-05-23):** `meta_write` wired on `PATCH /api/meta/campaigns/:id/budget` and `POST /api/meta/campaigns/:id/:action`; `klaviyo_admin` wired on `POST /api/klaviyo/events/clearCache`. Read-only Google + Typeform endpoints stay authenticated-only (reserved write codes left in migration 005 for future) |
|
||||
| 6.3 | Login rate-limit + `/verify` rate-limit | **Done** | `auth/server.js` uses `shared/rate-limit/login.js` (`loginLimiter`, `verifyLimiter`) |
|
||||
| 6.4 | JWT_SECRET as startup precondition + ecosystem footgun fix | **Live — 2026-05-24** | Both auth-server and inventory-server `process.exit(1)` if `JWT_SECRET` is unset. The `JWT_SECRET: process.env.JWT_SECRET` override that was shadowing `.env` is removed from the live ecosystem.cjs |
|
||||
| 6.5 | Structured request logging w/ redaction | **Done** | `shared/logging/request-log.js` (pino-http, redacts Authorization/Cookie); mounted in both `auth/server.js` and `src/server.js` |
|
||||
| 6.6 | CORS lockdown | **Done** | `src/middleware/cors.js` now re-exports `shared/cors/policy.js`. LAN wildcards (`192.168.*`, `10.*`) and `*` defaults gone |
|
||||
| 6.7 | Upload hardening | **Done** | Exact-match MIME+extension allowlist on `routes/import.js` and `routes/reusable-images.js`; dead `multer({ dest })` removed from `routes/products.js` (no upload route was using it — strongest hardening was deletion) |
|
||||
| 6.8 | Frontend token storage stays localStorage + XSS audit | **Audited** | Confirmed `dangerouslySetInnerHTML` is sanitized in `ProductEditor.tsx`. **Flagged: `ChatRoom.tsx:277,392` renders user-controlled chat content as raw HTML — real XSS vector, separate fix needed** |
|
||||
| 6.9 | Remove debug middleware | **Done** | The header-dumping `app.use((req,res,next)=>{ console.log(... req.headers ...) })` block removed from `src/server.js`. Replaced with `shared/logging/request-log.js` (which redacts). |
|
||||
| 6.10 | `lt-wordlist-api` token move | **Live — 2026-05-24** | Live PM2 entry runs `/opt/lt-wordlist-api/index.js` under matt's daemon; `ADD_WORD_TOKEN` is no longer inline in ecosystem.cjs and is read from `/opt/lt-wordlist-api/.env`. See Deviations #21–23 for the path corrections and the (incorrect) earlier assumption that this app lived under a separate root daemon |
|
||||
| 6.11 | Audit logging for sensitive ops | **Deferred** | Out of scope for this pass per user direction. Existing `import_audit_log` and `product_editor_audit_log` tables stay as-is; generic `system_audit_log` table + middleware is its own project |
|
||||
|
||||
### 6.1 Caddy `forward_auth` gate
|
||||
|
||||
Add to the `tools.acherryontop.com` block, before the `@api_routes` handler:
|
||||
|
||||
```caddyfile
|
||||
# Forward-auth gate for all API traffic
|
||||
@needs_auth path /api/* /chat-api/*
|
||||
handle @needs_auth {
|
||||
forward_auth localhost:3011 {
|
||||
uri /verify
|
||||
copy_headers Authorization
|
||||
# On 401/403, Caddy returns the auth-server's response body verbatim
|
||||
}
|
||||
# Existing per-vendor handle blocks remain below this line
|
||||
}
|
||||
|
||||
# /auth-inv/* stays public (you need to log in!)
|
||||
handle /auth-inv/* {
|
||||
uri strip_prefix /auth-inv
|
||||
reverse_proxy localhost:3011
|
||||
}
|
||||
```
|
||||
|
||||
The `forward_auth` directive subrequests `/verify` on the auth-server. If it returns 2xx, the request proceeds upstream. If 401/403, Caddy returns that response to the client and never hits the backend.
|
||||
|
||||
This is the **first** line of defense. Per-server middleware (`shared/auth/middleware.js`) is the **second** line — re-verifies the JWT independently. Defense in depth: a Caddyfile typo can't open a hole.
|
||||
|
||||
### 6.2 Per-route permission gates
|
||||
|
||||
After per-server `authenticate()`, add `requirePermission(code)` to destructive or sensitive routes. Audit needed in:
|
||||
|
||||
- `inventory-server/src/routes/config.js` — global config writes → `admin`
|
||||
- `inventory-server/src/routes/import.js` — uploads, deletes, generate-upc → `product_import`
|
||||
- `inventory-server/src/routes/data-management.js` — CSV operations → `data_management`
|
||||
- `inventory-server/src/routes/ai-prompts.js` — prompt edits → `ai_admin`
|
||||
- `inventory-server/src/routes/templates.js` — template writes → `templates_write`
|
||||
- `inventory-server/src/routes/reusable-images.js` — image management → `image_admin`
|
||||
- `inventory-server/src/routes/products.js` — only one POST (`/resolve-identifiers`); evaluate whether it needs a permission code or authenticated-only is fine
|
||||
- `inventory-server/src/routes/product-editor-audit-log.js` and `import-audit-log.js` — read-only by sensitive users → `audit_read`
|
||||
- `dashboard-server` Klaviyo/Meta/Google/Typeform write endpoints → vendor-specific codes per above
|
||||
|
||||
Migration: a single SQL script that inserts the new permission codes into the `permissions` table and assigns them to existing admin users. Non-admin users get permissions explicitly granted via the user management UI.
|
||||
|
||||
```sql
|
||||
INSERT INTO permissions (code, name) VALUES
|
||||
('product_import', 'Product Import'),
|
||||
('data_management', 'Data Management'),
|
||||
('ai_admin', 'AI Settings Admin'),
|
||||
('templates_write', 'Template Editing'),
|
||||
('image_admin', 'Image Management'),
|
||||
('audit_read', 'Audit Log Access'),
|
||||
('klaviyo_write', 'Klaviyo Write'),
|
||||
('klaviyo_admin', 'Klaviyo Admin'),
|
||||
('meta_write', 'Meta Write'),
|
||||
('google_write', 'Google Analytics Write'),
|
||||
('typeform_write', 'Typeform Write'),
|
||||
('acot_admin', 'ACOT Server Admin')
|
||||
ON CONFLICT (code) DO NOTHING;
|
||||
```
|
||||
|
||||
### 6.3 Rate limiting on login
|
||||
|
||||
`shared/rate-limit/login.js`:
|
||||
|
||||
```js
|
||||
import rateLimit from 'express-rate-limit';
|
||||
export const loginLimiter = rateLimit({
|
||||
windowMs: 15 * 60 * 1000, // 15 minutes
|
||||
max: 10, // 10 attempts per IP per window
|
||||
message: { error: 'Too many login attempts, try again later' },
|
||||
standardHeaders: true,
|
||||
legacyHeaders: false,
|
||||
});
|
||||
```
|
||||
|
||||
Apply in `auth-server` on the `/login` route. Consider also rate-limiting `/verify` and `/me` (much higher cap, ~600/min — they're called legitimately by every page load).
|
||||
|
||||
### 6.4 JWT secret rotation
|
||||
|
||||
- Rotate `JWT_SECRET` to a fresh 32-byte random string as part of the deployment.
|
||||
- Document that rotation logs out all users — acceptable for an internal tool, do it during off-hours.
|
||||
- Add `JWT_SECRET` to the env var validation block in `auth-server/server.js` (refuse to start if not set).
|
||||
- **Fix the existing footgun**: `/var/www/ecosystem.config.cjs` currently has `JWT_SECRET: process.env.JWT_SECRET` *after* `...inventoryEnv` in the new-auth-server block. This shadows the `.env` value with whatever the shell exported when PM2 was started — which has already silently diverged at least once (detected and fixed 2026-05-23 by a clean PM2 restart in a shell without JWT_SECRET exported). Delete that override line during rotation; let `.env` be the single source of truth.
|
||||
|
||||
### 6.5 Request logging
|
||||
|
||||
`shared/logging/request-log.js` — log method, path, status, duration, user-id (if authenticated). **Never** log `Authorization` or `Cookie` headers. Remove the current `server.js:79-87` debug middleware in inventory-server (it logs full headers including the bearer token).
|
||||
|
||||
### 6.6 CORS lockdown
|
||||
|
||||
Current `middleware/cors.js` allows `192.168.*.*` and `10.*.*.*` with `credentials: true`. Tighten to explicit known origins:
|
||||
|
||||
```js
|
||||
origin: [
|
||||
'https://tools.acherryontop.com',
|
||||
'https://inventory.kent.pw',
|
||||
/^http:\/\/localhost:(5174|5175)$/,
|
||||
]
|
||||
```
|
||||
|
||||
If anyone genuinely needs LAN access, add their specific IP, not a `/16` range.
|
||||
|
||||
### 6.7 Upload hardening
|
||||
|
||||
`POST /api/import/upload-image` (multer-backed) needs:
|
||||
|
||||
- File-size limit set on multer config (current limit may be defaulted — verify).
|
||||
- MIME-type allowlist (image/jpeg, image/png, image/webp; reject everything else).
|
||||
- Filename sanitization (no `..`, no absolute paths, generate UUID-based names server-side).
|
||||
- The Caddy `/uploads/*` handler currently serves any file in the uploads directory publicly. Move this **behind** the auth gate: include `/uploads/*` in `@needs_auth`. If some images are referenced from public emails (Klaviyo newsletter), put **those** in a separate public bucket; everything else stays gated.
|
||||
|
||||
### 6.8 Frontend token storage
|
||||
|
||||
**Decision: stay on `localStorage`.** This is an internal tool with no untrusted user-generated HTML being rendered, so the XSS-token-theft surface is small. The `forward_auth` gate is the main security gap we're addressing; cookie-based auth would be a larger, separate project (cookie-parser, CSRF double-submit pattern, AuthContext refactor) that doesn't change the threat model for an internal tool with no public sign-up.
|
||||
|
||||
Sanity check during this refactor: grep the React codebase for `dangerouslySetInnerHTML`. If any usages exist, verify each one is rendering trusted (server-controlled, not user-supplied) content. If a user-supplied content path exists, that's a real XSS vector and needs separate remediation regardless of token-storage choice.
|
||||
|
||||
### 6.9 Remove debug middleware
|
||||
|
||||
[inventory-server/src/server.js:79-87](inventory-server/src/server.js#L79-L87) logs full request headers including `Authorization`. Delete this block. Replace with `shared/logging/request-log.js`.
|
||||
|
||||
### 6.10 `lt-wordlist-api` token
|
||||
|
||||
`ADD_WORD_TOKEN` is currently hardcoded in `/var/www/ecosystem.config.cjs`. Move to `/opt/lt-wordlist-api/.env`, rotate the token value, update any callers.
|
||||
|
||||
### 6.11 Audit logging for sensitive operations
|
||||
|
||||
Already have `import-audit-log` and `product-editor-audit-log` tables. Extend the pattern:
|
||||
|
||||
- Log `user_id`, `endpoint`, `params`, `result` for `config.js` writes and `data-management.js` operations.
|
||||
- Schema: reuse the existing audit table pattern or add a generic `system_audit_log` table.
|
||||
- Don't log request bodies wholesale (may contain large blobs); log the action and the target ID.
|
||||
|
||||
---
|
||||
|
||||
## Phase F1 — Frontend fetch wrapper (NEW — 2026-05-23)
|
||||
|
||||
Status: **Complete (live) — 2026-05-24.** Two wrappers landed at `inventory/src/utils/api.ts` and `inventory/src/utils/apiClient.ts`. Migration touched 87 files (76 fetch, 11 axios) covering ~200 call sites. Type-check clean; production build clean. Intentional exclusions: AuthContext `/login`+`/me` (own auth flow), App.tsx initial `/me` session check, and `services/apiv2.ts` (calls the separate PHP backend at backend.acherryontop.com which has its own cookie auth, out of scope per the plan). Shipped alongside the Phase 3+6 pm2 reload.
|
||||
|
||||
### The discovery
|
||||
|
||||
While wiring `authenticate()` on `/api/*` in Phase 6.1/6.2, we audited the frontend's fetch usage and found:
|
||||
|
||||
- **7** call sites send `Authorization: Bearer ${token}` explicitly (all in `AuthContext.tsx` for `/me` + `/login`, plus a couple of `settings/*` pages).
|
||||
- **~220** other `fetch(...)` / `axios.*(...)` call sites across `inventory/src/services/`, `inventory/src/pages/`, `inventory/src/components/` send **no** Authorization header at all.
|
||||
- There is no global fetch wrapper, axios interceptor, or service-worker shim that injects the token.
|
||||
|
||||
Today this works because nothing on the server checks. Caddy currently has no `forward_auth` gate (Phase 6.1 is a Caddyfile change that hasn't shipped yet) and the previous inventory-server had no `authenticate()` middleware. The frontend's auth model was "you log in once to get the token; the token is checked only by `/me`; everything else is implicitly trusted at the network layer."
|
||||
|
||||
With Phase 6 code in production, **every page refresh 401s** on the first API call after the next pm2 reload. The user explicitly accepted this when authorising the Phase 6 work — but the fix is its own deliverable, and shipping Phase 3+6 to PM2 without F1 in the same window means an outage window measured in *however long F1 takes* (not minutes).
|
||||
|
||||
### Recommended approach
|
||||
|
||||
Add a single fetch wrapper at `inventory/src/utils/api.ts` (or similar) and migrate the ~220 call sites to use it. The wrapper:
|
||||
|
||||
1. Reads `localStorage.getItem('token')` on every call (cheap; localStorage is sync).
|
||||
2. Merges `Authorization: Bearer ${token}` into the request headers if a token exists.
|
||||
3. Intercepts 401 responses → fires `window.dispatchEvent(new Event('auth:logout'))` (a listener already exists in `AuthContext.tsx:117`) so the user gets bounced to `/login` cleanly instead of seeing broken pages.
|
||||
4. Preserves the existing call shape — `apiFetch(url, init)` should be a drop-in for `fetch(url, init)` so the migration is mechanical.
|
||||
|
||||
```ts
|
||||
// inventory/src/utils/api.ts (sketch)
|
||||
export async function apiFetch(input: RequestInfo | URL, init: RequestInit = {}): Promise<Response> {
|
||||
const token = localStorage.getItem('token');
|
||||
const headers = new Headers(init.headers);
|
||||
if (token && !headers.has('Authorization')) {
|
||||
headers.set('Authorization', `Bearer ${token}`);
|
||||
}
|
||||
const res = await fetch(input, { ...init, headers });
|
||||
if (res.status === 401 && token) {
|
||||
// Token expired or revoked — bounce to /login. AuthContext already listens.
|
||||
window.dispatchEvent(new Event('auth:logout'));
|
||||
}
|
||||
return res;
|
||||
}
|
||||
```
|
||||
|
||||
Same shape for axios:
|
||||
|
||||
```ts
|
||||
// inventory/src/utils/apiClient.ts (sketch)
|
||||
import axios from 'axios';
|
||||
export const apiClient = axios.create();
|
||||
apiClient.interceptors.request.use((config) => {
|
||||
const token = localStorage.getItem('token');
|
||||
if (token) config.headers.Authorization = `Bearer ${token}`;
|
||||
return config;
|
||||
});
|
||||
apiClient.interceptors.response.use(
|
||||
(r) => r,
|
||||
(err) => {
|
||||
if (err?.response?.status === 401) window.dispatchEvent(new Event('auth:logout'));
|
||||
return Promise.reject(err);
|
||||
},
|
||||
);
|
||||
```
|
||||
|
||||
### Migration plan
|
||||
|
||||
1. Land the two wrapper modules above. ~50 LOC total.
|
||||
2. Codemod or sed-loop: in `inventory/src/`, replace `fetch(` → `apiFetch(` (with the right import) and `axios.get/post/...` → `apiClient.get/post/...`. ~220 call sites — a half-day of careful find-and-replace plus per-page verification. Spot-check the ones with custom `Content-Type` (multipart uploads especially) so the wrapper doesn't clobber multipart boundaries.
|
||||
3. Leave the `AuthContext.tsx` `/login` and `/me` calls alone — they already work and migrating them adds no value.
|
||||
4. Run the SPA: log in, exercise Overview / Products / Analytics / Dashboard / etc. with browser devtools open watching for `Authorization` header on every `/api/*` request.
|
||||
|
||||
### Sequencing with Phase 3+6 deploy
|
||||
|
||||
**Two options:**
|
||||
|
||||
A) **Ship F1 first** (recommended). Frontend goes out with the wrapper; nothing changes server-side. Then `pm2 reload` Phase 3+6. Zero-downtime, zero broken-page window.
|
||||
|
||||
B) **Ship together.** F1 and Phase 3+6 land in the same deploy. Brief window (seconds) where the frontend has the wrapper but the server hasn't reloaded yet — wrapper just sends extra headers the old server ignores. Safe.
|
||||
|
||||
Do **not** ship Phase 3+6 first and F1 second. That gives a broken app for as long as F1 takes.
|
||||
|
||||
### Out of scope (kept on `localStorage`)
|
||||
|
||||
Per Phase 6.8, we're not migrating to httpOnly cookie auth. F1 is the minimum work to make the per-service `authenticate()` (Phase 6) actually usable. A future Phase F2 could move to cookies + CSRF double-submit, but that's a much larger change touching the AuthContext, the login flow, and every backend that reads tokens. Not justified for an internal tool with no public sign-up.
|
||||
|
||||
### Note on `/uploads/*` gating (Phase 6.7's Caddyfile change)
|
||||
|
||||
**Applied as-spec (2026-05-24):** `/uploads/*` is behind `forward_auth` in the live Caddyfile. `<img src="/uploads/...">` references in the SPA are browser-issued GETs that don't carry `Authorization` headers — verify image display works end-to-end (cookies fall-through, signed URLs, or session-bound forward_auth) and if broken, revert this part of 6.7 to keep `/uploads/*` public, OR issue per-image signed URLs from the API.
|
||||
|
||||
---
|
||||
|
||||
## Phase 7 — Caddyfile final form
|
||||
|
||||
Status: **Complete — applied 2026-05-24.** Final Caddyfile live at `/etc/caddy/Caddyfile`; vendor handles point at the merged dashboard-server on :3015. The `inventory-server/deploy/` staging folder (which held `Caddyfile.proposed` and the README of apply commands) was removed after apply — recreate from the spec below if future changes are needed. Apply pattern (admin-API `curl -X POST :2020/load` + `sudo cp` to persist on-disk) is captured in Deviation #8. Backup convention: `/etc/caddy/Caddyfile.bak.YYYY-MM-DD`.
|
||||
|
||||
After all phases, the `tools.acherryontop.com` block looks like:
|
||||
|
||||
```caddyfile
|
||||
tools.acherryontop.com {
|
||||
import security_headers
|
||||
|
||||
# Public: login endpoint
|
||||
handle /auth-inv/* {
|
||||
uri strip_prefix /auth-inv
|
||||
reverse_proxy localhost:3011
|
||||
}
|
||||
|
||||
# Public: static frontend assets
|
||||
@static path *.js *.css *.png *.jpg *.jpeg *.gif *.ico *.svg *.woff *.woff2
|
||||
handle @static {
|
||||
header Cache-Control "public, max-age=2592000"
|
||||
root * /var/www/inventory/frontend/build
|
||||
file_server
|
||||
}
|
||||
|
||||
# All API + uploads: auth gate first
|
||||
@gated path /api/* /chat-api/* /uploads/*
|
||||
handle @gated {
|
||||
forward_auth localhost:3011 {
|
||||
uri /verify
|
||||
copy_headers Authorization
|
||||
}
|
||||
|
||||
# Uploaded files
|
||||
handle /uploads/* {
|
||||
root * /var/www/inventory
|
||||
file_server
|
||||
}
|
||||
|
||||
# Vendor dashboard routes → merged dashboard-server
|
||||
handle /api/klaviyo/* { reverse_proxy localhost:3015 }
|
||||
handle /api/meta/* { reverse_proxy localhost:3015 }
|
||||
handle /api/dashboard-analytics/* { reverse_proxy localhost:3015 }
|
||||
handle /api/typeform/* { reverse_proxy localhost:3015 }
|
||||
|
||||
# ACOT-specific
|
||||
handle /api/acot/* { reverse_proxy localhost:3012 }
|
||||
|
||||
# Chat
|
||||
handle /chat-api/* {
|
||||
uri strip_prefix /chat-api
|
||||
reverse_proxy localhost:3014
|
||||
}
|
||||
|
||||
# Catch-all: inventory-server
|
||||
handle /api/* { reverse_proxy localhost:3010 }
|
||||
}
|
||||
|
||||
handle /health { reverse_proxy localhost:3010 }
|
||||
|
||||
# SPA fallback
|
||||
handle {
|
||||
root * /var/www/inventory/frontend/build
|
||||
try_files {path} /index.html
|
||||
file_server
|
||||
encode gzip
|
||||
}
|
||||
|
||||
handle_errors {
|
||||
respond "{err.status_code} {err.status_text}"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Removed: `/dashboard-auth/*`, `/api/aircall/*`, `/api/gorgias/*`, `/api/clarity/*`, the LAN/`Access-Control-Allow-Origin "*"` permissive defaults on `/api/*`. Kept: `/apiv2/*` and `/apiv2-test/*` proxies to backend.acherryontop.com (out of scope, separate system).
|
||||
|
||||
---
|
||||
|
||||
## Phase 8 — ecosystem.config.cjs final form
|
||||
|
||||
Status: **Complete — applied 2026-05-24.** Live PM2 list matches the spec below: 5 application apps (auth-server, inventory-server, dashboard-server, acot-server, chat-server) plus acot-phone-server + lt-wordlist-api = 7 total. Includes the Phase 6.4 `JWT_SECRET` shadow-override fix and the Phase 6.10 `lt-wordlist-api` token move. The `inventory-server/deploy/ecosystem.config.cjs.proposed` staging file was removed after apply — recreate from the spec below if future changes are needed.
|
||||
|
||||
```js
|
||||
module.exports = {
|
||||
apps: [
|
||||
{
|
||||
name: 'auth-server',
|
||||
script: './inventory/auth/server.js',
|
||||
cwd: '/var/www',
|
||||
env: { NODE_ENV: 'production', AUTH_PORT: 3011 },
|
||||
...commonSettings,
|
||||
},
|
||||
{
|
||||
name: 'inventory-server',
|
||||
script: './inventory/src/server.js',
|
||||
cwd: '/var/www',
|
||||
env: { NODE_ENV: 'production', PORT: 3010, UPLOADS_DIR: '/var/www/inventory/uploads' },
|
||||
...commonSettings,
|
||||
},
|
||||
{
|
||||
name: 'dashboard-server',
|
||||
script: './inventory/dashboard/server.js',
|
||||
cwd: '/var/www',
|
||||
env: { NODE_ENV: 'production', DASHBOARD_PORT: 3015 },
|
||||
...commonSettings,
|
||||
},
|
||||
{
|
||||
name: 'acot-server',
|
||||
script: './inventory/dashboard/acot-server/server.js',
|
||||
cwd: '/var/www',
|
||||
env: { NODE_ENV: 'production', ACOT_PORT: 3012 },
|
||||
...commonSettings,
|
||||
},
|
||||
{
|
||||
name: 'chat-server',
|
||||
script: './inventory/chat/server.js',
|
||||
cwd: '/var/www',
|
||||
env: { NODE_ENV: 'production', PORT: 3014 },
|
||||
...commonSettings,
|
||||
},
|
||||
// acot-phone-server and lt-wordlist-api unchanged
|
||||
],
|
||||
};
|
||||
```
|
||||
|
||||
Five entries instead of twelve. Each app loads its own `.env` from its directory (already handled by `dotenv.config`).
|
||||
|
||||
---
|
||||
|
||||
## Sequencing & dependencies
|
||||
|
||||
```
|
||||
Phase 1 (decommission) ──┬─────────────────────────────────────────┐
|
||||
│ │
|
||||
▼ │
|
||||
Phase 2 (shared lib/) │
|
||||
│ │
|
||||
┌──────────────┼──────────────┐ │
|
||||
▼ ▼ ▼ ▼
|
||||
Phase 3a Phase 3b Phase 4 Phase 6 (auth hardening
|
||||
inventory-server auth-server dashboard-server runs alongside 3+4+5,
|
||||
to ESM to ESM + /verify build & test completes after them)
|
||||
│ │ │ │
|
||||
└──────────────┼──────────────┘ │
|
||||
▼ │
|
||||
Phase 5 (acot-server to ESM) ──────────────────►│
|
||||
▼
|
||||
Phase 7 (Caddy cutover)
|
||||
│
|
||||
▼
|
||||
Phase 8 (PM2 final state)
|
||||
```
|
||||
|
||||
Phase 1 unblocks everything (fewer services to convert).
|
||||
Phase 2 is the foundation; nothing else can start until shared `lib/` exists.
|
||||
Phases 3–5 can run in parallel; they touch independent services.
|
||||
Phase 6's sub-items can be developed alongside 3–5 but **enabled** only after them (no point adding `requirePermission` to a route that doesn't yet have `authenticate`).
|
||||
**Phase F1 must precede the Phase 3+6 pm2 reload** — without the fetch wrapper, the moment the new code goes live the SPA breaks. Discovered during Phase 3+6 implementation; see Phase F1.
|
||||
Phase 7 is the cutover: Caddyfile flip happens after F1 ships AND after the `/uploads/*` gating decision in F1 is made.
|
||||
Phase 8 is cleanup: remove dead PM2 entries.
|
||||
|
||||
Estimated effort, end-to-end: **~3 weeks of focused work** by one engineer. Phase 1 ≈ 1 day, Phase 2 ≈ 2 days, Phase 3 ≈ 3 days (both services), Phase 4 ≈ 5–7 days (the merge), Phase 5 ≈ 2–3 days, Phase 6 ≈ 3–4 days, Phase F1 ≈ 0.5–1 day, Phase 7+8 ≈ 1 day.
|
||||
|
||||
---
|
||||
|
||||
## Testing strategy
|
||||
|
||||
No formal test suite exists today (per CLAUDE.md). For a refactor this size, that's a gap to close — but writing tests retroactively for 15K LOC of routes is a separate, larger project. For this refactor:
|
||||
|
||||
### Manual smoke testing per phase
|
||||
|
||||
A checklist of representative endpoints to hit after each deploy:
|
||||
|
||||
- `inventory-server`: `/api/products`, `/api/dashboard/overview`, `/api/analytics/revenue`, `/api/orders`, `/api/purchase-orders`, `/api/import/list-uploads`, `/api/config/global`
|
||||
- `dashboard-server`: `/api/klaviyo/campaigns`, `/api/meta/insights`, `/api/google-analytics/...`, `/api/typeform/responses`
|
||||
- `acot-server`: `/api/acot/...` (top-3 endpoints by call volume — pull from access logs)
|
||||
- `auth-server`: `/login`, `/me`, `/verify`
|
||||
|
||||
Each smoke test runs (a) without a token → expect 401, (b) with an invalid token → expect 401, (c) with a valid token → expect 2xx.
|
||||
|
||||
### Frontend integration check
|
||||
|
||||
After deploys, log into the SPA and exercise each major page (Overview, Products, Analytics, Dashboard, Klaviyo, Meta, etc.). If everything loads and dashboards populate, the auth + routing layer is intact.
|
||||
|
||||
### Test scaffold during Phase 2 (committed)
|
||||
|
||||
While building `shared/`, set up `vitest` (lightweight, ESM-native, fast) as the standard test runner for the repo. Initial coverage focuses on the security-critical surface only:
|
||||
|
||||
- `shared/auth/verify.js` — known good token, expired token, wrong-signature token, malformed token, missing token.
|
||||
- `shared/auth/middleware.js` — request with no header → 401; bad header → 401; valid token + inactive user → 403; valid token + missing permission → 403; valid token + correct permission → next() called with `req.user` populated.
|
||||
- `shared/auth/middleware.js` user-cache TTL: same token within 60s → one DB hit; same token after 61s → two DB hits.
|
||||
|
||||
`package.json` gets a `"test": "vitest run"` script at the repo root and per-service. Set up but don't backfill broader test coverage — that's a separate, larger project. The vitest scaffold gives future work a foothold; this refactor commits to having tests for the auth boundary specifically because that's what's load-bearing for the whole security model.
|
||||
|
||||
---
|
||||
|
||||
## Rollback strategy
|
||||
|
||||
Each phase produces an independently deployable state. Rollback per phase:
|
||||
|
||||
- **Phase 1**: re-add removed services to ecosystem; restore from git. Don't roll back data deletions — only do those after a week of stable production.
|
||||
- **Phases 3, 5**: ESM conversion is per-service; if one service breaks, `pm2 restart <name>` to the previous commit. Other services unaffected.
|
||||
- **Phase 4**: the dashboard-server merge is the highest-risk change. Plan: deploy `dashboard-server` to a non-conflicting port (3015) while leaving the old per-vendor servers running. Cut over Caddy routes one vendor at a time (start with Meta — smallest). If any vendor breaks, point Caddy back to the old server (still running) for that vendor, debug, retry. Only delete the old servers after all four are stable on `dashboard-server`.
|
||||
- **Phases 6, 7**: Caddy config is git-tracked. `git revert` + `caddy reload` rolls back in seconds. Auth changes are additive (defense in depth) — if `forward_auth` causes problems, comment it out and per-server middleware continues protecting routes.
|
||||
|
||||
---
|
||||
|
||||
## Out of scope (intentional)
|
||||
|
||||
These came up in the audit but aren't part of this refactor:
|
||||
|
||||
- `httpOnly` cookie auth ("Phase F2" — deferred). Phase F1 keeps `localStorage` + Bearer header because that's the minimum to unblock the Phase 6 `authenticate()` rollout. A future move to cookie auth would touch `AuthContext`, every backend that reads tokens, and introduce CSRF concerns — much larger project.
|
||||
- Replacing PM2 with systemd or Docker.
|
||||
- Test coverage beyond the auth-critical surface.
|
||||
- `apiv2`/`apiv2-test` proxies to `backend.acherryontop.com` — separate system, not touched.
|
||||
- `acot-phone-server` and `lt-wordlist-api` — staying as-is.
|
||||
- Centralized observability stack (Prometheus, Grafana). The logger work in Phase 6.5 sets up the data, but shipping it somewhere is future work.
|
||||
- ChatRoom XSS remediation (flagged during Phase 6.8 audit — `inventory/src/components/chat/ChatRoom.tsx:277,392` renders user-controlled chat content via `dangerouslySetInnerHTML` without sanitization). Real vulnerability for an internal-but-multi-user tool; separate fix.
|
||||
|
||||
---
|
||||
|
||||
## Concrete deliverables
|
||||
|
||||
State as of 2026-05-24: everything below is **shipped** except Phase 5 (acot-server ESM conversion), which is the only remaining work item. Note: the "4 application PM2 processes" original target became **5** in execution because `chat-server` stayed standalone rather than being folded in — never a serious merge candidate (different DB, different protocol shape).
|
||||
|
||||
- ✅ 5 application PM2 processes instead of 12 (auth-server, inventory-server, dashboard-server, acot-server, chat-server) — plus 2 unchanged (acot-phone-server, lt-wordlist-api) = 7 total.
|
||||
- ✅ All `/api/*`, `/chat-api/*`, and `/uploads/*` requests gated at Caddy (`forward_auth`) and re-verified at each upstream (`authenticate()`).
|
||||
- ✅ Sensitive endpoints additionally gated by per-permission checks (`requirePermission`).
|
||||
- ⚠️ One ESM standard — done for auth/inventory/dashboard/chat. **acot-server still CJS (Phase 5 pending).**
|
||||
- ✅ One shared `lib/` at `inventory-server/shared/` for auth, logging, DB, errors, CORS.
|
||||
- ✅ Login rate-limited (`shared/rate-limit/login.js`).
|
||||
- ✅ `JWT_SECRET` rotated + ecosystem shadow-override removed.
|
||||
- ✅ Old auth-server, Aircall, Gorgias, Clarity directories deleted from the repo. Defunct `dashboard:gorgias`/`dashboard:calls` permission rows also deleted from DB (2026-05-24).
|
||||
- ✅ Caddyfile slimmed to one auth-gated block.
|
||||
- ✅ Permission codes inserted into `permissions` table for granular authorization.
|
||||
- ✅ No half-finished pieces, no `// TODO: add auth later` comments, no deferred secrets cleanup.
|
||||
|
||||
---
|
||||
|
||||
## Deviations from original plan (recorded during execution)
|
||||
|
||||
These are decisions made during Phase 1/2 implementation that amend the spec above. Future phases should follow the deviated path, not the original sketch.
|
||||
|
||||
1. **`shared/` location.** Original plan placed `shared/` at the repo root as a sibling of `inventory/` and `acot-phone/`. Implemented at `inventory-server/shared/` (= `/var/www/inventory/shared/` on the server) instead. Reason: the actual project root *is* `/var/www/inventory/`; placing shared/ outside it would have meant building a deployment story for it that doesn't exist. Import paths change accordingly:
|
||||
- From `inventory-server/{auth,src,chat}/server.js` → `../shared/...`
|
||||
- From `inventory-server/dashboard/{vendor}-server/server.js` → `../../shared/...`
|
||||
|
||||
2. **`/verify` response headers.** Plan specified `X-User-Id` + `X-User-Is-Admin`. Implemented as `X-User-Id` + `X-User-Username` (both available from the JWT payload). `X-User-Is-Admin` was dropped because `is_admin` isn't in the JWT today and returning it would require a DB lookup — violating the "no DB hit" principle. To restore `X-User-Is-Admin`, enrich the JWT payload at login time (one-line change in `auth/routes.js`) during Phase 6, then echo from `/verify`. Upstreams don't trust these headers anyway (they re-verify), so the omission is informational, not security-relevant.
|
||||
|
||||
3. **User cache key in `shared/auth/middleware.js`.** Plan sketch mentioned "60s TTL keyed by token jti". Implemented as keyed by `userId` instead — the JWT doesn't currently include a `jti` claim, and the cache's invalidation semantics are "this user was deactivated/changed permissions" (per-user), not "this token was revoked" (per-token). The plan's pseudocode already used `loadUserCached(pool, decoded.userId)` so this matches the spirit.
|
||||
|
||||
4. **Redis client safety.** `shared/db/redis.js` sets `enableOfflineQueue: false` and `lazyConnect: true`. Plan didn't specify but these defaults mean a Redis hiccup fails fast (route fall-through to upstream API as designed in Phase 4 risk notes) rather than queueing commands indefinitely.
|
||||
|
||||
5. **CORS allowed origins kept `https://acot.site`.** Plan example listed three origins; production has acot.site as a redirect to tools.acherryontop.com but also reaches the API directly in some flows. Kept it to avoid breakage. LAN wildcards (`192.168.*`, `10.*`) and `Access-Control-Allow-Origin "*"` are NOT included in the new `shared/cors/policy.js` per the plan's Phase 6.6 spirit, but the legacy `inventory-server/src/middleware/cors.js` still has them until services are migrated to consume `shared/cors/`.
|
||||
|
||||
6. **Defunct permission codes cleaned up (2026-05-24).** Removed the `dashboard:gorgias` and `dashboard:calls` Protected blocks from the frontend; the corresponding permission rows in the `permissions` table (and their user_permission grants) were deleted in a follow-up migration alongside the Phase 6.2 permissions seed. Verified post-migration: `permissions` table contains only the in-use `dashboard:*` codes (analytics, campaigns, feed, financial, meta_campaigns, operations, payroll, products, realtime, sales, stats, typeform, user_behavior).
|
||||
|
||||
7. **PM2 process names retained `new-auth-server` (not `auth-server`).** Plan's Phase 8 final form names it `auth-server` (after the legacy 3003 one is removed). Decided to keep the existing `new-auth-server` name through Phase 2 to avoid a rename mid-stream. Phase 8 can rename if desired, but it's cosmetic — all wiring is by port (3011) not name.
|
||||
|
||||
8. **Caddyfile changes via admin API on `:2020`.** The Caddyfile is owned by root and matt has no passwordless sudo. Cutover used `curl -X POST .../load` on the Caddy admin port (which matt can hit), then a separate `sudo cp /home/matt/Caddyfile.new /etc/caddy/Caddyfile` step to persist the on-disk file. Future Caddyfile changes can follow the same pattern. Backup convention: `/etc/caddy/Caddyfile.bak.YYYY-MM-DD`.
|
||||
|
||||
9. **Path-naming.** Plan uses `inventory/` as the top-level (server-side path convention). Locally the equivalent is `inventory-server/`. Whenever the plan says `inventory/dashboard/foo/`, read that as `/var/www/inventory/dashboard/foo/` on the server or `inventory-server/dashboard/foo/` locally.
|
||||
|
||||
10. **Scripts directory kept CJS via package.json shim.** Original plan called for converting "any spawned script" to ESM alongside its caller. Implemented: added `inventory-server/scripts/package.json` with `"type": "commonjs"`. Node's package-type resolution walks up directory by directory, so this overrides the parent's `"type": "module"` for the entire `scripts/` tree (≈15 files including `import/*.js`, `metrics-new/utils/*`, the orchestrator scripts) without renaming any file or touching any `spawn()` callsite. Convert individual scripts to ESM when touched; don't bulk-migrate.
|
||||
|
||||
11. **`src/routes/products.js` had dead multer setup.** Phase 6.7 spec called for hardening the upload route in products.js. There was no upload route — the `multer({ dest })` instance and `importProductsFromCSV` import were dead code left over from a long-ago migration. Strongest 6.7 hardening was deletion: no upload handler = no attack surface. The two real upload paths (`/api/import/upload-image` and `/api/reusable-images/upload`) got tightened MIME+extension allowlists instead.
|
||||
|
||||
12. **Two pre-existing syntax errors in shared/db/ surfaced.** `shared/db/pg.js:13` and `shared/db/redis.js:22` both had `?? Number(...) || N` — mixing `??` and `||` without parentheses is a TC39 syntax error. They passed Phase 2 because nothing imported them yet; Phase 3 smoke-test exposed it. Fixed with parens.
|
||||
|
||||
13. **`import { Pool } from 'pg'` doesn't work in ESM.** The `pg` package is CJS using `module.exports = { Pool, ... }`. Node's ESM-from-CJS interop fails to detect `Pool` as a named export via static analysis. The bulletproof pattern, now used everywhere: `import pg from 'pg'; const { Pool } = pg;`. Same idea for any future CJS-only deps. `src/utils/db.js` already had it; the two auth files needed the fix during execution.
|
||||
|
||||
14. **Frontend Bearer-header gap discovered (drives new Phase F1).** Phase 6 was specified assuming the frontend already sends `Authorization: Bearer` on every API call. It does not — only 7 of ~220 call sites do. Phase 6's `authenticate()` middleware is shipped and ready to enable, but until F1 lands the SPA will 401 on every page. The plan now has Phase F1 to address this explicitly; until then, the Phase 3+6 pm2 reload should not ship unless F1 ships in the same window.
|
||||
|
||||
15. **macOS NFS workflow note.** The `inventory-server/` directory locally is an NFS mount of `/var/www/inventory/` on netcup. Bulk operations (`find`/`grep -r`/mass `node --check`/`npm install`) hang or take minutes locally and pollute file listings with macOS AppleDouble `._*` sidecar files. Default to `ssh netcup` for any sweep across the tree — individual file edits via the editor are fine.
|
||||
|
||||
16. **dashboard-server lives at `inventory-server/dashboard/` (not its own top-level dir).** Plan's Phase 4 diagram implied a sibling of `inventory-server/` etc. The merged service lives at `inventory-server/dashboard/server.js` with `package.json` declaring `"type": "module"`. Per-vendor subdirectories (`klaviyo-server/`, `meta-server/`, `google-server/`, `typeform-server/`) each have their own `package.json` so Node's "nearest parent package.json" walk stops there — they are unaffected by the new parent type. Added `"type": "commonjs"` defensively to meta/google/typeform package.json so a future deletion of their files (cutover cleanup) plus a stray `*.js` left under `dashboard/` wouldn't accidentally try to ESM-parse it.
|
||||
|
||||
17. **Klaviyo `RedisService` kept as a wrapper, but accepts injected client.** Plan said "replace each server's per-instance pool/redis with the injected one." The Klaviyo codebase has ~3K LOC of service code (`events.service.js` alone is 2.2K) that calls `this.redisService._getCacheKey()`, `.get()`, `.set()`, `.getEventData()`, `.clearCache()`, `._getTTL()`. Rewriting all of that to call ioredis directly would risk breaking the cache-key/TTL invariants. Decision: keep `services/klaviyo/redis.service.js` as a thin facade with the same public surface, but its constructor now takes an ioredis client instead of constructing one. The 3 service classes (`EventsService`, `CampaignsService`, `ReportingService`) all take `(apiKey, apiRevision, redis)` and pass `redis` to `new RedisService(redis)`. `MetricsService` doesn't use Redis — left unchanged.
|
||||
|
||||
18. **dashboard-server `.env` layering.** Plan called for "Single `.env` at `inventory/dashboard/.env`, prefixed keys: KLAVIYO_*, META_*, ... JWT_SECRET ... # shared with auth-server." Implemented as two-file layering: `server.js` loads `/var/www/inventory/.env` FIRST (provides JWT_SECRET, DB_*, REDIS_*) then `inventory-server/dashboard/.env` SECOND for vendor-specific keys (KLAVIYO_API_KEY, META_*, GA_*, TYPEFORM_*). dotenv defaults to `override:false`, so the first file wins on collisions — security-critical vars live in one place, vendor keys in the other. `.env.example` template committed at `dashboard/.env.example`. **Pre-cutover step**: copy the vendor keys from the current per-vendor `.env` files into either of those two files before `pm2 reload`, else KLAVIYO_API_KEY etc. will not be set and routes will 500.
|
||||
|
||||
19. **Caddyfile typo fixed: `/api/google-analytics` → `/api/dashboard-analytics`.** The pre-Phase-4 `Caddyfile.proposed` listed a `handle /api/google-analytics/*` block. The live Caddyfile and the frontend (`inventory/src/config/dashboard.ts`) both use `/api/dashboard-analytics/*` (the live file has a `uri replace /api/dashboard-analytics /api/analytics` rewrite to land on google-server's `/api/analytics` mount). The merged dashboard-server now mounts the Google router at `/api/dashboard-analytics` directly — Caddy no longer needs the rewrite, just a straight reverse_proxy. Fixed in `deploy/Caddyfile.proposed`.
|
||||
|
||||
20. **`metrics.routes.js` had a latent router-scope bug.** The Klaviyo `metrics.routes.js` declared `const router = express.Router()` at MODULE scope (outside the `createMetricsRouter` factory), so calling the factory twice would have re-mounted handlers on the same router (cumulative). Benign for a single-mount PM2 service, but fixed during the Phase 4 copy — the router now lives inside the factory. Also renamed the export from `createMetricsRoutes` (plural) to `createMetricsRouter` (matches the convention used by every other vendor's index.js).
|
||||
|
||||
21. **PM2 log paths use per-server `logs/pm2/` (NOT `/var/log/pm2/`).** Discovered during the first apply attempt: the previously-shipped `ecosystem.config.cjs.proposed` carried over `/var/log/pm2/...` from the live file, but matt has no write perms on `/var/log` (root:syslog 775) so the entries silently failed to launch (chat-server + acot-server came up because they had no explicit log path; new-auth-server, inventory-server, dashboard-server bailed). The actual convention — already in place via pre-created folders on disk — is per-service `logs/pm2/` directly under each service's directory (`./inventory/auth/logs/pm2/`, `./inventory/chat/logs/pm2/`, `./inventory/dashboard/acot-server/logs/pm2/`, `./inventory/dashboard/logs/pm2/` for the merged dashboard-server, `./inventory/logs/pm2/` for inventory-server, `/opt/lt-wordlist-api/logs/pm2/`, `/var/www/acot-phone/logs/pm2/`). All folders are matt:matt. `pm2-logrotate` (already loaded in matt's daemon) rotates them in place.
|
||||
|
||||
22. **All PM2 apps run under matt's single daemon — no root daemon.** The earlier `OUT OF SCOPE` comment block in the proposed ecosystem incorrectly claimed `lt-wordlist-api` and `acot-phone-server` were managed by a separate root PM2 daemon. They are not — matt's daemon manages everything. Removed the bogus block; both apps are now first-class entries in the proposed ecosystem with corrected script paths:
|
||||
- `lt-wordlist-api` script is `/opt/lt-wordlist-api/index.js` (was `/opt/lt-wordlist-api/server.js` in the live file — wrong; that file doesn't exist). `/opt/lt-wordlist-api` is matt:matt 0750.
|
||||
- `acot-phone-server` script is `/var/www/acot-phone/dist/server.js` (was `./inventory/acot-phone/server.js` in the live file — wrong; that path doesn't exist). `/var/www/acot-phone/` is matt:matt with its own `.env` and is a separate repo from inventory-server.
|
||||
|
||||
23. **Phase 6.10 ADD_WORD_TOKEN move stays in this ecosystem.** Per Deviation #22, `lt-wordlist-api` is in matt's ecosystem, so the §6.10 work to remove inline `ADD_WORD_TOKEN` and load it from `/opt/lt-wordlist-api/.env` instead is implemented directly in `deploy/ecosystem.config.cjs.proposed` (no inline `ADD_WORD_TOKEN`; script reads its own .env). When applying, rotate the token value in `/opt/lt-wordlist-api/.env` and update any callers.
|
||||
@@ -0,0 +1,375 @@
|
||||
# Product Import Module - Enhancement & Issues Outline
|
||||
|
||||
This document outlines the investigation and implementation requirements for each requested enhancement to the product import module.
|
||||
|
||||
---
|
||||
|
||||
## 1. UPC Import - Strip Quotes and Spaces ✅ IMPLEMENTED
|
||||
|
||||
**Issue:** When importing UPCs, strip `'`, `"` characters and any spaces, leaving only numbers.
|
||||
|
||||
**Implementation (Completed):**
|
||||
- Modified `normalizeUpcValue()` in [Import.tsx:661-667](inventory/src/pages/Import.tsx#L661-L667)
|
||||
- Strips single quotes, double quotes, smart quotes (`'"`), and whitespace before processing
|
||||
- Then handles scientific notation and extracts only digits
|
||||
|
||||
**Files Modified:**
|
||||
- `inventory/src/pages/Import.tsx` - `normalizeUpcValue()` function
|
||||
|
||||
---
|
||||
|
||||
## 2. AI Context Columns in Validation Payloads ✅ IMPLEMENTED
|
||||
|
||||
**Issue:** The match columns step has a setting to use a field only for AI context (`isAiSupplemental`). Update AI description validation to include any columns selected with this option in the payload. Also include in sanity check payload. Not needed for names.
|
||||
|
||||
**Current Implementation:**
|
||||
- AI Supplemental toggle: [MatchColumnsStep.tsx:102-118](inventory/src/components/product-import/steps/MatchColumnsStep/MatchColumnsStep.tsx#L102-L118)
|
||||
- AI supplemental data stored in `__aiSupplemental` field on each row
|
||||
- Description payload builder: [inlineAiPayload.ts:183-195](inventory/src/components/product-import/steps/ValidationStep/utils/inlineAiPayload.ts#L183-L195)
|
||||
|
||||
**Implementation:**
|
||||
1. **Update `buildDescriptionValidationPayload()` in `inlineAiPayload.ts`** to include AI supplemental data:
|
||||
```typescript
|
||||
export const buildDescriptionValidationPayload = (
|
||||
row: Data<string>,
|
||||
fieldOptions: FieldOptionsMap,
|
||||
productLinesCache: Map<string, SelectOption[]>,
|
||||
sublinesCache: Map<string, SelectOption[]>
|
||||
) => {
|
||||
const payload: Record<string, unknown> = {
|
||||
name: row.name,
|
||||
description: row.description,
|
||||
company_name: getFieldOptionLabel(row.company, fieldOptions, 'company'),
|
||||
company_id: row.company,
|
||||
categories: getFieldOptionLabel(row.category, fieldOptions, 'category'),
|
||||
};
|
||||
|
||||
// Add AI supplemental context if present
|
||||
if (row.__aiSupplemental && typeof row.__aiSupplemental === 'object') {
|
||||
payload.additional_context = row.__aiSupplemental;
|
||||
}
|
||||
|
||||
return payload;
|
||||
};
|
||||
```
|
||||
|
||||
2. **Update sanity check payload** - Locate sanity check submission logic and include `__aiSupplemental` data
|
||||
|
||||
3. **Verify `__aiSupplemental` is properly populated** from MatchColumnsStep when columns are marked as AI context only
|
||||
|
||||
**Files to Modify:**
|
||||
- `inventory/src/components/product-import/steps/ValidationStep/utils/inlineAiPayload.ts`
|
||||
- Backend sanity check endpoint (if separate from description validation)
|
||||
- Verify data flow in `MatchColumnsStep.tsx` → `ValidationStep`
|
||||
|
||||
---
|
||||
|
||||
## 3. Fresh Taxonomy Data Per Session ✅ IMPLEMENTED
|
||||
|
||||
**Issue:** Ensure taxonomy data is brought in fresh with each session - cache should be invalidated if we exit the import flow and start again.
|
||||
|
||||
**Current Implementation:**
|
||||
- Field options cached 5 minutes: [ValidationStep/index.tsx:128-133](inventory/src/components/product-import/steps/ValidationStep/index.tsx#L128-L133)
|
||||
- Product lines cache: `productLinesCache` in Zustand store
|
||||
- Sublines cache: `sublinesCache` in Zustand store
|
||||
- Caches set to 10-minute stale time
|
||||
|
||||
**Implementation:**
|
||||
1. **Add cache invalidation on import flow mount/unmount** in `UploadFlow.tsx`:
|
||||
```typescript
|
||||
useEffect(() => {
|
||||
// On mount - invalidate import-related query cache
|
||||
queryClient.invalidateQueries({ queryKey: ['import-field-options'] });
|
||||
|
||||
return () => {
|
||||
// On unmount - clear caches
|
||||
queryClient.removeQueries({ queryKey: ['import-field-options'] });
|
||||
queryClient.removeQueries({ queryKey: ['product-lines'] });
|
||||
queryClient.removeQueries({ queryKey: ['sublines'] });
|
||||
};
|
||||
}, []);
|
||||
```
|
||||
|
||||
2. **Clear Zustand store caches** when exiting import flow:
|
||||
- Add action to `validationStore.ts` to clear `productLinesCache` and `sublinesCache`
|
||||
- Call this action on unmount of `UploadFlow` or when navigating away
|
||||
|
||||
3. **Consider adding a `sessionId`** that changes on each import flow start, used as part of cache keys
|
||||
|
||||
**Files to Modify:**
|
||||
- `inventory/src/components/product-import/steps/UploadFlow.tsx` - Add cleanup effect
|
||||
- `inventory/src/components/product-import/steps/ValidationStep/store/validationStore.ts` - Add cache clear action
|
||||
- Potentially `inventory/src/components/product-import/steps/ValidationStep/index.tsx` - Query key updates
|
||||
|
||||
---
|
||||
|
||||
## 4. Save Template from Confirmation Page ✅ IMPLEMENTED
|
||||
|
||||
**Issue:** Add option to save rows of submitted data as a new template on the confirmation page after completing the import flow. Verify this works with new validation step changes.
|
||||
|
||||
**Current Implementation:**
|
||||
- **Import Results section already exists** inline in [Import.tsx:968-1150](inventory/src/pages/Import.tsx#L968-L1150)
|
||||
- Shows created products (lines 1021-1097) with image, name, UPC, item number
|
||||
- Shows errored products (lines 1100-1138) with error details
|
||||
- "Fix products with errors" button resumes validation flow for failed items
|
||||
- Template saving logic in ValidationStep: [useTemplateManagement.ts:204-266](inventory/src/components/product-import/steps/ValidationStep/hooks/useTemplateManagement.ts#L204-L266)
|
||||
- Saves via `POST /api/templates`
|
||||
- `importOutcome.submittedProducts` contains the full product data for each row
|
||||
|
||||
**Implementation:**
|
||||
1. **Add "Save as Template" button** to each created product row in the results section (around line 1087-1092 in Import.tsx):
|
||||
```typescript
|
||||
// Add button after the item number display
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => handleSaveAsTemplate(index)}
|
||||
>
|
||||
<BookmarkPlus className="h-4 w-4" />
|
||||
</Button>
|
||||
```
|
||||
|
||||
2. **Add state and dialog** for template saving in Import.tsx:
|
||||
```typescript
|
||||
const [templateSaveDialogOpen, setTemplateSaveDialogOpen] = useState(false);
|
||||
const [selectedProductForTemplate, setSelectedProductForTemplate] = useState<NormalizedProduct | null>(null);
|
||||
```
|
||||
|
||||
3. **Extract/reuse template save logic** from `useTemplateManagement.ts`:
|
||||
- The `saveNewTemplate()` function (lines 204-266) can be extracted into a shared utility
|
||||
- Or create a `SaveTemplateDialog` component that can be used in both places
|
||||
- Key fields needed: `company` (for template name), `product_type`, and all product field values
|
||||
|
||||
4. **Data mapping consideration:**
|
||||
- `importOutcome.submittedProducts` uses `NormalizedProduct` type
|
||||
- Templates expect raw field values - may need to map back from normalized format
|
||||
- Exclude metadata fields: `['id', '__index', '__meta', '__template', '__original', '__corrected', '__changes', '__aiSupplemental']`
|
||||
|
||||
**Files to Modify:**
|
||||
- `inventory/src/pages/Import.tsx` - Add save template button, state, and dialog
|
||||
- Consider creating `inventory/src/components/product-import/SaveTemplateDialog.tsx` for reusability
|
||||
- Potentially extract core save logic from `useTemplateManagement.ts` into shared utility
|
||||
|
||||
---
|
||||
|
||||
## 5. Sheet Preview on Select Sheet Step ✅ IMPLEMENTED
|
||||
|
||||
**Issue:** On the select sheet step, show a preview of the first 10 lines or so of each sheet underneath the options.
|
||||
|
||||
**Implementation (Completed):**
|
||||
- Added `workbook` prop to `SelectSheetStep` component
|
||||
- Added `sheetPreviews` memoized computation using `XLSXLib.utils.sheet_to_json()`
|
||||
- Shows first 10 rows, 8 columns max per sheet
|
||||
- Added `truncateCell()` helper to limit cell content to 30 characters with ellipsis
|
||||
- Each sheet option is now a clickable card with:
|
||||
- Radio button and sheet name
|
||||
- Row count indicator
|
||||
- Scrollable preview table with horizontal scroll
|
||||
- Selected state highlighted with primary border
|
||||
- Updated `UploadFlow.tsx` to pass workbook prop
|
||||
|
||||
**Files Modified:**
|
||||
- `inventory/src/components/product-import/steps/SelectSheetStep/SelectSheetStep.tsx`
|
||||
- `inventory/src/components/product-import/steps/UploadFlow.tsx`
|
||||
|
||||
---
|
||||
|
||||
## 6. Empty Row Removal ✅ IMPLEMENTED
|
||||
|
||||
**Issue:** When importing a sheet, automatically remove completely empty rows.
|
||||
|
||||
**Current Implementation:**
|
||||
- Empty columns are filtered: [MatchColumnsStep.tsx:616-634](inventory/src/components/product-import/steps/MatchColumnsStep/MatchColumnsStep.tsx#L616-L634)
|
||||
- A "Remove empty/duplicates" button exists that removes empty rows, single-value rows, AND duplicates
|
||||
- The automatic removal should ONLY remove completely empty rows, not duplicates or single-value rows
|
||||
|
||||
**Implementation (Completed):**
|
||||
- Added `isRowCompletelyEmpty()` helper function to [SelectHeaderStep.tsx](inventory/src/components/product-import/steps/SelectHeaderStep/SelectHeaderStep.tsx)
|
||||
- Added `useMemo` to filter empty rows on initial data load
|
||||
- Uses `Object.values(row)` to check all cell values (matches existing button logic)
|
||||
- Only removes rows where ALL values are undefined, null, or whitespace-only strings
|
||||
- Manual "Remove Empty/Duplicates" button still available for additional cleanup (duplicates, single-value rows)
|
||||
|
||||
**Files Modified:**
|
||||
- `inventory/src/components/product-import/steps/SelectHeaderStep/SelectHeaderStep.tsx`
|
||||
|
||||
---
|
||||
|
||||
## 7. Unit Conversion for Weight/Dimensions ✅ IMPLEMENTED
|
||||
|
||||
**Issue:** Add unit conversion feature for weight and dimensions columns - similar to calculator button on cost/msrp, add button that opens popover with options to convert grams → oz, lbs → oz for the whole column at once.
|
||||
|
||||
**Current Implementation:**
|
||||
- Calculator button on price columns: [ValidationTable.tsx:1491-1627](inventory/src/components/product-import/steps/ValidationStep/components/ValidationTable.tsx#L1491-L1627)
|
||||
- `PriceColumnHeader` component shows calculator icon on hover
|
||||
- Weight field defined in config with validation
|
||||
|
||||
**Implementation:**
|
||||
1. **Create `UnitConversionColumnHeader` component** (similar to `PriceColumnHeader`):
|
||||
```typescript
|
||||
const UnitConversionColumnHeader = ({ field, table }) => {
|
||||
const [showPopover, setShowPopover] = useState(false);
|
||||
|
||||
const conversions = {
|
||||
weight: [
|
||||
{ label: 'Grams → Ounces', factor: 0.035274 },
|
||||
{ label: 'Pounds → Ounces', factor: 16 },
|
||||
{ label: 'Kilograms → Ounces', factor: 35.274 },
|
||||
],
|
||||
dimensions: [
|
||||
{ label: 'Centimeters → Inches', factor: 0.393701 },
|
||||
{ label: 'Millimeters → Inches', factor: 0.0393701 },
|
||||
]
|
||||
};
|
||||
|
||||
const applyConversion = (factor: number) => {
|
||||
// Batch update all cells in column
|
||||
table.rows.forEach((row, index) => {
|
||||
const currentValue = parseFloat(row[field.key]);
|
||||
if (!isNaN(currentValue)) {
|
||||
updateCell(index, field.key, (currentValue * factor).toFixed(2));
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
return (
|
||||
<Popover open={showPopover} onOpenChange={setShowPopover}>
|
||||
<PopoverTrigger>
|
||||
<Scale className="h-4 w-4" /> {/* or similar icon */}
|
||||
</PopoverTrigger>
|
||||
<PopoverContent>
|
||||
{conversions[fieldType].map(conv => (
|
||||
<Button key={conv.label} onClick={() => applyConversion(conv.factor)}>
|
||||
{conv.label}
|
||||
</Button>
|
||||
))}
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
);
|
||||
};
|
||||
```
|
||||
|
||||
2. **Identify weight/dimension fields** in config:
|
||||
- `weight_oz`, `length_in`, `width_in`, `height_in` (check actual field keys)
|
||||
|
||||
3. **Add to column header render logic** in ValidationTable
|
||||
|
||||
**Files to Modify:**
|
||||
- `inventory/src/components/product-import/steps/ValidationStep/components/ValidationTable.tsx`
|
||||
- Potentially create new component file for `UnitConversionColumnHeader`
|
||||
- Update column header rendering to use new component for weight/dimension fields
|
||||
|
||||
---
|
||||
|
||||
## 8. Expanded MSRP Auto-Fill from Cost ✅ IMPLEMENTED
|
||||
|
||||
**Issue:** Expand auto-fill functionality for MSRP from cost - open small popover with options for 2x, 2.1x, 2.2x, 2.3x, 2.4x, 2.5x multipliers, plus checkbox to round up to nearest 9.
|
||||
|
||||
**Current Implementation:**
|
||||
- Calculator on MSRP column: [ValidationTable.tsx:1540-1584](inventory/src/components/product-import/steps/ValidationStep/components/ValidationTable.tsx#L1540-L1584)
|
||||
- Currently only does `Cost × 2` then subtracts 0.01 if whole number
|
||||
|
||||
**Implementation:**
|
||||
1. **Replace simple click with popover** in `PriceColumnHeader`:
|
||||
```typescript
|
||||
const [selectedMultiplier, setSelectedMultiplier] = useState(2.0);
|
||||
const [roundToNine, setRoundToNine] = useState(false);
|
||||
const multipliers = [2.0, 2.1, 2.2, 2.3, 2.4, 2.5];
|
||||
|
||||
const roundUpToNine = (value: number): number => {
|
||||
// 1.41 → 1.49, 2.78 → 2.79, 12.32 → 12.39
|
||||
const wholePart = Math.floor(value);
|
||||
const decimal = value - wholePart;
|
||||
if (decimal <= 0.09) return wholePart + 0.09;
|
||||
if (decimal <= 0.19) return wholePart + 0.19;
|
||||
// ... continue pattern, or:
|
||||
const lastDigit = Math.floor(decimal * 10);
|
||||
return wholePart + (lastDigit / 10) + 0.09;
|
||||
};
|
||||
|
||||
const calculateMsrp = (cost: number): number => {
|
||||
let result = cost * selectedMultiplier;
|
||||
if (roundToNine) {
|
||||
result = roundUpToNine(result);
|
||||
}
|
||||
return result;
|
||||
};
|
||||
```
|
||||
|
||||
2. **Create popover UI**:
|
||||
```tsx
|
||||
<Popover>
|
||||
<PopoverTrigger><Calculator className="h-4 w-4" /></PopoverTrigger>
|
||||
<PopoverContent className="w-48">
|
||||
<div className="space-y-2">
|
||||
<Label>Multiplier</Label>
|
||||
<div className="grid grid-cols-3 gap-1">
|
||||
{multipliers.map(m => (
|
||||
<Button
|
||||
key={m}
|
||||
variant={selectedMultiplier === m ? 'default' : 'outline'}
|
||||
size="sm"
|
||||
onClick={() => setSelectedMultiplier(m)}
|
||||
>
|
||||
{m}x
|
||||
</Button>
|
||||
))}
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Checkbox checked={roundToNine} onCheckedChange={setRoundToNine} />
|
||||
<Label>Round to .X9</Label>
|
||||
</div>
|
||||
<Button onClick={applyCalculation} className="w-full">
|
||||
Apply
|
||||
</Button>
|
||||
</div>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
```
|
||||
|
||||
**Files to Modify:**
|
||||
- `inventory/src/components/product-import/steps/ValidationStep/components/ValidationTable.tsx` - `PriceColumnHeader` component
|
||||
|
||||
---
|
||||
|
||||
## 9. Debug Mode - Skip API Submission ✅ IMPLEMENTED
|
||||
|
||||
**Issue:** Add a third switch in the footer of image upload step (visible only to users with `admin:debug` permission) that will not submit data to any API, only complete the process and show results page as if it had worked.
|
||||
|
||||
**Implementation (Completed):**
|
||||
- Added `skipApiSubmission` state to `ImageUploadStep.tsx`
|
||||
- Added amber-colored "Skip API (Debug)" switch (visible only with `admin:debug` permission)
|
||||
- When skip is active, "Use Test API" and "Use Test Database" switches are hidden
|
||||
- Added `skipApiSubmission?: boolean` to `SubmitOptions` type in `types.ts`
|
||||
- In `Import.tsx`, when `skipApiSubmission` is true:
|
||||
- Skips the actual API call entirely
|
||||
- Generates mock success response with mock PIDs
|
||||
- Shows `[DEBUG]` prefix in toast and result message
|
||||
- Displays results page as if submission succeeded
|
||||
|
||||
**Files Modified:**
|
||||
- `inventory/src/components/product-import/types.ts` - Added `skipApiSubmission` to `SubmitOptions`
|
||||
- `inventory/src/components/product-import/steps/ImageUploadStep/ImageUploadStep.tsx` - Added switch UI
|
||||
- `inventory/src/pages/Import.tsx` - Added skip logic in `handleData()`
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
| # | Enhancement | Complexity | Status |
|
||||
|---|-------------|------------|--------|
|
||||
| 1 | Strip UPC quotes/spaces | Low | ✅ Implemented |
|
||||
| 2 | AI context in validation | Medium | ✅ Implemented |
|
||||
| 3 | Fresh taxonomy per session | Medium | ✅ Implemented |
|
||||
| 4 | Save template from confirmation | Medium-High | ✅ Implemented |
|
||||
| 5 | Sheet preview | Low-Medium | ✅ Implemented |
|
||||
| 6 | Remove empty rows | Low | ✅ Implemented |
|
||||
| 7 | Unit conversion | Medium | ✅ Implemented |
|
||||
| 8 | MSRP multiplier options | Medium | ✅ Implemented |
|
||||
| 9 | Debug skip API | Low-Medium | ✅ Implemented |
|
||||
|
||||
**Implemented:** 9 of 9 items - All enhancements complete!
|
||||
|
||||
---
|
||||
|
||||
*Document generated: 2026-01-25*
|
||||
@@ -0,0 +1,346 @@
|
||||
# Metrics Calculation Pipeline Audit
|
||||
|
||||
**Date:** 2026-02-07
|
||||
**Scope:** All 6 SQL calculation scripts, custom DB functions, import pipeline, and live data verification
|
||||
|
||||
## Overview
|
||||
|
||||
The metrics pipeline in `inventory-server/scripts/calculate-metrics-new.js` runs 6 SQL scripts sequentially:
|
||||
|
||||
1. `update_daily_snapshots.sql` — Aggregates daily per-product sales/receiving data
|
||||
2. `update_product_metrics.sql` — Calculates the main product_metrics table (KPIs, forecasting, status)
|
||||
3. `update_periodic_metrics.sql` — ABC classification, average lead time
|
||||
4. `calculate_brand_metrics.sql` — Brand-level aggregated metrics
|
||||
5. `calculate_vendor_metrics.sql` — Vendor-level aggregated metrics
|
||||
6. `calculate_category_metrics.sql` — Category-level metrics with hierarchy rollups
|
||||
|
||||
### Database Scale
|
||||
| Table | Row Count |
|
||||
|---|---|
|
||||
| products | 681,912 |
|
||||
| orders | 2,883,982 |
|
||||
| purchase_orders | 256,809 |
|
||||
| receivings | 313,036 |
|
||||
| daily_product_snapshots | 678,312 (601 distinct dates, since 2024-06-01) |
|
||||
| product_metrics | 681,912 |
|
||||
| brand_metrics | 1,789 |
|
||||
| vendor_metrics | 281 |
|
||||
| category_metrics | 610 |
|
||||
|
||||
---
|
||||
|
||||
## Issues Found
|
||||
|
||||
### ISSUE 1: [HIGH] Order status filter is non-functional — numeric codes vs text comparison
|
||||
|
||||
**Files:** `update_daily_snapshots.sql` lines 86-101, `update_product_metrics.sql` lines 89, 178-183
|
||||
**Confirmed by data:** All order statuses are numeric strings ('100', '50', '55', etc.)
|
||||
**Status mappings from:** `docs/prod_registry.class.php`
|
||||
|
||||
**Description:** The SQL filters `COALESCE(o.status, 'pending') NOT IN ('canceled', 'returned')` and `o.status NOT IN ('canceled', 'returned')` are used throughout the pipeline to exclude canceled/returned orders. However, the import pipeline stores order statuses as their **raw numeric codes** from the production MySQL database (e.g., '100', '50', '55', '90', '92'). There are **zero text status values** in the orders table.
|
||||
|
||||
This means these filters **never exclude any rows** — every comparison is `'100' NOT IN ('canceled', 'returned')` which is always true.
|
||||
|
||||
**Actual status distribution (with confirmed meanings):**
|
||||
| Status | Meaning | Count | Negative Qty | Assessment |
|
||||
|---|---|---|---|---|
|
||||
| 100 | shipped | 2,862,792 | 3,352 | Completed — correct to include |
|
||||
| 50 | awaiting_products | 11,109 | 0 | In-progress — not yet shipped |
|
||||
| 55 | shipping_later | 5,689 | 0 | In-progress — not yet shipped |
|
||||
| 56 | shipping_together | 2,863 | 0 | In-progress — not yet shipped |
|
||||
| 90 | awaiting_shipment | 38 | 0 | Near-complete — not yet shipped |
|
||||
| 92 | awaiting_pickup | 71 | 0 | Near-complete — awaiting customer |
|
||||
| 95 | shipped_confirmed | 5 | 0 | Completed — correct to include |
|
||||
| 15 | cancelled | 1 | 0 | Should be excluded |
|
||||
|
||||
**Full status reference (from prod_registry.class.php):**
|
||||
- 0=created, 10=unfinished, **15=cancelled**, 16=combined, 20=placed, 22=placed_incomplete
|
||||
- 30=cancelled_old (historical), 40=awaiting_payment, 50=awaiting_products
|
||||
- 55=shipping_later, 56=shipping_together, 60=ready, 61=flagged
|
||||
- 62=fix_before_pick, 65=manual_picking, 70=in_pt, 80=picked
|
||||
- 90=awaiting_shipment, 91=remote_wait, **92=awaiting_pickup**, 93=fix_before_ship
|
||||
- **95=shipped_confirmed**, **100=shipped**
|
||||
|
||||
**Severity revised to HIGH (from CRITICAL):** Now that we know the actual meanings, no cancelled/refunded orders are being miscounted (only 1 cancelled order exists, status=15). The real concern is twofold:
|
||||
1. **The text-based filter is dead code** — it can never match any row. Either map statuses to text during import (like POs do) or change SQL to use numeric comparisons.
|
||||
2. **~19,775 unfulfilled orders** (statuses 50/55/56/90/92) are counted as completed sales. These are orders in various stages of fulfillment that haven't shipped yet. While most will eventually ship, counting them now inflates current-period metrics. At 0.69% of total orders, the financial impact is modest but the filter should work correctly on principle.
|
||||
|
||||
**Note:** PO statuses ARE properly mapped to text ('canceled', 'done', etc.) in the import pipeline. Only order statuses are numeric.
|
||||
|
||||
---
|
||||
|
||||
### ISSUE 2: [CRITICAL] Daily Snapshots use current stock instead of historical EOD stock
|
||||
|
||||
**File:** `update_daily_snapshots.sql`, lines 126-135, 173
|
||||
**Confirmed by data:** Top product (pid 666925) shows `eod_stock_quantity = 0` for ALL dates even though it sold 28 units on Jan 28 (clearly had stock then)
|
||||
|
||||
**Description:** The `CurrentStock` CTE reads `stock_quantity` directly from the `products` table at query execution time. When the script processes historical dates (today minus 1-4 days), it writes **today's stock** as if it were the end-of-day stock for those past dates.
|
||||
|
||||
**Cascading impact on product_metrics:**
|
||||
- `avg_stock_units_30d` / `avg_stock_cost_30d` — Wrong averages
|
||||
- `stockout_days_30d` — Undercounts (only based on current stock state, not historical)
|
||||
- `stockout_rate_30d`, `service_level_30d`, `fill_rate_30d` — All derived from wrong stockout data
|
||||
- `gmroi_30d` — Wrong denominator (avg stock cost)
|
||||
- `stockturn_30d` — Wrong denominator (avg stock units)
|
||||
- `sell_through_30d` — Affected by stock level inaccuracy
|
||||
|
||||
---
|
||||
|
||||
### ISSUE 3: [CRITICAL] Snapshot coverage is 0.17% — most products have no snapshot data
|
||||
|
||||
**Confirmed by data:** 678,312 snapshot rows across 601 dates = ~1,128 products/day out of 681,912 total
|
||||
|
||||
**Description:** The daily snapshots script only creates rows for products with sales or receiving activity on that date (`ProductsWithActivity` CTE, line 136). This means:
|
||||
- 91.1% of products (621,221) have NULL `sales_30d` — they had no orders in the last 30 days so no snapshot rows exist
|
||||
- `AVG(eod_stock_quantity)` averages only across days with activity, not 30 days
|
||||
- `stockout_days_30d` only counts stockout days where there was ALSO some activity
|
||||
- A product out of stock with zero sales gets zero stockout_days even though it was stocked out
|
||||
|
||||
This is by design (to avoid creating 681K rows/day) but means stock-related metrics are systematically biased.
|
||||
|
||||
---
|
||||
|
||||
### ISSUE 4: [HIGH] `costeach` fallback to 50% of price in import pipeline
|
||||
|
||||
**File:** `inventory-server/scripts/import/orders.js` (line ~573)
|
||||
|
||||
**Description:** When the MySQL `order_costs` table has no record for an order item, `costeach` defaults to `price * 0.5`. There is **no flag** in the PostgreSQL data to distinguish actual costs from estimated ones.
|
||||
|
||||
**Data impact:** 385,545 products (56.5%) have `current_cost_price = 0` AND `current_landing_cost_price = 0`. For these products, the COGS calculation in daily_snapshots falls through the chain:
|
||||
1. `o.costeach` — May be the 50% estimate from import
|
||||
2. `get_weighted_avg_cost()` — Returns NULL if no receivings exist
|
||||
3. `p.landing_cost_price` — Always NULL (hardcoded in import)
|
||||
4. `p.cost_price` — 0 for 56.5% of products
|
||||
|
||||
Only 27 products have zero COGS with positive sales, meaning the `costeach` field is doing its job for products that sell, but the 50% fallback means margins for those products are estimates, not actuals.
|
||||
|
||||
---
|
||||
|
||||
### ISSUE 5: [HIGH] `landing_cost_price` is always NULL
|
||||
|
||||
**File:** `inventory-server/scripts/import/products.js` (line ~175)
|
||||
|
||||
**Description:** The import explicitly sets `landing_cost_price = NULL` for all products. The daily_snapshots COGS calculation uses it as a fallback: `COALESCE(o.costeach, get_weighted_avg_cost(...), p.landing_cost_price, p.cost_price)`. Since it's always NULL, this fallback step is useless and the chain jumps straight to `cost_price`.
|
||||
|
||||
The `product_metrics` field `current_landing_cost_price` is populated as `COALESCE(p.landing_cost_price, p.cost_price, 0.00)`, so it equals `cost_price` for all products. Any UI showing "landing cost" is actually just showing `cost_price`.
|
||||
|
||||
---
|
||||
|
||||
### ISSUE 6: [HIGH] Vendor lead time is drastically wrong — missing supplier_id join
|
||||
|
||||
**File:** `calculate_vendor_metrics.sql`, lines 62-82
|
||||
**Confirmed by data:** Vendor-level lead times are 2-10x higher than product-level lead times
|
||||
|
||||
**Description:** The vendor metrics lead time joins POs to receivings only by `pid`:
|
||||
```sql
|
||||
LEFT JOIN public.receivings r ON r.pid = po.pid
|
||||
```
|
||||
But the periodic metrics lead time correctly matches supplier:
|
||||
```sql
|
||||
JOIN public.receivings r ON r.pid = po.pid AND r.supplier_id = po.supplier_id
|
||||
```
|
||||
|
||||
Without supplier matching, a PO for product X from Vendor A can match a receiving of product X from Vendor B, creating inflated/wrong lead times.
|
||||
|
||||
**Measured discrepancies:**
|
||||
| Vendor | Vendor Metrics Lead Time | Avg Product Lead Time |
|
||||
|---|---|---|
|
||||
| doodlebug design inc. | 66 days | 14 days |
|
||||
| Notions | 55 days | 4 days |
|
||||
| Simple Stories | 59 days | 27 days |
|
||||
| Ranger Industries | 31 days | 5 days |
|
||||
|
||||
---
|
||||
|
||||
### ISSUE 7: [MEDIUM] Net revenue does not subtract returns
|
||||
|
||||
**File:** `update_daily_snapshots.sql`, line 184
|
||||
|
||||
**Description:** `net_revenue = gross_revenue - discounts`. Standard accounting: `net_revenue = gross_revenue - discounts - returns`. The `returns_revenue` is calculated separately but not deducted.
|
||||
|
||||
**Data impact:** There are 3,352 orders with negative quantities (returns), totaling -5,499 units. These returns are tracked in `returns_revenue` but not reflected in `net_revenue`, which means all downstream revenue-based metrics are slightly overstated.
|
||||
|
||||
---
|
||||
|
||||
### ISSUE 8: [MEDIUM] Lifetime revenue subquery references wrong table columns
|
||||
|
||||
**File:** `update_product_metrics.sql`, lines 323-329
|
||||
|
||||
**Description:** The lifetime revenue estimation fallback queries:
|
||||
```sql
|
||||
SELECT revenue_7d / NULLIF(sales_7d, 0)
|
||||
FROM daily_product_snapshots
|
||||
WHERE pid = ci.pid AND sales_7d > 0
|
||||
```
|
||||
But `daily_product_snapshots` does NOT have `revenue_7d` or `sales_7d` columns — those exist in `product_metrics`. This subquery either errors silently or returns NULL. The effect is that the estimation always falls back to `current_price * total_sold`.
|
||||
|
||||
---
|
||||
|
||||
### ISSUE 9: [MEDIUM] Brand/Vendor metrics COGS filter inflates margins
|
||||
|
||||
**Files:** `calculate_brand_metrics.sql` lines 31, `calculate_vendor_metrics.sql` line 32
|
||||
|
||||
**Description:** `SUM(CASE WHEN pm.cogs_30d > 0 THEN pm.cogs_30d ELSE 0 END)` excludes products with zero COGS. But if a product has sales revenue and zero COGS (missing cost data), the brand/vendor totals will include the revenue but not the COGS, artificially inflating the margin.
|
||||
|
||||
**Data context:** Brand metrics revenue matches product_metrics aggregation exactly for sales counts, but shows small discrepancies in revenue (e.g., Stamperia: $7,613.98 brand vs $7,611.11 actual). These tiny diffs come from the `> 0` filtering excluding products with negative revenue.
|
||||
|
||||
---
|
||||
|
||||
### ISSUE 10: [MEDIUM] Extreme margin values from $0.01 price orders
|
||||
|
||||
**Confirmed by data:** 73 products with margin > 100%, 119 with margin < -100%
|
||||
|
||||
**Examples:**
|
||||
| Product | Revenue | COGS | Margin |
|
||||
|---|---|---|---|
|
||||
| Flower Gift Box Die (pid 624756) | $0.02 | $29.98 | -149,800% |
|
||||
| Special Flowers Stamp Set (pid 614513) | $0.01 | $11.97 | -119,632% |
|
||||
|
||||
These are products with extremely low prices (likely samples, promos, or data errors) where the order price was $0.01. The margin calculation is mathematically correct but these outliers skew any aggregate margin statistics.
|
||||
|
||||
---
|
||||
|
||||
### ISSUE 11: [MEDIUM] Sell-through rate has edge cases yielding negative/extreme values
|
||||
|
||||
**File:** `update_product_metrics.sql`, lines 358-361
|
||||
**Confirmed by data:** 30 products with negative sell-through, 10 with sell-through > 200%
|
||||
|
||||
**Description:** Beginning inventory is approximated as `current_stock + sales - received + returns`. When inventory adjustments, shrinkage, or manual corrections occur, this approximation breaks. Edge cases:
|
||||
- Products with many manual stock adjustments → negative denominator → negative sell-through
|
||||
- Products with beginning stock near zero but decent sales → sell-through > 100%
|
||||
|
||||
---
|
||||
|
||||
### ISSUE 12: [MEDIUM] `total_sold` uses different status filter than orders import
|
||||
|
||||
**Import pipeline confirmed:**
|
||||
- Orders import: `order_status >= 15` (includes processing/pending orders)
|
||||
- `total_sold` in products: `order_status >= 20` (more restrictive)
|
||||
|
||||
This means `lifetime_sales` (from `total_sold`) is systematically lower than what you'd calculate by summing the orders table. The discrepancy is confirmed:
|
||||
| Product | total_sold | orders sum | Gap |
|
||||
|---|---|---|---|
|
||||
| pid 31286 | 13,786 | 4,241 | 9,545 |
|
||||
| pid 44309 | 11,978 | 3,119 | 8,859 |
|
||||
|
||||
The large gaps are because the orders table only has data from the import start date (~2024), while `total_sold` includes all-time sales from MySQL. This is expected behavior, not a bug, but it means the `lifetime_revenue_quality` flag is important — most products show 'estimated' quality.
|
||||
|
||||
---
|
||||
|
||||
### ISSUE 13: [MEDIUM] Category rollup may double-count products in multiple hierarchy levels
|
||||
|
||||
**File:** `calculate_category_metrics.sql`, lines 42-66
|
||||
|
||||
**Description:** The `RolledUpMetrics` CTE uses:
|
||||
```sql
|
||||
dcm.cat_id = ch.cat_id OR dcm.cat_id = ANY(SELECT cat_id FROM category_hierarchy WHERE ch.cat_id = ANY(ancestor_ids))
|
||||
```
|
||||
If products are assigned to categories at multiple levels in the same branch (e.g., both "Paper Crafts" and "Scrapbook Paper" which is a child of "Paper Crafts"), those products' metrics would be counted twice in the parent's rollup.
|
||||
|
||||
---
|
||||
|
||||
### ISSUE 14: [LOW] `exclude_forecast` removes products from metrics entirely
|
||||
|
||||
**File:** `update_product_metrics.sql`, line 509
|
||||
|
||||
**Description:** `WHERE s.exclude_forecast IS FALSE OR s.exclude_forecast IS NULL` is on the main INSERT's WHERE clause. Products with `exclude_forecast = TRUE` won't appear in `product_metrics` at all, rather than just having forecast fields nulled. Currently all 681,912 products are in product_metrics so this appears to not affect any products yet.
|
||||
|
||||
---
|
||||
|
||||
### ISSUE 15: [LOW] Daily snapshots only look back 5 days
|
||||
|
||||
**File:** `update_daily_snapshots.sql`, line 14 — `_process_days INT := 5`
|
||||
|
||||
If import data arrives late (>5 days), those days will never get snapshots populated. There is a separate `backfill/rebuild_daily_snapshots.sql` for historical rebuilds.
|
||||
|
||||
---
|
||||
|
||||
### ISSUE 16: [INFO] Timezone risk in order date import
|
||||
|
||||
**File:** `inventory-server/scripts/import/orders.js`
|
||||
|
||||
MySQL `DATETIME` values are timezone-naive. The import uses `new Date(order.date)` which interprets them using the import server's local timezone. The SSH config specifies `timezone: '-05:00'` for MySQL (always EST). If the import server is in a different timezone, orders near midnight could land on the wrong date in the daily snapshots calculation.
|
||||
|
||||
---
|
||||
|
||||
## Custom Functions Review
|
||||
|
||||
### `calculate_sales_velocity(sales_30d, stockout_days_30d)`
|
||||
- Divides `sales_30d` by effective selling days: `GREATEST(30 - stockout_days, CASE WHEN sales > 0 THEN 14 ELSE 30 END)`
|
||||
- The 14-day floor prevents extreme velocity for products mostly out of stock
|
||||
- **Sound approach** — the only concern is that stockout_days is unreliable (Issues 2, 3)
|
||||
|
||||
### `get_weighted_avg_cost(pid, date)`
|
||||
- Weighted average of last 10 receivings by cost*qty/qty
|
||||
- Returns NULL if no receivings — sound fallback behavior
|
||||
- **Correct implementation**
|
||||
|
||||
### `safe_divide(numerator, denominator)`
|
||||
- Returns NULL on divide-by-zero — **correct**
|
||||
|
||||
### `std_numeric(value, precision)`
|
||||
- Rounds to precision digits — **correct**
|
||||
|
||||
### `classify_demand_pattern(avg_demand, cv)`
|
||||
- Uses coefficient of variation thresholds: ≤0.2 = stable, ≤0.5 = variable, low-volume+high-CV = sporadic, else lumpy
|
||||
- **Reasonable classification**, though only based on 30-day window
|
||||
|
||||
### `detect_seasonal_pattern(pid)`
|
||||
- CROSS JOIN LATERAL (runs per product) — **expensive**: queries `daily_product_snapshots` twice per product
|
||||
- Compares current month average to yearly average — very simplistic
|
||||
- **Functional but could be a performance bottleneck** with 681K products
|
||||
|
||||
### `category_hierarchy` (materialized view)
|
||||
- Recursive CTE building tree from categories — **correct implementation**
|
||||
- Refreshed concurrently before category metrics calculation — **good practice**
|
||||
|
||||
---
|
||||
|
||||
## Data Health Summary
|
||||
|
||||
| Metric | Count | % of Total |
|
||||
|---|---|---|
|
||||
| Products with zero cost_price | 385,545 | 56.5% |
|
||||
| Products with NULL sales_30d | 621,221 | 91.1% |
|
||||
| Products with no lifetime_sales | 321,321 | 47.1% |
|
||||
| Products with zero COGS but positive sales | 27 | <0.01% |
|
||||
| Products with margin > 100% | 73 | <0.01% |
|
||||
| Products with margin < -100% | 119 | <0.01% |
|
||||
| Products with negative sell-through | 30 | <0.01% |
|
||||
| Products with NULL status | 0 | 0% |
|
||||
| Duplicate daily snapshots (same pid+date) | 0 | 0% |
|
||||
| Net revenue formula mismatches | 0 | 0% |
|
||||
|
||||
### ABC Classification Distribution (replenishable products only)
|
||||
| Class | Products | Revenue % |
|
||||
|---|---|---|
|
||||
| A | 7,727 | 80.72% |
|
||||
| B | 12,048 | 15.10% |
|
||||
| C | 113,647 | 4.18% |
|
||||
|
||||
ABC distribution looks healthy — A ≈ 80%, A+B ≈ 96%.
|
||||
|
||||
### Brand Metrics Consistency
|
||||
Product counts and sales_30d match exactly between `brand_metrics` and direct aggregation from `product_metrics`. Revenue shows sub-dollar discrepancies due to the `> 0` filter excluding products with negative revenue. **Consistent within expected tolerance.**
|
||||
|
||||
---
|
||||
|
||||
## Priority Recommendations
|
||||
|
||||
### Must Fix (Correctness Issues)
|
||||
1. **Issue 1: Fix order status handling** — The text-based filter (`NOT IN ('canceled', 'returned')`) is dead code against numeric statuses. Two options: (a) map numeric statuses to text during import (like POs already do), or (b) change SQL to filter on numeric codes (e.g., `o.status::int >= 20` to exclude cancelled/unfinished, or `o.status IN ('100', '95')` for shipped-only). The ~19.7K unfulfilled orders (0.69%) are a minor financial impact but the filter should be functional.
|
||||
2. **Issue 6: Add supplier_id join to vendor lead time** — One-line fix in `calculate_vendor_metrics.sql`
|
||||
3. **Issue 8: Fix lifetime revenue subquery** — Use correct column names from `daily_product_snapshots` (e.g., `net_revenue / NULLIF(units_sold, 0)`)
|
||||
|
||||
### Should Fix (Data Quality)
|
||||
4. **Issue 2/3: Snapshot coverage** — Consider creating snapshot rows for all in-stock products, not just those with activity. Or at minimum, calculate stockout metrics by comparing snapshot existence to product existence.
|
||||
5. **Issue 5: Populate landing_cost_price** — If available in the source system, import it. Otherwise remove references to avoid confusion.
|
||||
6. **Issue 7: Subtract returns from net_revenue** — `net_revenue = gross_revenue - discounts - returns_revenue`
|
||||
7. **Issue 9: Remove > 0 filter on COGS** — Use `SUM(pm.cogs_30d)` instead of conditional sums
|
||||
|
||||
### Nice to Fix (Edge Cases)
|
||||
8. **Issue 4: Flag estimated costs** — Add a `costeach_estimated BOOLEAN` to orders during import
|
||||
9. **Issue 10: Cap or flag extreme margins** — Exclude $0.01-price orders from margin calculations
|
||||
10. **Issue 11: Clamp sell-through** — `GREATEST(0, LEAST(sell_through_30d, 200))` or flag outliers
|
||||
11. **Issue 12: Verify category assignment policy** — Check if products are assigned to leaf categories only
|
||||
12. **Issue 13: Category rollup query** — Verify no double-counting with actual data
|
||||
@@ -0,0 +1,276 @@
|
||||
# Metrics Pipeline Audit Report
|
||||
|
||||
**Date:** 2026-02-08
|
||||
**Scope:** All 6 SQL scripts in `inventory-server/scripts/metrics-new/`, import pipeline, custom functions, and post-calculation data verification.
|
||||
|
||||
---
|
||||
|
||||
## Executive Summary
|
||||
|
||||
The metrics pipeline is architecturally sound and the core calculations are mostly correct. The 30-day sales, revenue, replenishment, and aggregate metrics (brand/vendor/category) all cross-check accurately between the snapshots, product_metrics, and direct orders queries. However, several issues were found ranging from **critical data bugs** to **design limitations** that affect accuracy of specific metrics.
|
||||
|
||||
**Issues found: 13** (3 Critical, 4 Medium, 6 Low/Informational)
|
||||
|
||||
---
|
||||
|
||||
## CRITICAL Issues
|
||||
|
||||
### C1. `net_revenue` in daily snapshots never subtracts returns ($35.6K affected)
|
||||
|
||||
**Location:** `update_daily_snapshots.sql`, line 181
|
||||
**Symptom:** `net_revenue` is stored as `gross_revenue - discounts` but should be `gross_revenue - discounts - returns_revenue`.
|
||||
|
||||
The SQL formula on line 181 appears correct:
|
||||
```sql
|
||||
COALESCE(sd.gross_revenue_unadjusted, 0.00) - COALESCE(sd.discounts, 0.00) - COALESCE(sd.returns_revenue, 0.00) AS net_revenue
|
||||
```
|
||||
|
||||
However, actual data shows `net_revenue = gross_revenue - discounts` for ALL 3,252 snapshots that have returns. Total returns not subtracted: **$35,630.03** across 2,946 products. This may be caused by the `returns_revenue` in the SalesData CTE not properly flowing through to the INSERT, or by a prior version of the code that stored these values differently. The profit column (line 184) has the same issue: `(gross - discounts) - cogs` instead of `(gross - discounts - returns) - cogs`.
|
||||
|
||||
**Impact:** Net revenue and profit are overstated by the amount of returns. This cascades to all metrics derived from snapshots: `revenue_30d`, `profit_30d`, `margin_30d`, `avg_ros_30d`, and all brand/vendor/category aggregate revenue.
|
||||
|
||||
**Recommended fix:** Debug why the returns subtraction isn't taking effect. The formula in the SQL looks correct, so this may be a data-type issue or an execution path issue. After fixing, rebuild snapshots.
|
||||
|
||||
**Status:** Owner will resolve. Code formula is correct; snapshots need rebuilding after prior fix deployment.
|
||||
|
||||
---
|
||||
|
||||
### C2. `eod_stock_quantity` uses CURRENT stock, not historical end-of-day stock
|
||||
|
||||
**Location:** `update_daily_snapshots.sql`, lines 123-132 (CurrentStock CTE)
|
||||
**Symptom:** Every snapshot for a given product shows the same stock quantity regardless of the snapshot date.
|
||||
|
||||
The `CurrentStock` CTE simply reads `stock_quantity` from the `products` table:
|
||||
```sql
|
||||
SELECT pid, stock_quantity, ... FROM public.products
|
||||
```
|
||||
|
||||
This means a snapshot from January 10 shows the SAME stock as today (February 8). Verified in data:
|
||||
- Product 662561: stock = 36 on every date (Feb 1-7)
|
||||
- Product 665397: stock = 25 on every date (Feb 1-7)
|
||||
- All products checked show identical stock across all snapshot dates
|
||||
|
||||
**Impact:** All stock-derived metrics are inaccurate for historical analysis:
|
||||
- `eod_stock_cost`, `eod_stock_retail`, `eod_stock_gross` (all wrong for past dates)
|
||||
- `stockout_flag` (based on current stock, not historical)
|
||||
- `stockout_days_30d` (undercounted since stockout_flag uses current stock)
|
||||
- `avg_stock_units_30d`, `avg_stock_cost_30d` (no variance, just current stock repeated)
|
||||
- `gmroi_30d`, `stockturn_30d` (based on avg_stock which is flat)
|
||||
- `sell_through_30d` (denominator uses current stock assumption)
|
||||
- `service_level_30d`, `fill_rate_30d`
|
||||
|
||||
**This is a known architectural limitation** noted in MEMORY.md. Fixing requires either:
|
||||
1. Storing stock snapshots separately at end-of-day (ideally via a cron job that records stock before any changes)
|
||||
2. Reconstructing historical stock from orders and receivings (complex but possible)
|
||||
|
||||
**Status: FIXED.** MySQL's `snap_product_value` table (daily EOD stock per product since 2012) is now imported into PostgreSQL `stock_snapshots` table via `scripts/import/stock-snapshots.js`. The `CurrentStock` CTE in `update_daily_snapshots.sql` now uses `LEFT JOIN stock_snapshots` for historical stock, falling back to `products.stock_quantity` when no historical data exists. Requires: run import, then rebuild daily snapshots.
|
||||
|
||||
---
|
||||
|
||||
### C3. `ON CONFLICT DO UPDATE WHERE` check skips 91%+ of product_metrics updates
|
||||
|
||||
**Location:** `update_product_metrics.sql`, lines 558-574
|
||||
**Symptom:** 623,205 of 681,912 products (91.4%) have `last_calculated` older than 1 day. 592,369 are over 30 days old. 914 products with active 30-day sales haven't been updated in over 7 days.
|
||||
|
||||
The upsert's `WHERE` clause only updates if specific fields changed:
|
||||
```sql
|
||||
WHERE product_metrics.current_stock IS DISTINCT FROM EXCLUDED.current_stock OR
|
||||
product_metrics.current_price IS DISTINCT FROM EXCLUDED.current_price OR ...
|
||||
```
|
||||
|
||||
Fields NOT checked include: `stockout_days_30d`, `margin_30d`, `gmroi_30d`, `demand_pattern`, `seasonality_index`, `sales_growth_*`, `service_level_30d`, and many others. If a product's stock, price, sales, and revenue haven't changed, the entire row is skipped even though growth metrics, variability, and other derived fields may need updating.
|
||||
|
||||
**Impact:** Most derived metrics (growth, demand patterns, seasonality) are stale for the majority of products. Products with steady sales but unchanged stock/price never get their growth metrics recalculated.
|
||||
|
||||
**Recommended fix:** Either:
|
||||
1. Remove the `WHERE` clause entirely (accept the performance cost of writing all rows every run)
|
||||
2. Add `last_calculated` age check: `OR product_metrics.last_calculated < NOW() - INTERVAL '7 days'`
|
||||
3. Add the missing fields to the change-detection check
|
||||
|
||||
**Status: FIXED.** Added 12 derived fields to the `IS DISTINCT FROM` check (`profit_30d`, `cogs_30d`, `margin_30d`, `stockout_days_30d`, `sell_through_30d`, `sales_growth_30d_vs_prev`, `revenue_growth_30d_vs_prev`, `demand_pattern`, `seasonal_pattern`, `seasonality_index`, `service_level_30d`, `fill_rate_30d`) plus a time-based safety net: `OR product_metrics.last_calculated < NOW() - INTERVAL '1 day'`. This guarantees every row is refreshed at least daily.
|
||||
|
||||
---
|
||||
|
||||
## MEDIUM Issues
|
||||
|
||||
### M1. Demand variability calculated only over activity days, not full 30-day window
|
||||
|
||||
**Location:** `update_product_metrics.sql`, DemandVariability CTE (lines 206-223)
|
||||
**Symptom:** Variance, std_dev, and CV are computed over only the days that appear in snapshots (activity days), not the full 30-day period including zero-sales days.
|
||||
|
||||
Example: Product 41141 (Mexican Poppy) sold 102 units in 30 days across only 3 snapshot days (1, 1, 100). The variance/CV is calculated over just those 3 data points instead of 30 (with 27 zero-sales days).
|
||||
|
||||
**Impact:**
|
||||
- CV is computed on sparse data (3-10 points instead of 30), making it statistically unreliable
|
||||
- Products with sporadic large orders appear less variable than they really are
|
||||
- `demand_pattern` classification is affected (stable/variable/sporadic/lumpy)
|
||||
|
||||
**Recommended fix:** Join against a generated 30-day date series and COALESCE missing days to 0 units sold before computing variance/stddev/CV.
|
||||
|
||||
**Status: FIXED.** Rewrote `DemandVariability` CTE to use `generate_series()` for the full 30-day date range, `CROSS JOIN` with distinct PIDs from snapshots, and `LEFT JOIN` actual snapshot data with `COALESCE(dps.units_sold, 0)` for missing days. Variance/stddev/CV now computed over all 30 data points.
|
||||
|
||||
---
|
||||
|
||||
### M2. `costeach` fallback to `price * 0.5` affects 32.5% of recent orders
|
||||
|
||||
**Location:** `orders.js`, line 600 and 634
|
||||
**Symptom:** When no cost record exists in `order_costs`, the import falls back to `price * 0.5`.
|
||||
|
||||
Data shows 9,839 of 30,266 recent orders (32.5%) use this fallback. Among these, 79 paid products have `costeach = 0` because `price = 0 * 0.5 = 0`, even though the product has a real cost_price.
|
||||
|
||||
The daily snapshot has a second line of defense (using `get_weighted_avg_cost()` and then `p.cost_price`), but the orders table's `costeach` column itself contains inaccurate data for ~1/3 of orders.
|
||||
|
||||
**Impact:** COGS calculations at the order level are approximate for 1/3 of orders. The snapshot's fallback chain mitigates this somewhat, but any analytics using `orders.costeach` directly will be affected.
|
||||
|
||||
**Status: FIXED.** Added `products.cost_price` as intermediate fallback: `COALESCE(oc.costeach, p.cost_price, oi.price * 0.5)`. The products table join was added to both the `order_totals` CTE and the outer SELECT in `orders.js`. Requires a full orders re-import to apply retroactively.
|
||||
|
||||
---
|
||||
|
||||
### M3. `lifetime_sales` uses MySQL `total_sold` (status >= 20) but orders import uses status >= 15
|
||||
|
||||
**Location:** `products.js` line 200 vs `orders.js` line 69
|
||||
**Symptom:** `total_sold` in the products table comes from MySQL with `order_status >= 20`, excluding status 15 (canceled) and 16 (combined). But the orders import fetches orders with `order_status >= 15`.
|
||||
|
||||
Verified in MySQL: For product 31286, `total_sold` (>=20) = 13,786 vs (>=15) = 13,905 (difference of 119 units).
|
||||
|
||||
**Impact:** `lifetime_sales` in product_metrics (sourced from `products.total_sold`) slightly understates compared to what the orders table contains. The `lifetime_revenue_quality` field correctly flags most as "estimated" since the orders table only covers ~5 years while `total_sold` is all-time. This is a minor inconsistency (< 1% difference).
|
||||
|
||||
**Status:** Accepted. < 1% difference, not worth the complexity of aligning thresholds.
|
||||
|
||||
---
|
||||
|
||||
### M4. `sell_through_30d` has 868 NULL values and 547 anomalous values for products with sales
|
||||
|
||||
**Location:** `update_product_metrics.sql`, lines 356-361
|
||||
**Formula:** `(sales_30d / (current_stock + sales_30d + returns_units_30d - received_qty_30d)) * 100`
|
||||
|
||||
- 868 products with sales but NULL sell_through (denominator = 0, which happens when `current_stock + sales - received = 0`, i.e. all stock came from receiving and was sold)
|
||||
- 259 products with sell_through > 100%
|
||||
- 288 products with negative sell_through
|
||||
|
||||
**Impact:** Sell-through rate is unreliable for products with significant receiving activity in the same period. The formula tries to approximate "beginning inventory" but the approximation breaks when current stock ≠ actual beginning stock (which is always, per issue C2).
|
||||
|
||||
**Status:** Will improve once C2 fix (historical stock) is deployed and snapshots are rebuilt, since `current_stock` in the formula will then reflect actual beginning inventory.
|
||||
|
||||
---
|
||||
|
||||
## LOW / INFORMATIONAL Issues
|
||||
|
||||
### L1. Snapshots only cover ~1,167 products/day out of 681K
|
||||
|
||||
Only products with order or receiving activity on a given day get snapshots. This is by design (the `ProductsWithActivity` CTE on line 133 of `update_daily_snapshots.sql`), but it means:
|
||||
- 560K+ products have zero snapshot history
|
||||
- Stockout tracking is impossible for products with no sales (they can't appear in snapshots)
|
||||
- The "avg_stock" metrics (avg_stock_units_30d, etc.) only average over activity days, not all 30 days
|
||||
|
||||
This is acceptable for storage efficiency but should be understood when interpreting metrics.
|
||||
|
||||
**Status:** Accepted (by design).
|
||||
|
||||
---
|
||||
|
||||
### L2. `detect_seasonal_pattern` function only compares current month to yearly average
|
||||
|
||||
The seasonality detection is simplistic: it compares current month's avg daily sales to yearly avg. This means:
|
||||
- It can only detect if the CURRENT month is above average, not identify historical seasonal patterns
|
||||
- Running in January vs July will give completely different results for the same product
|
||||
- The "peak_season" field always shows the current month/quarter when seasonal (not the actual peak)
|
||||
|
||||
This is noted as a P5 (low priority) feature and is adequate for a first pass but should not be relied upon for demand planning.
|
||||
|
||||
**Status: FIXED.** Rewrote `detect_seasonal_pattern` function to compare monthly average sales across the full last 12 months. Uses CV across months + peak-to-average ratio for classification: `strong` (CV > 0.5, peak > 150%), `moderate` (CV > 0.3, peak > 120%), `none`. Peak season now identifies the actual highest-sales month. Requires at least 3 months of data. Saved in `db/functions.sql`.
|
||||
|
||||
---
|
||||
|
||||
### L3. Free product with negative revenue in top sellers
|
||||
|
||||
Product 476848 ("Thank You, From ACOT!") shows 254 sales with -$1.00 revenue because one order applied a $1 discount to a $0 product. This is a data oddity, not a calculation bug. Could be addressed by excluding $0-price products from revenue metrics or by data cleanup.
|
||||
|
||||
**Status:** Accepted (data oddity, not a bug).
|
||||
|
||||
---
|
||||
|
||||
### L4. `landing_cost_price` is always NULL
|
||||
|
||||
`current_landing_cost_price` in product_metrics is mapped from `current_effective_cost` which is just `cost_price`. The `landing_cost_price` concept (cost + shipping + duties) is not implemented. The field exists but has no meaningful data.
|
||||
|
||||
**Status: FIXED.** Removed `landing_cost_price` from `db/schema.sql`, `current_landing_cost_price` from `db/metrics-schema-new.sql`, `update_product_metrics.sql`, and `backfill/populate_initial_product_metrics.sql`. Column should be dropped from the live database via `ALTER TABLE`.
|
||||
|
||||
---
|
||||
|
||||
### L5. Custom SQL functions not tracked in version control
|
||||
|
||||
All 6 custom functions (`calculate_sales_velocity`, `get_weighted_avg_cost`, `safe_divide`, `std_numeric`, `classify_demand_pattern`, `detect_seasonal_pattern`) and the `category_hierarchy` materialized view exist only in the database. They are not defined in any migration or schema file in the repository.
|
||||
|
||||
If the database needs to be recreated, these would be lost.
|
||||
|
||||
**Status: FIXED.** All 6 functions and the `category_hierarchy` materialized view definition saved to `inventory-server/db/functions.sql`. File is re-runnable via `psql -f functions.sql`.
|
||||
|
||||
---
|
||||
|
||||
### L6. `get_weighted_avg_cost` limited to last 10 receivings
|
||||
|
||||
The function `LIMIT 10` for performance, but this means products with many small receivings may not accurately reflect the true weighted average cost if the cost has changed significantly beyond the last 10 receiving records.
|
||||
|
||||
**Status: FIXED.** Removed `LIMIT 10` from `get_weighted_avg_cost`. Data shows max receivings per product is 142 (p95 = 11, avg = 3), so performance impact is negligible. Updated definition in `db/functions.sql`.
|
||||
|
||||
---
|
||||
|
||||
## Verification Summary
|
||||
|
||||
### What's Working Correctly
|
||||
|
||||
| Check | Result |
|
||||
|-------|--------|
|
||||
| 30d sales: product_metrics vs orders vs snapshots | **MATCH** (verified top 10 sellers) |
|
||||
| Replenishment formula: manual calc vs stored | **MATCH** (verified 10 products) |
|
||||
| Brand metrics vs sum of product_metrics | **MATCH** (0 difference across all brands) |
|
||||
| Order status mapping (numeric → text) | **CORRECT** (all statuses mapped, no numeric remain) |
|
||||
| Cost price: PostgreSQL vs MySQL source | **MATCH** (within rounding, verified 5 products) |
|
||||
| total_sold: PostgreSQL vs MySQL source | **MATCH** (verified 5 products) |
|
||||
| Category rollups (rolled-up > direct for parents) | **CORRECT** |
|
||||
| ABC classification distribution | **REASONABLE** (A: 8K, B: 12.5K, C: 113K) |
|
||||
| Lead time calculation (PO → receiving) | **CORRECT** (verified examples) |
|
||||
|
||||
### Data Overview
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Total products | 681,912 |
|
||||
| Products in product_metrics | 681,912 (100%) |
|
||||
| Products with 30d sales | 10,291 (1.5%) |
|
||||
| Products with negative profit & revenue | 139 (mostly cost > price) |
|
||||
| Products with negative stock | 0 |
|
||||
| Snapshot date range | 2020-06-18 to 2026-02-08 |
|
||||
| Avg products per snapshot day | 1,167 |
|
||||
| Order date range | 2020-06-18 to 2026-02-08 |
|
||||
| Total orders | 2,885,825 |
|
||||
| 'returned' status orders | 0 (returns via negative quantity only) |
|
||||
|
||||
---
|
||||
|
||||
## Fix Status Summary
|
||||
|
||||
| Issue | Severity | Status | Deployment Action Needed |
|
||||
|-------|----------|--------|--------------------------|
|
||||
| C1 | Critical | Owner resolving | Rebuild daily snapshots |
|
||||
| C2 | Critical | **FIXED** | Run import, rebuild daily snapshots |
|
||||
| C3 | Critical | **FIXED** | Deploy updated `update_product_metrics.sql` |
|
||||
| M1 | Medium | **FIXED** | Deploy updated `update_product_metrics.sql` |
|
||||
| M2 | Medium | **FIXED** | Full orders re-import (`--full`) |
|
||||
| M3 | Medium | Accepted | None |
|
||||
| M4 | Medium | Pending C2 | Will improve after C2 deployment |
|
||||
| L1 | Low | Accepted | None |
|
||||
| L2 | Low | **FIXED** | Deploy `db/functions.sql` to database |
|
||||
| L3 | Low | Accepted | None |
|
||||
| L4 | Low | **FIXED** | `ALTER TABLE` to drop columns |
|
||||
| L5 | Low | **FIXED** | None (file committed) |
|
||||
| L6 | Low | **FIXED** | Deploy `db/functions.sql` to database |
|
||||
|
||||
### Deployment Steps
|
||||
|
||||
1. Deploy `db/functions.sql` to PostgreSQL: `psql -d inventory_db -f db/functions.sql` (L2, L6)
|
||||
2. Run import (includes stock snapshots first load) (C2, M2)
|
||||
3. Drop stale columns: `ALTER TABLE products DROP COLUMN IF EXISTS landing_cost_price; ALTER TABLE product_metrics DROP COLUMN IF EXISTS current_landing_cost_price;` (L4)
|
||||
4. Rebuild daily snapshots (C1, C2)
|
||||
5. Re-run metrics calculation (C3, M1 take effect automatically)
|
||||
@@ -0,0 +1,204 @@
|
||||
# Permission System Documentation
|
||||
|
||||
This document outlines the permission system implemented in the Inventory Manager application.
|
||||
|
||||
## Permission Structure
|
||||
|
||||
Permissions follow this naming convention:
|
||||
|
||||
- Page access: `access:{page_name}`
|
||||
- Settings sections: `settings:{section_name}`
|
||||
- Admin features: `admin:{feature}`
|
||||
|
||||
Examples:
|
||||
- `access:products` - Can access the Products page
|
||||
- `settings:user_management` - Can access User Management settings
|
||||
- `admin:debug` - Can see debug information
|
||||
|
||||
## Permission Components
|
||||
|
||||
### PermissionGuard
|
||||
|
||||
The core component that conditionally renders content based on permissions.
|
||||
|
||||
```tsx
|
||||
<PermissionGuard
|
||||
permission="settings:user_management"
|
||||
fallback={<p>No permission</p>}
|
||||
>
|
||||
<button>Manage Users</button>
|
||||
</PermissionGuard>
|
||||
```
|
||||
|
||||
Options:
|
||||
- `permission`: Single permission code
|
||||
- `anyPermissions`: Array of permissions (ANY match grants access)
|
||||
- `allPermissions`: Array of permissions (ALL required)
|
||||
- `adminOnly`: For admin-only sections
|
||||
- `page`: Page name (checks `access:{page}` permission)
|
||||
- `fallback`: Content to show if permission check fails
|
||||
|
||||
### PermissionProtectedRoute
|
||||
|
||||
Protects entire pages based on page access permissions.
|
||||
|
||||
```tsx
|
||||
<Route path="/products" element={
|
||||
<PermissionProtectedRoute page="products">
|
||||
<Products />
|
||||
</PermissionProtectedRoute>
|
||||
} />
|
||||
```
|
||||
|
||||
### ProtectedSection
|
||||
|
||||
Protects sections within a page based on action permissions.
|
||||
|
||||
```tsx
|
||||
<ProtectedSection page="products" action="create">
|
||||
<button>Add Product</button>
|
||||
</ProtectedSection>
|
||||
```
|
||||
|
||||
### PermissionButton
|
||||
|
||||
Button that automatically handles permissions.
|
||||
|
||||
```tsx
|
||||
<PermissionButton
|
||||
page="products"
|
||||
action="create"
|
||||
onClick={handleCreateProduct}
|
||||
>
|
||||
Add Product
|
||||
</PermissionButton>
|
||||
```
|
||||
|
||||
### SettingsSection
|
||||
|
||||
Specific component for settings with built-in permission checks.
|
||||
|
||||
```tsx
|
||||
<SettingsSection
|
||||
title="System Settings"
|
||||
description="Configure global settings"
|
||||
permission="settings:global"
|
||||
>
|
||||
{/* Settings content */}
|
||||
</SettingsSection>
|
||||
```
|
||||
|
||||
## Permission Hooks
|
||||
|
||||
### usePermissions
|
||||
|
||||
Core hook for checking any permission.
|
||||
|
||||
```tsx
|
||||
const { hasPermission, hasPageAccess, isAdmin } = usePermissions();
|
||||
if (hasPermission('settings:user_management')) {
|
||||
// Can access user management
|
||||
}
|
||||
```
|
||||
|
||||
### usePagePermission
|
||||
|
||||
Specialized hook for page-level permissions.
|
||||
|
||||
```tsx
|
||||
const { canView, canCreate, canEdit, canDelete } = usePagePermission('products');
|
||||
if (canView()) {
|
||||
// Can view products
|
||||
}
|
||||
```
|
||||
|
||||
## Database Schema
|
||||
|
||||
Permissions are stored in the database:
|
||||
- `permissions` table: Stores all available permissions
|
||||
- `user_permissions` junction table: Maps permissions to users
|
||||
|
||||
Admin users automatically have all permissions.
|
||||
|
||||
## Implemented Permission Codes
|
||||
|
||||
### Page Access Permissions
|
||||
| Code | Description |
|
||||
|------|-------------|
|
||||
| `access:dashboard` | Access to Dashboard page |
|
||||
| `access:overview` | Access to Overview page |
|
||||
| `access:products` | Access to Products page |
|
||||
| `access:categories` | Access to Categories page |
|
||||
| `access:brands` | Access to Brands page |
|
||||
| `access:vendors` | Access to Vendors page |
|
||||
| `access:purchase_orders` | Access to Purchase Orders page |
|
||||
| `access:analytics` | Access to Analytics page |
|
||||
| `access:forecasting` | Access to Forecasting page |
|
||||
| `access:import` | Access to Import page |
|
||||
| `access:settings` | Access to Settings page |
|
||||
| `access:chat` | Access to Chat Archive page |
|
||||
|
||||
### Settings Permissions
|
||||
| Code | Description |
|
||||
|------|-------------|
|
||||
| `settings:global` | Access to Global Settings section |
|
||||
| `settings:products` | Access to Product Settings section |
|
||||
| `settings:vendors` | Access to Vendor Settings section |
|
||||
| `settings:data_management` | Access to Data Management settings |
|
||||
| `settings:calculation_settings` | Access to Calculation Settings |
|
||||
| `settings:library_management` | Access to Image Library Management |
|
||||
| `settings:performance_metrics` | Access to Performance Metrics |
|
||||
| `settings:prompt_management` | Access to AI Prompt Management |
|
||||
| `settings:stock_management` | Access to Stock Management |
|
||||
| `settings:templates` | Access to Template Management |
|
||||
| `settings:user_management` | Access to User Management |
|
||||
|
||||
### Admin Permissions
|
||||
| Code | Description |
|
||||
|------|-------------|
|
||||
| `admin:debug` | Can see debug information and features |
|
||||
|
||||
## Implementation Examples
|
||||
|
||||
### Page Protection
|
||||
|
||||
In `App.tsx`:
|
||||
```tsx
|
||||
<Route path="/products" element={
|
||||
<PermissionProtectedRoute page="products">
|
||||
<Products />
|
||||
</PermissionProtectedRoute>
|
||||
} />
|
||||
```
|
||||
|
||||
### Component Level Protection
|
||||
|
||||
```tsx
|
||||
const { hasPermission } = usePermissions();
|
||||
|
||||
function handleAction() {
|
||||
if (!hasPermission('settings:user_management')) {
|
||||
toast.error("You don't have permission");
|
||||
return;
|
||||
}
|
||||
// Action logic
|
||||
}
|
||||
```
|
||||
|
||||
### UI Element Protection
|
||||
|
||||
```tsx
|
||||
<PermissionGuard permission="settings:user_management">
|
||||
<button onClick={handleManageUsers}>
|
||||
Manage Users
|
||||
</button>
|
||||
</PermissionGuard>
|
||||
```
|
||||
|
||||
## Notes
|
||||
|
||||
- **Page Access**: These permissions control which pages a user can navigate to
|
||||
- **Settings Access**: These permissions control access to different sections within the Settings page
|
||||
- **Admin Features**: Special permissions for administrative functions
|
||||
- **CRUD Operations**: The application currently focuses on viewing and managing data rather than creating/editing/deleting individual records
|
||||
- **User Management**: User CRUD operations are handled through the settings interface rather than dedicated user management pages
|
||||
@@ -0,0 +1,396 @@
|
||||
# ValidationStep Component Refactoring Plan
|
||||
|
||||
## Overview
|
||||
|
||||
This document outlines a comprehensive plan to refactor the current ValidationStep component (4000+ lines) into a more maintainable, modular structure. The new implementation will be developed alongside the existing component without modifying the original code. Once completed, the previous step in the workflow will offer the option to continue to either the original ValidationStep or the new implementation.
|
||||
|
||||
## Table of Contents
|
||||
|
||||
1. [Current Component Analysis](#current-component-analysis)
|
||||
2. [New Architecture Design](#new-architecture-design)
|
||||
3. [Component Structure](#component-structure)
|
||||
4. [State Management](#state-management)
|
||||
5. [Key Features Implementation](#key-features-implementation)
|
||||
6. [Integration Plan](#integration-plan)
|
||||
7. [Testing Strategy](#testing-strategy)
|
||||
8. [Project Timeline](#project-timeline)
|
||||
9. [Design Principles](#design-principles)
|
||||
10. [Appendix: Function Reference](#appendix-function-reference)
|
||||
|
||||
## Current Component Analysis
|
||||
|
||||
The current ValidationStep component has several issues:
|
||||
|
||||
- **Size**: At over 4000 lines, it's difficult to maintain and understand
|
||||
- **Multiple responsibilities**: Handles validation, UI rendering, template management, and more
|
||||
- **Special cases**: Contains numerous special case handlers and exceptions
|
||||
- **Complex state management**: State is distributed across multiple useState calls
|
||||
- **Tightly coupled concerns**: UI, validation logic, and business rules are intertwined
|
||||
|
||||
### Key Features to Preserve
|
||||
|
||||
1. **Data Validation**
|
||||
- Field-level validation (required, regex, unique)
|
||||
- Row-level validation (supplier, company fields)
|
||||
- UPC validation with API integration
|
||||
- AI-assisted validation
|
||||
|
||||
2. **Template Management**
|
||||
- Saving, loading, and applying templates
|
||||
- Template-based validation
|
||||
|
||||
3. **UI Components**
|
||||
- Editable table with specialized cell renderers
|
||||
- Error display and management
|
||||
- Filtering and sorting capabilities
|
||||
- Status indicators and progress tracking
|
||||
|
||||
4. **Special Field Handling**
|
||||
- Input fields with price formatting
|
||||
- Multi-input fields with separator configuration
|
||||
- Select fields with dropdown options
|
||||
- Checkbox fields with boolean value mapping
|
||||
- UPC fields with specialized validation
|
||||
|
||||
5. **User Interaction Flows**
|
||||
- Tab and keyboard navigation
|
||||
- Bulk operations (select all, apply template)
|
||||
- Row validation on value change
|
||||
- Error reporting and display
|
||||
|
||||
## New Architecture Design
|
||||
|
||||
The new architecture will follow these principles:
|
||||
|
||||
1. **Separation of Concerns**
|
||||
- UI rendering separate from business logic
|
||||
- Validation logic isolated from state management
|
||||
- Clear interfaces between components
|
||||
|
||||
2. **Composable Components**
|
||||
- Small, focused components with single responsibilities
|
||||
- Reusable pattern for different field types
|
||||
|
||||
3. **Centralized State Management**
|
||||
- Custom hooks for state management
|
||||
- Clear data flow patterns
|
||||
- Reduced prop drilling
|
||||
|
||||
4. **Consistent Error Handling**
|
||||
- Standardized error structure
|
||||
- Predictable error propagation
|
||||
- User-friendly error display
|
||||
|
||||
5. **Performance Optimization**
|
||||
- Virtualized table rendering
|
||||
- Memoization of expensive computations
|
||||
- Deferred validation for better user experience
|
||||
|
||||
## Component Structure
|
||||
|
||||
The new ValidationStepNew folder has the following structure:
|
||||
|
||||
```
|
||||
ValidationStepNew/
|
||||
├── index.tsx # Main entry point that composes all pieces
|
||||
├── components/ # UI Components
|
||||
│ ├── ValidationContainer.tsx # Main wrapper component
|
||||
│ ├── ValidationTable.tsx # Table implementation
|
||||
│ ├── ValidationCell.tsx # Cell component
|
||||
│ ├── ValidationSidebar.tsx # Sidebar with controls
|
||||
│ ├── ValidationToolbar.tsx # Top toolbar (removed as unnecessary)
|
||||
│ ├── TemplateManager.tsx # Template management
|
||||
│ ├── FilterPanel.tsx # Filtering interface (integrated into Container)
|
||||
│ └── cells/ # Specialized cell renderers
|
||||
│ ├── InputCell.tsx
|
||||
│ ├── SelectCell.tsx
|
||||
│ ├── MultiInputCell.tsx
|
||||
│ └── CheckboxCell.tsx
|
||||
├── hooks/ # Custom hooks
|
||||
│ ├── useValidationState.tsx # Main state management
|
||||
│ ├── useTemplates.tsx # Template-related logic (integrated into ValidationState)
|
||||
│ ├── useFilters.tsx # Filtering logic (integrated into ValidationState)
|
||||
│ └── useUpcValidation.tsx # UPC-specific validation
|
||||
└── utils/ # Utility functions
|
||||
├── validationUtils.ts # Validation helper functions
|
||||
├── formatters.ts # Value formatting utilities
|
||||
└── constants.ts # Constant values and configuration
|
||||
```
|
||||
|
||||
### Component Responsibilities
|
||||
|
||||
#### ValidationContainer
|
||||
- Main container component
|
||||
- Coordinates between subcomponents
|
||||
- Manages global state
|
||||
- Handles navigation events (next, back)
|
||||
- Contains filter controls
|
||||
|
||||
#### ValidationTable
|
||||
- Displays the data in tabular form
|
||||
- Manages selection state
|
||||
- Handles keyboard navigation
|
||||
- Integrates with TanStack Table
|
||||
- Displays properly styled rows and cells
|
||||
|
||||
#### ValidationCell
|
||||
- Factory component that renders appropriate cell type
|
||||
- Manages cell-level state
|
||||
- Handles validation errors display
|
||||
- Manages edit mode
|
||||
- Shows consistent error indicators
|
||||
|
||||
#### TemplateManager
|
||||
- Handles template selection UI
|
||||
- Provides template save/load functionality
|
||||
- Manages template application to rows
|
||||
|
||||
#### Cell Components
|
||||
- **InputCell**: Handles text input with multiline and price support
|
||||
- **MultiInputCell**: Handles multiple values with separator configuration
|
||||
- **SelectCell**: Command/popover component for single selection
|
||||
- **CheckboxCell**: Boolean value selection with mapping support
|
||||
|
||||
## State Management
|
||||
|
||||
### Core State Interface
|
||||
|
||||
```typescript
|
||||
interface ValidationState<T extends string> {
|
||||
// Core data
|
||||
data: RowData<T>[];
|
||||
filteredData: RowData<T>[];
|
||||
|
||||
// Validation state
|
||||
isValidating: boolean;
|
||||
validationErrors: Map<number, Record<string, Error[]>>;
|
||||
rowValidationStatus: Map<number, 'pending' | 'validating' | 'validated' | 'error'>;
|
||||
|
||||
// Selection state
|
||||
rowSelection: RowSelectionState;
|
||||
|
||||
// Template state
|
||||
templates: Template[];
|
||||
selectedTemplateId: string | null;
|
||||
|
||||
// Filter state
|
||||
filters: FilterState;
|
||||
|
||||
// Methods
|
||||
updateRow: (rowIndex: number, key: T, value: any) => void;
|
||||
validateRow: (rowIndex: number) => Promise<void>;
|
||||
validateUpc: (rowIndex: number, upcValue: string) => Promise<void>;
|
||||
applyTemplate: (templateId: string, rowIndexes: number[]) => void;
|
||||
saveTemplate: (name: string, type: string) => void;
|
||||
setFilters: (newFilters: Partial<FilterState>) => void;
|
||||
// Additional methods...
|
||||
}
|
||||
```
|
||||
|
||||
### useValidationState Hook
|
||||
|
||||
The main state management hook handles:
|
||||
|
||||
- Data manipulation (update, sort, filter)
|
||||
- Selection management
|
||||
- Validation coordination
|
||||
- Integration with validation utilities
|
||||
- Template management
|
||||
- Filtering and sorting
|
||||
|
||||
## Key Features Implementation
|
||||
|
||||
### 1. Field Type Handling
|
||||
|
||||
Implemented a strategy pattern for different field types:
|
||||
|
||||
```typescript
|
||||
// In ValidationCell
|
||||
const renderCellContent = () => {
|
||||
const fieldType = field.fieldType.type
|
||||
|
||||
switch (fieldType) {
|
||||
case 'input':
|
||||
return <InputCell<T> field={field} value={value} onChange={onChange} ... />
|
||||
case 'multi-input':
|
||||
return <MultiInputCell<T> field={field} value={value} onChange={onChange} ... />
|
||||
case 'select':
|
||||
return <SelectCell<T> field={field} value={value} onChange={onChange} ... />
|
||||
// etc.
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Validation Logic
|
||||
|
||||
Validation is broken down into clear steps:
|
||||
|
||||
1. **Field Validation**: Apply field-level validations (required, regex, etc.)
|
||||
2. **Row Validation**: Apply row-level validations and rowHook
|
||||
3. **Table Validation**: Apply table-level validations (unique) and tableHook
|
||||
|
||||
Validation now happens automatically without explicit buttons, with immediate feedback on field blur.
|
||||
|
||||
### 3. UI Components
|
||||
|
||||
UI components follow these principles:
|
||||
|
||||
1. **Consistent Styling**: All components use shadcn UI for consistent look and feel
|
||||
2. **Visual Feedback**: Errors are clearly indicated with icons and border styling
|
||||
3. **Intuitive Editing**: Fields show outlines even when not in focus, and edit on click
|
||||
4. **Proper Command Pattern**: Select and multi-select fields use command/popover pattern for better UX
|
||||
5. **Focus Management**: Fields close when clicking away and perform validation on blur
|
||||
|
||||
## Design Principles
|
||||
|
||||
Based on user preferences and best practices, the following design principles guide this refactoring:
|
||||
|
||||
1. **Automatic Validation**
|
||||
- Validation should happen automatically without explicit buttons
|
||||
- All validation should run on initial data load
|
||||
- Fields should validate on blur (when user clicks away)
|
||||
|
||||
2. **Modern UI Patterns**
|
||||
- Command/popover components for all selects and multi-selects
|
||||
- Consistent field outlines and borders even when not in focus
|
||||
- Badge patterns for multi-select items
|
||||
- Clear visual indicators for errors
|
||||
|
||||
3. **Reduced Complexity**
|
||||
- Remove unnecessary UI elements like "validate all" buttons
|
||||
- Eliminate redundant state and toast notifications
|
||||
- Simplify component hierarchy where possible
|
||||
- Find root causes rather than adding special cases
|
||||
|
||||
4. **Consistent Component Behavior**
|
||||
- Fields should close when clicking away
|
||||
- All inputs should follow the same editing pattern
|
||||
- Error handling should be consistent across all field types
|
||||
- Multi-select fields should allow selecting multiple items with clear visual feedback
|
||||
|
||||
## Integration Plan
|
||||
|
||||
### 1. Creating the New Component Structure
|
||||
|
||||
Folder structure has been created without modifying the existing code:
|
||||
|
||||
```bash
|
||||
mkdir -p inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/{components,hooks,utils}
|
||||
mkdir -p inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells
|
||||
```
|
||||
|
||||
### 2. Implementing Basic Components
|
||||
|
||||
Core components have been implemented:
|
||||
|
||||
1. Created index.tsx as the main entry point
|
||||
2. Implemented ValidationContainer with basic state management
|
||||
3. Created ValidationTable for data display
|
||||
4. Implemented basic cell rendering with specialized cell types
|
||||
|
||||
### 3. Implementing State Management
|
||||
|
||||
State management has been implemented:
|
||||
|
||||
1. Created useValidationState hook
|
||||
2. Implemented data transformation utilities
|
||||
3. Added validation logic
|
||||
|
||||
### 4. Integrating with Previous Step
|
||||
|
||||
The previous step component allows choosing between validation implementations, enabling gradual testing and adoption.
|
||||
|
||||
## Testing Strategy
|
||||
|
||||
1. **Unit Tests**
|
||||
- Test individual utility functions
|
||||
- Test hooks in isolation
|
||||
- Test individual UI components
|
||||
|
||||
2. **Integration Tests**
|
||||
- Test component interactions
|
||||
- Test state management flow
|
||||
- Test validation logic integration
|
||||
|
||||
3. **Comparison Tests**
|
||||
- Compare output of new component with original
|
||||
- Verify that all functionality works the same
|
||||
|
||||
4. **Performance Tests**
|
||||
- Measure render times
|
||||
- Evaluate memory usage
|
||||
- Compare against original component
|
||||
|
||||
## Project Timeline
|
||||
|
||||
1. **Phase 1: Initial Structure (Completed)**
|
||||
- Set up folder structure
|
||||
- Implement basic components
|
||||
- Create core state management
|
||||
|
||||
2. **Phase 2: Core Functionality (In Progress)**
|
||||
- Implement validation logic (completed)
|
||||
- Create cell renderers (completed)
|
||||
- Add template management (in progress)
|
||||
|
||||
3. **Phase 3: Special Features (Upcoming)**
|
||||
- Implement UPC validation
|
||||
- Add AI validation
|
||||
- Handle special cases
|
||||
|
||||
4. **Phase 4: UI Refinement (Ongoing)**
|
||||
- Improve error display (completed)
|
||||
- Enhance user interactions (completed)
|
||||
- Optimize performance (in progress)
|
||||
|
||||
5. **Phase 5: Testing and Integration (Upcoming)**
|
||||
- Write tests
|
||||
- Fix bugs
|
||||
- Integrate with previous step
|
||||
|
||||
## Appendix: Function Reference
|
||||
|
||||
This section documents the core functions from the original ValidationStep that need to be preserved in the new implementation.
|
||||
|
||||
### Validation Functions
|
||||
|
||||
1. **validateRegex** - Validates values against regex patterns
|
||||
2. **getValidationError** - Determines field-level validation errors
|
||||
3. **validateAndCommit** - Validates and commits new values
|
||||
4. **validateData** - Validates all data rows
|
||||
5. **validateUpcAndGenerateItemNumbers** - Validates UPC codes and generates item numbers
|
||||
|
||||
### Formatting Functions
|
||||
|
||||
1. **formatPrice** - Formats price values
|
||||
2. **getDisplayValue** - Gets formatted display value based on field type
|
||||
3. **isMultiInputType** - Checks if field is multi-input type
|
||||
4. **getMultiInputSeparator** - Gets separator for multi-input fields
|
||||
5. **isPriceField** - Checks if field should be formatted as price
|
||||
|
||||
### Template Functions
|
||||
|
||||
1. **loadTemplates** - Loads templates from storage
|
||||
2. **saveTemplate** - Saves a new template
|
||||
3. **applyTemplate** - Applies a template to selected rows
|
||||
4. **getTemplateDisplayText** - Gets display text for a template
|
||||
|
||||
### AI Validation Functions
|
||||
|
||||
1. **handleAiValidation** - Triggers AI validation
|
||||
2. **showCurrentPrompt** - Shows current AI prompt
|
||||
3. **getFieldDisplayValue** - Gets display value for a field
|
||||
4. **highlightDifferences** - Highlights differences between original and corrected values
|
||||
5. **getFieldDisplayValueWithHighlight** - Gets display value with highlighted differences
|
||||
6. **revertAiChange** - Reverts an AI-suggested change
|
||||
7. **isChangeReverted** - Checks if an AI change has been reverted
|
||||
|
||||
### Event Handlers
|
||||
|
||||
1. **handleUpcValueUpdate** - Handles UPC value updates
|
||||
2. **handleBlur** - Handles input blur events
|
||||
3. **handleWheel** - Handles wheel events for navigation
|
||||
4. **copyValueDown** - Copies a value to cells below
|
||||
5. **handleSkuGeneration** - Generates SKUs
|
||||
|
||||
By following this refactoring plan, we continue to transform the monolithic ValidationStep component into a modular, maintainable set of components while preserving all existing functionality and aligning with user preferences for design and behavior.
|
||||
@@ -0,0 +1,137 @@
|
||||
# ValidationStepNew Implementation Status
|
||||
|
||||
## Overview
|
||||
|
||||
This document outlines the current status of the ValidationStepNew implementation, a refactored version of the original ValidationStep component. The goal is to create a more maintainable, modular component that preserves all functionality of the original while eliminating technical debt and implementing modern UI patterns.
|
||||
|
||||
## Design Principles
|
||||
|
||||
Based on the user's preferences, we're following these core design principles:
|
||||
|
||||
1. **Automatic Validation**
|
||||
- ✅ Validation runs automatically on data load
|
||||
- ✅ No explicit "validate all" button needed
|
||||
- ✅ Fields validate on blur when user clicks away
|
||||
- ✅ Immediate visual feedback for validation errors
|
||||
|
||||
2. **Modern UI Patterns**
|
||||
- ✅ Command/popover components for selects and multi-selects
|
||||
- ✅ Consistent field outlines and borders even when not in focus
|
||||
- ✅ Badge pattern for multi-select field items
|
||||
- ✅ Visual indicators for errors with appropriate styling
|
||||
|
||||
3. **Reduced Complexity**
|
||||
- ✅ Removed unnecessary UI elements like "validate all" button
|
||||
- ✅ Eliminated redundant toast notifications
|
||||
- ✅ Simplified component hierarchy
|
||||
- ✅ Fixed root causes rather than adding special cases
|
||||
|
||||
4. **Consistent Behavior**
|
||||
- ✅ Fields close when clicking away
|
||||
- ✅ All inputs follow the same editing pattern
|
||||
- ✅ Error handling is consistent across field types
|
||||
- ✅ Multi-select fields allow selecting multiple items
|
||||
|
||||
## Completed Components
|
||||
|
||||
### Core Structure
|
||||
- ✅ Main component structure
|
||||
- ✅ Directory organization
|
||||
- ✅ TypeScript interfaces
|
||||
- ✅ Props definition and passing
|
||||
|
||||
### State Management
|
||||
- ✅ `useValidationState` hook for centralized state
|
||||
- ✅ Data validation logic
|
||||
- ✅ Integration with rowHook and tableHook
|
||||
- ✅ Error tracking and management
|
||||
- ✅ Row selection
|
||||
- ✅ Automatic validation on data load
|
||||
|
||||
### UI Components
|
||||
- ✅ ValidationContainer with appropriate layout
|
||||
- ✅ ValidationTable with shadcn UI components
|
||||
- ✅ ValidationCell factory component
|
||||
- ✅ Row select/deselect functionality
|
||||
- ✅ Error display and indicators
|
||||
- ✅ Selection action bar
|
||||
|
||||
### Cell Components
|
||||
- ✅ InputCell with price and multiline support
|
||||
- ✅ MultiInputCell with separator configuration
|
||||
- ✅ SelectCell using command/popover pattern
|
||||
- ✅ CheckboxCell with boolean mapping
|
||||
- ✅ Consistent styling across all field types
|
||||
- ✅ Proper edit/view state management
|
||||
- ✅ Outlined borders in both edit and view modes
|
||||
|
||||
### Utility Functions
|
||||
- ✅ Value formatting for display
|
||||
- ✅ Field type detection
|
||||
- ✅ Error creation and management
|
||||
- ✅ Price formatting
|
||||
|
||||
### UI Improvements
|
||||
- ✅ Consistent borders and field outlines
|
||||
- ✅ Fields that properly close when clicking away
|
||||
- ✅ Multi-select with badge UI pattern
|
||||
- ✅ Command pattern for searchable select menus
|
||||
- ✅ Better visual error indication
|
||||
|
||||
## Pending Tasks
|
||||
|
||||
### Enhanced Validation
|
||||
- ⏳ AI validation system
|
||||
- ⏳ Custom validation hooks
|
||||
- ⏳ Enhanced UPC validation with API integration
|
||||
- ⏳ Validation visualizations
|
||||
|
||||
### Advanced UI Features
|
||||
- ⏳ Table virtualization for performance
|
||||
- ⏳ Drag-and-drop reordering
|
||||
- ⏳ Bulk operations (copy down, fill all, etc.)
|
||||
- ⏳ Keyboard navigation improvements
|
||||
- ⏳ Template dialogs and management UI
|
||||
|
||||
### Special Features
|
||||
- ⏳ Image preview integration
|
||||
- ⏳ SKU generation system
|
||||
- ⏳ Item number generation
|
||||
- ⏳ Dependent dropdown values
|
||||
|
||||
### Testing
|
||||
- ⏳ Unit tests for utility functions
|
||||
- ⏳ Component tests
|
||||
- ⏳ Integration tests
|
||||
- ⏳ Performance benchmarks
|
||||
|
||||
## Known Issues
|
||||
|
||||
1. TypeScript error for `validationDisabled` property in ValidationCell.tsx
|
||||
2. Some type casting is needed due to complex generic types
|
||||
3. Need to address edge cases for multi-select fields validation
|
||||
4. Proper error handling for API calls needs implementation
|
||||
|
||||
## Next Steps
|
||||
|
||||
1. Fix TypeScript errors in ValidationCell and related components
|
||||
2. Complete template management functionality
|
||||
3. Implement UPC validation with API integration
|
||||
4. Make multi-select field validation more robust
|
||||
5. Add comprehensive tests
|
||||
|
||||
## Performance Improvements
|
||||
|
||||
We've already implemented several performance optimizations:
|
||||
|
||||
1. ✅ More efficient state updates by removing unnecessary re-renders
|
||||
2. ✅ Better error handling to prevent cascading validations
|
||||
3. ✅ Improved component isolation to prevent unnecessary re-renders
|
||||
4. ✅ Automatic validation that doesn't block the UI
|
||||
|
||||
Additional planned improvements:
|
||||
|
||||
1. Virtualized table rendering for large datasets
|
||||
2. Memoization of expensive calculations
|
||||
3. Optimized state updates to minimize re-renders
|
||||
4. Batched API calls for validation
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,72 @@
|
||||
# Solution: Keeping Dropdowns Open During Multiple Selections
|
||||
|
||||
## The Problem
|
||||
|
||||
When implementing a multi-select dropdown in React, a common issue occurs:
|
||||
|
||||
1. You select an item in the dropdown
|
||||
2. The `onChange` handler is called, updating the data
|
||||
3. This triggers a re-render of the parent component (in this case, the entire table)
|
||||
4. During the re-render, the dropdown is unmounted and remounted
|
||||
5. This causes the dropdown to close before you can make multiple selections
|
||||
|
||||
## The Solution: Deferred State Updates
|
||||
|
||||
The key insight is to **separate local state management from parent state updates**:
|
||||
|
||||
```typescript
|
||||
// Step 1: Add local state to track selections
|
||||
const [internalValue, setInternalValue] = useState<string[]>(value)
|
||||
|
||||
// Step 2: Handle popover open state changes
|
||||
const handleOpenChange = useCallback((newOpen: boolean) => {
|
||||
if (open && !newOpen) {
|
||||
// Only update parent state when dropdown closes
|
||||
if (JSON.stringify(internalValue) !== JSON.stringify(value)) {
|
||||
onChange(internalValue);
|
||||
}
|
||||
}
|
||||
|
||||
setOpen(newOpen);
|
||||
|
||||
if (newOpen) {
|
||||
// Sync internal state with external state when opening
|
||||
setInternalValue(value);
|
||||
}
|
||||
}, [open, internalValue, value, onChange]);
|
||||
|
||||
// Step 3: Toggle selection only updates internal state
|
||||
const toggleSelection = useCallback((selectedValue: string) => {
|
||||
setInternalValue(prev => {
|
||||
if (prev.includes(selectedValue)) {
|
||||
return prev.filter(v => v !== selectedValue);
|
||||
} else {
|
||||
return [...prev, selectedValue];
|
||||
}
|
||||
});
|
||||
}, []);
|
||||
```
|
||||
|
||||
## Why This Works
|
||||
|
||||
1. **No parent re-renders during selection**: Since we're only updating local state, the parent component doesn't re-render during selection.
|
||||
2. **Consistent UI**: The dropdown shows accurate selected states using the internal value.
|
||||
3. **Data integrity**: The final selections are properly synchronized back to the parent when done.
|
||||
4. **Resilient to external changes**: Initial state is synchronized when opening the dropdown.
|
||||
|
||||
## Implementation Steps
|
||||
|
||||
1. Create a local state variable to track selections inside the component
|
||||
2. Only make selections against this local state while the dropdown is open
|
||||
3. Defer updating the parent until the dropdown is explicitly closed
|
||||
4. When opening, synchronize the internal state with the external value
|
||||
|
||||
## Benefits
|
||||
|
||||
This pattern:
|
||||
- Avoids re-render cycles that would unmount the dropdown
|
||||
- Maintains UI consistency during multi-selection
|
||||
- Simplifies the component's interaction with parent components
|
||||
- Works with existing component lifecycles rather than fighting against them
|
||||
|
||||
This solution is much simpler than trying to prevent event propagation or manipulating DOM events, and addresses the root cause of the issue: premature re-rendering.
|
||||
@@ -0,0 +1,342 @@
|
||||
# MySQL to PostgreSQL Import Process Documentation
|
||||
|
||||
This document outlines the data import process from the production MySQL database to the local PostgreSQL database, focusing on column mappings, data transformations, and the overall import architecture.
|
||||
|
||||
## Table of Contents
|
||||
1. [Overview](#overview)
|
||||
2. [Import Architecture](#import-architecture)
|
||||
3. [Column Mappings](#column-mappings)
|
||||
- [Categories](#categories)
|
||||
- [Products](#products)
|
||||
- [Product Categories (Relationship)](#product-categories-relationship)
|
||||
- [Orders](#orders)
|
||||
- [Purchase Orders](#purchase-orders)
|
||||
- [Metadata Tables](#metadata-tables)
|
||||
4. [Special Calculations](#special-calculations)
|
||||
5. [Implementation Notes](#implementation-notes)
|
||||
|
||||
## Overview
|
||||
|
||||
The import process extracts data from a MySQL 5.7 production database and imports it into a PostgreSQL database. It can operate in two modes:
|
||||
|
||||
- **Full Import**: Imports all data regardless of last sync time
|
||||
- **Incremental Import**: Only imports data that has changed since the last import
|
||||
|
||||
The process handles four main data types:
|
||||
- Categories (product categorization hierarchy)
|
||||
- Products (inventory items)
|
||||
- Orders (sales records)
|
||||
- Purchase Orders (vendor orders)
|
||||
|
||||
## Import Architecture
|
||||
|
||||
The import process follows these steps:
|
||||
|
||||
1. **Establish Connection**: Creates a SSH tunnel to the production server and establishes database connections
|
||||
2. **Setup Import History**: Creates a record of the current import operation
|
||||
3. **Import Categories**: Processes product categories in hierarchical order
|
||||
4. **Import Products**: Processes products with their attributes and category relationships
|
||||
5. **Import Orders**: Processes customer orders with line items, taxes, and discounts
|
||||
6. **Import Purchase Orders**: Processes vendor purchase orders with line items
|
||||
7. **Record Results**: Updates the import history with results
|
||||
8. **Close Connections**: Cleans up connections and resources
|
||||
|
||||
Each import step uses temporary tables for processing and wraps operations in transactions to ensure data consistency.
|
||||
|
||||
## Column Mappings
|
||||
|
||||
### Categories
|
||||
| PostgreSQL Column | MySQL Source | Transformation |
|
||||
|-------------------|---------------------------------|----------------------------------------------|
|
||||
| cat_id | product_categories.cat_id | Direct mapping |
|
||||
| name | product_categories.name | Direct mapping |
|
||||
| type | product_categories.type | Direct mapping |
|
||||
| parent_id | product_categories.master_cat_id| NULL for top-level categories (types 10, 20) |
|
||||
| description | product_categories.combined_name| Direct mapping |
|
||||
| status | N/A | Hard-coded 'active' |
|
||||
| created_at | N/A | Current timestamp |
|
||||
| updated_at | N/A | Current timestamp |
|
||||
|
||||
**Notes:**
|
||||
- Categories are processed in hierarchical order by type: [10, 20, 11, 21, 12, 13]
|
||||
- Type 10/20 are top-level categories with no parent
|
||||
- Types 11/21/12/13 are child categories that reference parent categories
|
||||
|
||||
### Products
|
||||
| PostgreSQL Column | MySQL Source | Transformation |
|
||||
|----------------------|----------------------------------|---------------------------------------------------------------|
|
||||
| pid | products.pid | Direct mapping |
|
||||
| title | products.description | Direct mapping |
|
||||
| description | products.notes | Direct mapping |
|
||||
| sku | products.itemnumber | Fallback to 'NO-SKU' if empty |
|
||||
| stock_quantity | shop_inventory.available_local | Capped at 5000, minimum 0 |
|
||||
| preorder_count | current_inventory.onpreorder | Default 0 |
|
||||
| notions_inv_count | product_notions_b2b.inventory | Default 0 |
|
||||
| price | product_current_prices.price_each| Default 0, filtered on active=1 |
|
||||
| regular_price | products.sellingprice | Default 0 |
|
||||
| cost_price | product_inventory | Weighted average: SUM(costeach * count) / SUM(count) when count > 0, or latest costeach |
|
||||
| vendor | suppliers.companyname | Via supplier_item_data.supplier_id |
|
||||
| vendor_reference | supplier_item_data | supplier_itemnumber or notions_itemnumber based on vendor |
|
||||
| notions_reference | supplier_item_data.notions_itemnumber | Direct mapping |
|
||||
| brand | product_categories.name | Linked via products.company |
|
||||
| line | product_categories.name | Linked via products.line |
|
||||
| subline | product_categories.name | Linked via products.subline |
|
||||
| artist | product_categories.name | Linked via products.artist |
|
||||
| categories | product_category_index | Comma-separated list of category IDs |
|
||||
| created_at | products.date_created | Validated date, NULL if invalid |
|
||||
| first_received | products.datein | Validated date, NULL if invalid |
|
||||
| landing_cost_price | NULL | Not set |
|
||||
| barcode | products.upc | Direct mapping |
|
||||
| harmonized_tariff_code| products.harmonized_tariff_code | Direct mapping |
|
||||
| updated_at | products.stamp | Validated date, NULL if invalid |
|
||||
| visible | shop_inventory | Calculated from show + buyable > 0 |
|
||||
| managing_stock | N/A | Hard-coded true |
|
||||
| replenishable | Multiple fields | Complex calculation based on reorder, dates, etc. |
|
||||
| permalink | N/A | Constructed URL with product ID |
|
||||
| moq | supplier_item_data | notions_qty_per_unit or supplier_qty_per_unit, minimum 1 |
|
||||
| uom | N/A | Hard-coded 1 |
|
||||
| rating | products.rating | Direct mapping |
|
||||
| reviews | products.rating_votes | Direct mapping |
|
||||
| weight | products.weight | Direct mapping |
|
||||
| length | products.length | Direct mapping |
|
||||
| width | products.width | Direct mapping |
|
||||
| height | products.height | Direct mapping |
|
||||
| country_of_origin | products.country_of_origin | Direct mapping |
|
||||
| location | products.location | Direct mapping |
|
||||
| total_sold | order_items | SUM(qty_ordered) for all order_items where prod_pid = pid |
|
||||
| baskets | mybasket | COUNT of records where mb.item = pid and qty > 0 |
|
||||
| notifies | product_notify | COUNT of records where pn.pid = pid |
|
||||
| date_last_sold | product_last_sold.date_sold | Validated date, NULL if invalid |
|
||||
| image | N/A | Constructed from pid and image URL pattern |
|
||||
| image_175 | N/A | Constructed from pid and image URL pattern |
|
||||
| image_full | N/A | Constructed from pid and image URL pattern |
|
||||
| options | NULL | Not set |
|
||||
| tags | NULL | Not set |
|
||||
|
||||
**Notes:**
|
||||
- Replenishable calculation:
|
||||
```javascript
|
||||
CASE
|
||||
WHEN p.reorder < 0 THEN 0
|
||||
WHEN (
|
||||
(COALESCE(pls.date_sold, '0000-00-00') = '0000-00-00' OR pls.date_sold <= DATE_SUB(CURRENT_DATE, INTERVAL 5 YEAR))
|
||||
AND (p.datein = '0000-00-00 00:00:00' OR p.datein <= DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 5 YEAR))
|
||||
AND (p.date_refill = '0000-00-00 00:00:00' OR p.date_refill <= DATE_SUB(CURRENT_TIMESTAMP, INTERVAL 5 YEAR))
|
||||
) THEN 0
|
||||
ELSE 1
|
||||
END
|
||||
```
|
||||
|
||||
In business terms, a product is considered NOT replenishable only if:
|
||||
- It was manually flagged as not replenishable (negative reorder value)
|
||||
- OR it shows no activity across ALL metrics (no sales AND no receipts AND no refills in the past 5 years)
|
||||
- Image URLs are constructed using this pattern:
|
||||
```javascript
|
||||
const paddedPid = pid.toString().padStart(6, '0');
|
||||
const prefix = paddedPid.slice(0, 3);
|
||||
const basePath = `${imageUrlBase}${prefix}/${pid}`;
|
||||
return {
|
||||
image: `${basePath}-t-${iid}.jpg`,
|
||||
image_175: `${basePath}-175x175-${iid}.jpg`,
|
||||
image_full: `${basePath}-o-${iid}.jpg`
|
||||
};
|
||||
```
|
||||
|
||||
### Product Categories (Relationship)
|
||||
|
||||
| PostgreSQL Column | MySQL Source | Transformation |
|
||||
|-------------------|-----------------------------------|---------------------------------------------------------------|
|
||||
| pid | products.pid | Direct mapping |
|
||||
| cat_id | product_category_index.cat_id | Direct mapping, filtered by category types |
|
||||
|
||||
**Notes:**
|
||||
- Only categories of types 10, 20, 11, 21, 12, 13 are imported
|
||||
- Categories 16 and 17 are explicitly excluded
|
||||
|
||||
### Orders
|
||||
|
||||
| PostgreSQL Column | MySQL Source | Transformation |
|
||||
|-------------------|-----------------------------------|---------------------------------------------------------------|
|
||||
| order_number | order_items.order_id | Direct mapping |
|
||||
| pid | order_items.prod_pid | Direct mapping |
|
||||
| sku | order_items.prod_itemnumber | Fallback to 'NO-SKU' if empty |
|
||||
| date | _order.date_placed_onlydate | Via join to _order table |
|
||||
| price | order_items.prod_price | Direct mapping |
|
||||
| quantity | order_items.qty_ordered | Direct mapping |
|
||||
| discount | Multiple sources | Complex calculation (see notes) |
|
||||
| tax | order_tax_info_products.item_taxes_to_collect | Via latest order_tax_info record |
|
||||
| tax_included | N/A | Hard-coded false |
|
||||
| shipping | N/A | Hard-coded 0 |
|
||||
| customer | _order.order_cid | Direct mapping |
|
||||
| customer_name | users | CONCAT(users.firstname, ' ', users.lastname) |
|
||||
| status | _order.order_status | Direct mapping |
|
||||
| canceled | _order.date_cancelled | Boolean: true if date_cancelled is not '0000-00-00 00:00:00' |
|
||||
| costeach | order_costs | From latest record or fallback to price * 0.5 |
|
||||
|
||||
**Notes:**
|
||||
- Only orders with order_status >= 15 and with a valid date_placed are processed
|
||||
- For incremental imports, only orders modified since last sync are processed
|
||||
- Discount calculation combines three sources:
|
||||
1. Base discount: order_items.prod_price_reg - order_items.prod_price
|
||||
2. Promo discount: SUM of order_discount_items.amount
|
||||
3. Proportional order discount: Calculation based on order subtotal proportion
|
||||
```javascript
|
||||
(oi.base_discount +
|
||||
COALESCE(ot.promo_discount, 0) +
|
||||
CASE
|
||||
WHEN om.summary_discount > 0 AND om.summary_subtotal > 0 THEN
|
||||
ROUND((om.summary_discount * (oi.price * oi.quantity)) / NULLIF(om.summary_subtotal, 0), 2)
|
||||
ELSE 0
|
||||
END)::DECIMAL(10,2)
|
||||
```
|
||||
- Taxes are taken from the latest tax record for an order
|
||||
- Cost data is taken from the latest non-pending cost record
|
||||
|
||||
### Purchase Orders
|
||||
|
||||
| PostgreSQL Column | MySQL Source | Transformation |
|
||||
|-------------------|-----------------------------------|---------------------------------------------------------------|
|
||||
| po_id | po.po_id | Default 0 if NULL |
|
||||
| pid | po_products.pid | Direct mapping |
|
||||
| sku | products.itemnumber | Fallback to 'NO-SKU' if empty |
|
||||
| name | products.description | Fallback to 'Unknown Product' |
|
||||
| cost_price | po_products.cost_each | Direct mapping |
|
||||
| po_cost_price | po_products.cost_each | Duplicate of cost_price |
|
||||
| vendor | suppliers.companyname | Fallback to 'Unknown Vendor' if empty |
|
||||
| date | po.date_ordered | Fallback to po.date_created if NULL |
|
||||
| expected_date | po.date_estin | Direct mapping |
|
||||
| status | po.status | Default 1 if NULL |
|
||||
| notes | po.short_note | Fallback to po.notes if NULL |
|
||||
| ordered | po_products.qty_each | Direct mapping |
|
||||
| received | N/A | Hard-coded 0 |
|
||||
| receiving_status | N/A | Hard-coded 1 |
|
||||
|
||||
**Notes:**
|
||||
- Only POs created within last 1 year (incremental) or 5 years (full) are processed
|
||||
- For incremental imports, only POs modified since last sync are processed
|
||||
|
||||
### Metadata Tables
|
||||
|
||||
#### import_history
|
||||
|
||||
| PostgreSQL Column | Source | Notes |
|
||||
|-------------------|-----------------------------------|---------------------------------------------------------------|
|
||||
| id | Auto-increment | Primary key |
|
||||
| table_name | Code | 'all_tables' for overall import |
|
||||
| start_time | NOW() | Import start time |
|
||||
| end_time | NOW() | Import completion time |
|
||||
| duration_seconds | Calculation | Elapsed seconds |
|
||||
| is_incremental | INCREMENTAL_UPDATE | Flag from config |
|
||||
| records_added | Calculation | Sum from all imports |
|
||||
| records_updated | Calculation | Sum from all imports |
|
||||
| status | Code | 'running', 'completed', 'failed', or 'cancelled' |
|
||||
| error_message | Exception | Error message if failed |
|
||||
| additional_info | JSON | Configuration and results |
|
||||
|
||||
#### sync_status
|
||||
|
||||
| PostgreSQL Column | Source | Notes |
|
||||
|----------------------|--------------------------------|---------------------------------------------------------------|
|
||||
| table_name | Code | Name of imported table |
|
||||
| last_sync_timestamp | NOW() | Timestamp of successful sync |
|
||||
| last_sync_id | NULL | Not used currently |
|
||||
|
||||
## Special Calculations
|
||||
|
||||
### Date Validation
|
||||
|
||||
MySQL dates are validated before insertion into PostgreSQL:
|
||||
|
||||
```javascript
|
||||
function validateDate(mysqlDate) {
|
||||
if (!mysqlDate || mysqlDate === '0000-00-00' || mysqlDate === '0000-00-00 00:00:00') {
|
||||
return null;
|
||||
}
|
||||
// Check if the date is valid
|
||||
const date = new Date(mysqlDate);
|
||||
return isNaN(date.getTime()) ? null : mysqlDate;
|
||||
}
|
||||
```
|
||||
|
||||
### Retry Mechanism
|
||||
|
||||
Operations that might fail temporarily are retried with exponential backoff:
|
||||
|
||||
```javascript
|
||||
async function withRetry(operation, errorMessage) {
|
||||
let lastError;
|
||||
for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
console.error(`${errorMessage} (Attempt ${attempt}/${MAX_RETRIES}):`, error);
|
||||
if (attempt < MAX_RETRIES) {
|
||||
const backoffTime = RETRY_DELAY * Math.pow(2, attempt - 1);
|
||||
await new Promise(resolve => setTimeout(resolve, backoffTime));
|
||||
}
|
||||
}
|
||||
}
|
||||
throw lastError;
|
||||
}
|
||||
```
|
||||
|
||||
### Progress Tracking
|
||||
|
||||
Progress is tracked with estimated time remaining:
|
||||
|
||||
```javascript
|
||||
function estimateRemaining(startTime, current, total) {
|
||||
if (current === 0) return "Calculating...";
|
||||
const elapsedSeconds = (Date.now() - startTime) / 1000;
|
||||
const itemsPerSecond = current / elapsedSeconds;
|
||||
const remainingItems = total - current;
|
||||
const remainingSeconds = remainingItems / itemsPerSecond;
|
||||
return formatElapsedTime(remainingSeconds);
|
||||
}
|
||||
```
|
||||
|
||||
## Implementation Notes
|
||||
|
||||
### Transaction Management
|
||||
|
||||
All imports use transactions to ensure data consistency:
|
||||
|
||||
- **Categories**: Uses savepoints for each category type
|
||||
- **Products**: Uses a single transaction for the entire import
|
||||
- **Orders**: Uses a single transaction with temporary tables
|
||||
- **Purchase Orders**: Uses a single transaction with temporary tables
|
||||
|
||||
### Memory Usage Optimization
|
||||
|
||||
To minimize memory usage when processing large datasets:
|
||||
|
||||
1. Data is processed in batches (100-5000 records per batch)
|
||||
2. Temporary tables are used for intermediate data
|
||||
3. Some queries use cursors to avoid loading all results at once
|
||||
|
||||
### MySQL vs PostgreSQL Compatibility
|
||||
|
||||
The scripts handle differences between MySQL and PostgreSQL:
|
||||
|
||||
1. MySQL-specific syntax like `USE INDEX` is removed for PostgreSQL
|
||||
2. `GROUP_CONCAT` in MySQL becomes string operations in PostgreSQL
|
||||
3. Transaction syntax differences are abstracted in the connection wrapper
|
||||
4. PostgreSQL's `ON CONFLICT` replaces MySQL's `ON DUPLICATE KEY UPDATE`
|
||||
|
||||
### SSH Tunnel
|
||||
|
||||
Database connections go through an SSH tunnel for security:
|
||||
|
||||
```javascript
|
||||
ssh.forwardOut(
|
||||
"127.0.0.1",
|
||||
0,
|
||||
sshConfig.prodDbConfig.host,
|
||||
sshConfig.prodDbConfig.port,
|
||||
async (err, stream) => {
|
||||
if (err) reject(err);
|
||||
resolve({ ssh, stream });
|
||||
}
|
||||
);
|
||||
```
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,271 @@
|
||||
**Analysis of Potential Issues**
|
||||
|
||||
1. **Obsolete Functionality:**
|
||||
* **`config.js` Legacy Endpoints:** The endpoints `GET /config/`, `PUT /config/stock-thresholds/:id`, `PUT /config/lead-time-thresholds/:id`, `PUT /config/sales-velocity/:id`, `PUT /config/abc-classification/:id`, `PUT /config/safety-stock/:id`, and `PUT /config/turnover/:id` appear **highly likely to be obsolete**. They reference older, single-row config tables (`stock_thresholds`, etc.) while newer endpoints (`/config/global`, `/config/products`, `/config/vendors`) manage settings in more structured tables (`settings_global`, `settings_product`, `settings_vendor`). Unless specifically required for backward compatibility, these legacy endpoints should be removed to avoid confusion and potential data conflicts.
|
||||
* **`analytics.js` Forecast Endpoint (`GET /analytics/forecast`):** This endpoint uses **MySQL syntax** (`DATEDIFF`, `DATE_FORMAT`, `JSON_OBJECT`, `?` placeholders) but seems intended to run within the analytics module which otherwise uses PostgreSQL (`req.app.locals.pool`, `date_trunc`, `::text`, `$1` placeholders). This endpoint is likely **obsolete or misplaced** and will not function correctly against the PostgreSQL database.
|
||||
* **`csv.js` Redundant Actions:**
|
||||
* `POST /csv/update` seems redundant with `POST /csv/full-update`. The latter uses the `runScript` helper and dedicated state (`activeFullUpdate`), appearing more robust. `/csv/update` might be older or incomplete.
|
||||
* `POST /csv/reset` seems redundant with `POST /csv/full-reset`. Similar reasoning applies; `/csv/full-reset` appears preferred.
|
||||
* **`products.js` Import Endpoint (`POST /products/import`):** This is **dangerous duplication**. The `/csv` module handles imports (`/csv/import`, `/csv/import-from-prod`) with locking (`activeImport`) to prevent concurrent operations. This endpoint lacks such locking and could corrupt data if run simultaneously with other CSV/reset operations. It should likely be removed.
|
||||
* **`products.js` Metrics Endpoint (`GET /products/:id/metrics`):** This is redundant. The `/metrics/:pid` endpoint provides the same, possibly more comprehensive, data directly from the `product_metrics` table. Clients should use `/metrics/:pid` instead.
|
||||
|
||||
2. **Overlap or Inappropriate Duplication of Effort:**
|
||||
* **AI Prompt Getters:** `GET /ai-prompts/type/general` and `GET /ai-prompts/type/system` could potentially be handled by adding a query parameter filter to `GET /ai-prompts/` (e.g., `GET /ai-prompts?prompt_type=general`). However, dedicated endpoints for single, specific items can sometimes be simpler. This is more of a design choice than a major issue.
|
||||
* **Vendor Performance/Metrics:** There are multiple ways to get vendor performance data:
|
||||
* `GET /analytics/vendors` (uses `vendor_metrics`)
|
||||
* `GET /dashboard/vendor/performance` (uses `purchase_orders`)
|
||||
* `GET /purchase-orders/vendor-metrics` (uses `purchase_orders`)
|
||||
* `GET /vendors-aggregate/` (uses `vendor_metrics`, augmented with `purchase_orders`)
|
||||
This suggests significant overlap. The `/vendors-aggregate` endpoint seems the most comprehensive, combining pre-aggregated data with some real-time info. The others, especially `/dashboard/vendor/performance` and `/purchase-orders/vendor-metrics` which calculate directly from `purchase_orders`, might be redundant or less performant.
|
||||
* **Product Listing:**
|
||||
* `GET /products/` lists products joining `products`, `product_metrics`, and `categories`.
|
||||
* `GET /metrics/` lists products primarily from `product_metrics`.
|
||||
They offer similar filtering/sorting. If `product_metrics` contains all necessary display fields, `GET /products/` might be partly redundant for simple listing views, although it does provide aggregated category names. Evaluate if both full list endpoints are necessary.
|
||||
* **Image Uploads/Management:** Image handling is split:
|
||||
* `products-import.js`: Uploads temporary images for product import to `/uploads/products/`, schedules deletion.
|
||||
* `reusable-images.js`: Uploads persistent images to `/uploads/reusable/`, stores metadata in DB.
|
||||
* `products-import.js` has `/check-file` and `/list-uploads` that can see *both* directories, while `reusable-images.js` has a `/check-file` that only sees its own. This separation could be confusing. Clarify the purpose and lifecycle of images in each directory.
|
||||
* **Background Task Management (`csv.js`):** The use of `activeImport` for multiple unrelated tasks (import, reset, metrics calc) prevents concurrency, which might be too restrictive. The cancellation logic (`/cancel`) only targets `full-update`/`full-reset`, not tasks locked by `activeImport`. This needs unification.
|
||||
* **Analytics/Dashboard Base Table Queries:** Several endpoints in `analytics.js` (`/pricing`, `/categories`) and `dashboard.js` (`/best-sellers`, `/sales/metrics`, `/trending/products`, `/key-metrics`, `/inventory-health`, `/sales-overview`) query base tables (`orders`, `products`, `purchase_orders`) directly, while many others leverage pre-aggregated `_metrics` tables. This inconsistency can lead to performance differences and suggests potential for optimization by using aggregates where possible.
|
||||
|
||||
3. **Obvious Mistakes / Data Issues:**
|
||||
* **AI Prompt Fetching:** `GET /ai-prompts/company/:companyId`, `/type/general`, `/type/system` return `result.rows[0]`. This assumes uniqueness. If the underlying DB constraints (`unique_company_prompt`, etc.) fail or aren't present, this could silently hide data if multiple rows match. The use of unique constraint handling in POST/PUT suggests this is likely intended and safe *if* DB constraints are solid.
|
||||
* **Mixed Databases & SSH Tunnels:** The heavy reliance in `ai_validation.js` and `products-import.js` on connecting to a production MySQL DB via SSH tunnel while also using a local PostgreSQL DB adds significant architectural complexity.
|
||||
* **Inefficiency:** In `ai_validation.js` (`generateDebugResponse`), an SSH tunnel and MySQL connection (`promptTunnel`, `promptConnection`) are established but seem unused when fetching prompts (which correctly come from the PG pool `res.app.locals.pool`). This is wasted effort.
|
||||
* **Improvement:** The `getDbConnection` function in `products-import.js` implements caching/pooling for the SSH/MySQL connection – this is much better and should ideally be used consistently wherever the production DB is accessed (e.g., in `ai_validation.js`).
|
||||
* **`products.js` Brand Filtering:** `GET /products/brands` filters brands based on having associated purchase orders with a cost >= 500. This seems arbitrary for a general list of brands and might return incomplete results depending on the use case.
|
||||
* **Type Handling:** Ensure `parseValue` handles all required types and edge cases correctly, especially for filtering complex queries in `*-aggregate` and `metrics` routes. Explicit type casting in SQL (`::numeric`, `::text`, etc.) is generally good practice in PostgreSQL.
|
||||
* **Dummy Data:** Several `dashboard.js` endpoints return hardcoded dummy data on errors or when no data is found. While this prevents UI crashes, it can mask real issues. Ensure logging is robust when fallbacks are used.
|
||||
|
||||
**Summary of Endpoints**
|
||||
|
||||
Here's a summary of the available endpoints, grouped by their likely file/module:
|
||||
|
||||
**1. AI Prompts (`ai_prompts.js`)**
|
||||
* `GET /`: Get all AI prompts.
|
||||
* `GET /:id`: Get a specific AI prompt by its ID.
|
||||
* `GET /company/:companyId`: Get the AI prompt for a specific company (expects one). **(Deprecated)**
|
||||
* `GET /type/general`: Get the general AI prompt (expects one). **(Deprecated)**
|
||||
* `GET /type/system`: Get the system AI prompt (expects one). **(Deprecated)**
|
||||
* `GET /by-type`: Get AI prompt by type (general, system, company_specific) with optional company parameter. **(New Consolidated Endpoint)**
|
||||
* `POST /`: Create a new AI prompt.
|
||||
* `PUT /:id`: Update an existing AI prompt.
|
||||
* `DELETE /:id`: Delete an AI prompt.
|
||||
|
||||
**2. AI Validation (`ai_validation.js`)**
|
||||
* `POST /debug`: Generate and view the structure of prompts and taxonomy data (for debugging, doesn't call OpenAI). Connects to Prod MySQL (taxonomy) and Local PG (prompts, performance).
|
||||
* `POST /validate`: Validate product data using OpenAI. Connects to Prod MySQL (taxonomy) and Local PG (prompts, performance).
|
||||
* `GET /test-taxonomy`: Test endpoint to query sample taxonomy data from Prod MySQL.
|
||||
|
||||
**3. Analytics (`analytics.js`)**
|
||||
* `GET /stats`: Get overall business statistics from metrics tables.
|
||||
* `GET /profit`: Get profit analysis data (by category, over time, top products) from metrics tables.
|
||||
* `GET /vendors`: Get vendor performance analysis from `vendor_metrics`.
|
||||
* `GET /stock`: Get stock analysis data (turnover, levels, critical items) from metrics tables.
|
||||
* `GET /pricing`: Get pricing analysis (price points, elasticity, recommendations) - **uses `orders` table**.
|
||||
* `GET /categories`: Get category performance analysis (revenue, profit, growth, distribution, trends) - **uses `orders` and `products` tables**.
|
||||
* `GET /forecast`: (**Likely Obsolete/Broken**) Attempts to get forecast data using MySQL syntax.
|
||||
|
||||
**4. Brands Aggregate (`brands-aggregate.js`)**
|
||||
* `GET /filter-options`: Get distinct brand names and statuses for UI filters (from `brand_metrics`).
|
||||
* `GET /stats`: Get overall statistics related to brands (from `brand_metrics`).
|
||||
* `GET /`: List brands with aggregated metrics, supporting filtering, sorting, pagination (from `brand_metrics`).
|
||||
|
||||
**5. Categories Aggregate (`categories-aggregate.js`)**
|
||||
* `GET /filter-options`: Get distinct category types, statuses, and counts for UI filters (from `category_metrics` & `categories`).
|
||||
* `GET /stats`: Get overall statistics related to categories (from `category_metrics` & `categories`).
|
||||
* `GET /`: List categories with aggregated metrics, supporting filtering, sorting (incl. hierarchy), pagination (from `category_metrics` & `categories`).
|
||||
|
||||
**6. Configuration (`config.js`)**
|
||||
* **(New)** `GET /global`: Get all global settings.
|
||||
* **(New)** `PUT /global`: Update global settings.
|
||||
* **(New)** `GET /products`: List product-specific settings with pagination/search.
|
||||
* **(New)** `PUT /products/:pid`: Update/Create product-specific settings.
|
||||
* **(New)** `POST /products/:pid/reset`: Reset product settings to defaults.
|
||||
* **(New)** `GET /vendors`: List vendor-specific settings with pagination/search.
|
||||
* **(New)** `PUT /vendors/:vendor`: Update/Create vendor-specific settings.
|
||||
* **(New)** `POST /vendors/:vendor/reset`: Reset vendor settings to defaults.
|
||||
* **(Legacy/Obsolete)** `GET /`: Get all config from old single-row tables.
|
||||
* **(Legacy/Obsolete)** `PUT /stock-thresholds/:id`: Update old stock thresholds.
|
||||
* **(Legacy/Obsolete)** `PUT /lead-time-thresholds/:id`: Update old lead time thresholds.
|
||||
* **(Legacy/Obsolete)** `PUT /sales-velocity/:id`: Update old sales velocity config.
|
||||
* **(Legacy/Obsolete)** `PUT /abc-classification/:id`: Update old ABC config.
|
||||
* **(Legacy/Obsolete)** `PUT /safety-stock/:id`: Update old safety stock config.
|
||||
* **(Legacy/Obsolete)** `PUT /turnover/:id`: Update old turnover config.
|
||||
|
||||
**7. CSV Operations & Background Tasks (`csv.js`)**
|
||||
* `GET /:type/progress`: SSE endpoint for full update/reset progress.
|
||||
* `GET /test`: Simple test endpoint.
|
||||
* `GET /status`: Check status of the generic background task lock (`activeImport`).
|
||||
* `GET /calculate-metrics/status`: Check status of metrics calculation.
|
||||
* `GET /history/import`: Get recent import history.
|
||||
* `GET /history/calculate`: Get recent metrics calculation history.
|
||||
* `GET /status/modules`: Get last calculation time per module.
|
||||
* `GET /status/tables`: Get last sync time per table.
|
||||
* `GET /status/table-counts`: Get record counts for key tables.
|
||||
* `POST /update`: (**Potentially Obsolete**) Trigger `update-csv.js` script.
|
||||
* `POST /import`: Trigger `import-csv.js` script.
|
||||
* `POST /cancel`: Cancel `/full-update` or `/full-reset` task.
|
||||
* `POST /reset`: (**Potentially Obsolete**) Trigger `reset-db.js` script.
|
||||
* `POST /reset-metrics`: Trigger `reset-metrics.js` script.
|
||||
* `POST /calculate-metrics`: Trigger `calculate-metrics.js` script.
|
||||
* `POST /import-from-prod`: Trigger `import-from-prod.js` script.
|
||||
* `POST /full-update`: Trigger `full-update.js` script (preferred update).
|
||||
* `POST /full-reset`: Trigger `full-reset.js` script (preferred reset).
|
||||
|
||||
**8. Dashboard (`dashboard.js`)**
|
||||
* `GET /stock/metrics`: Get dashboard stock summary metrics & brand breakdown.
|
||||
* `GET /purchase/metrics`: Get dashboard purchase order summary metrics & vendor breakdown.
|
||||
* `GET /replenishment/metrics`: Get dashboard replenishment summary & top variants.
|
||||
* `GET /forecast/metrics`: Get dashboard forecast summary, daily, and category breakdown.
|
||||
* `GET /overstock/metrics`: Get dashboard overstock summary & category breakdown.
|
||||
* `GET /overstock/products`: Get list of top overstocked products.
|
||||
* `GET /best-sellers`: Get dashboard best-selling products, brands, categories - **uses `orders`, `products`**.
|
||||
* `GET /sales/metrics`: Get dashboard sales summary for a period - **uses `orders`**.
|
||||
* `GET /low-stock/products`: Get list of top low stock/critical products.
|
||||
* `GET /trending/products`: Get list of trending products - **uses `orders`, `products`**.
|
||||
* `GET /vendor/performance`: Get dashboard vendor performance details - **uses `purchase_orders`**.
|
||||
* `GET /key-metrics`: Get dashboard summary KPIs - **uses multiple base tables**.
|
||||
* `GET /inventory-health`: Get dashboard inventory health overview - **uses `products`, `product_metrics`**.
|
||||
* `GET /replenish/products`: Get list of products needing replenishment (overlaps `/low-stock/products`).
|
||||
* `GET /sales-overview`: Get daily sales totals for chart - **uses `orders`**.
|
||||
|
||||
**9. Product Import Utilities (`products-import.js`)**
|
||||
* `POST /upload-image`: Upload temporary product image, schedule deletion.
|
||||
* `DELETE /delete-image`: Delete temporary product image.
|
||||
* `GET /field-options`: Get dropdown options for product fields from Prod MySQL (cached).
|
||||
* `GET /product-lines/:companyId`: Get product lines for a company from Prod MySQL (cached).
|
||||
* `GET /sublines/:lineId`: Get sublines for a line from Prod MySQL (cached).
|
||||
* `GET /check-file/:filename`: Check existence/permissions of uploaded file (temp or reusable).
|
||||
* `GET /list-uploads`: List files in upload directories.
|
||||
* `GET /search-products`: Search products in Prod MySQL DB.
|
||||
* `GET /check-upc-and-generate-sku`: Check UPC existence and generate SKU suggestion based on Prod MySQL data.
|
||||
* `GET /product-categories/:pid`: Get assigned categories for a product from Prod MySQL.
|
||||
|
||||
**10. Product Metrics (`product-metrics.js`)**
|
||||
* `GET /filter-options`: Get distinct filter values (vendor, brand, abcClass) from `product_metrics`.
|
||||
* `GET /`: List detailed product metrics with filtering, sorting, pagination (primary data access).
|
||||
* `GET /:pid`: Get full metrics record for a single product.
|
||||
|
||||
**11. Orders (`orders.js`)**
|
||||
* `GET /`: List orders with summary info, filtering, sorting, pagination, and stats.
|
||||
* `GET /:orderNumber`: Get details for a single order, including items.
|
||||
|
||||
**12. Products (`products.js`)**
|
||||
* `GET /brands`: Get distinct brands (filtered by PO value).
|
||||
* `GET /`: List products with core data + metrics, filtering, sorting, pagination.
|
||||
* `GET /trending`: Get trending products based on `product_metrics`.
|
||||
* `GET /:id`: Get details for a single product (core data + metrics).
|
||||
* `POST /import`: (**Likely Obsolete/Dangerous**) Import products from CSV.
|
||||
* `PUT /:id`: Update core product data.
|
||||
* `GET /:id/metrics`: (**Redundant**) Get metrics for a single product.
|
||||
* `GET /:id/time-series`: Get sales/PO history for a single product.
|
||||
|
||||
**13. Purchase Orders (`purchase-orders.js`)**
|
||||
* `GET /`: List purchase orders with summary info, filtering, sorting, pagination, and summary stats.
|
||||
* `GET /vendor-metrics`: Calculate vendor performance metrics from `purchase_orders`.
|
||||
* `GET /cost-analysis`: Calculate cost analysis by category from `purchase_orders`.
|
||||
* `GET /receiving-status`: Get summary counts based on PO receiving status.
|
||||
* `GET /order-vs-received`: List product ordered vs. received quantities.
|
||||
|
||||
**14. Reusable Images (`reusable-images.js`)**
|
||||
* `GET /`: List all reusable images.
|
||||
* `GET /by-company/:companyId`: List global and company-specific images.
|
||||
* `GET /global`: List only global images.
|
||||
* `GET /:id`: Get a single reusable image record.
|
||||
* `POST /upload`: Upload a new reusable image and create DB record.
|
||||
* `PUT /:id`: Update reusable image metadata (name, global, company).
|
||||
* `DELETE /:id`: Delete reusable image record and file.
|
||||
* `GET /check-file/:filename`: Check existence/permissions of a reusable image file.
|
||||
|
||||
**15. Templates (`templates.js`)**
|
||||
* `GET /`: List all product data templates.
|
||||
* `GET /:company/:productType`: Get a specific template.
|
||||
* `POST /`: Create a new template.
|
||||
* `PUT /:id`: Update an existing template.
|
||||
* `DELETE /:id`: Delete a template.
|
||||
|
||||
**16. Vendors Aggregate (`vendors-aggregate.js`)**
|
||||
* `GET /filter-options`: Get distinct vendor names and statuses for UI filters (from `vendor_metrics`).
|
||||
* `GET /stats`: Get overall statistics related to vendors (from `vendor_metrics` & `purchase_orders`).
|
||||
* `GET /`: List vendors with aggregated metrics, supporting filtering, sorting, pagination (from `vendor_metrics` & `purchase_orders`).
|
||||
|
||||
**Recommendations:**
|
||||
|
||||
1. **Address Obsolete Endpoints:** Prioritize removing or confirming the necessity of the endpoints marked as obsolete/redundant (legacy config, `/analytics/forecast`, `/csv/update`, `/csv/reset`, `/products/import`, `/products/:id/metrics`).
|
||||
2. **Consolidate Overlapping Functionality:** Review the multiple vendor performance and product listing endpoints. Decide on the primary method (e.g., using aggregate tables via `/vendors-aggregate` and `/metrics`) and refactor or remove the others. Clarify the image upload strategies.
|
||||
3. **Standardize Data Access:** Decide whether `dashboard` and `analytics` endpoints should primarily use aggregate tables (like `/metrics`, `/brands-aggregate`, etc.) or if direct access to base tables is sometimes necessary. Aim for consistency and document the reasoning. Optimize queries hitting base tables if they must remain.
|
||||
4. **Improve Background Task Management:** Refactor `csv.js` to use a unified locking mechanism (maybe separate locks per task type?) and a consistent cancellation strategy for all spawned/managed processes. Clarify the purpose of `update` vs `full-update` and `reset` vs `full-reset`.
|
||||
5. **Optimize DB Connections:** Ensure the `getDbConnection` pooling/caching helper from `products-import.js` is used *consistently* across all modules interacting with the production MySQL database (especially `ai_validation.js`). Remove unnecessary tunnel creations.
|
||||
6. **Review Data Integrity:** Double-check the assumptions made (e.g., uniqueness of AI prompts) and ensure database constraints enforce them. Review the `GET /products/brands` filtering logic.
|
||||
|
||||
## Changes Made
|
||||
|
||||
1. **Removed Obsolete Legacy Endpoints in `config.js`**:
|
||||
- Removed `GET /config/` endpoint
|
||||
- Removed `PUT /config/stock-thresholds/:id` endpoint
|
||||
- Removed `PUT /config/lead-time-thresholds/:id` endpoint
|
||||
- Removed `PUT /config/sales-velocity/:id` endpoint
|
||||
- Removed `PUT /config/abc-classification/:id` endpoint
|
||||
- Removed `PUT /config/safety-stock/:id` endpoint
|
||||
- Removed `PUT /config/turnover/:id` endpoint
|
||||
|
||||
These endpoints were obsolete as they referenced older, single-row config tables that have been replaced by newer endpoints using the structured tables `settings_global`, `settings_product`, and `settings_vendor`.
|
||||
|
||||
2. **Removed MySQL Syntax `/forecast` Endpoint in `analytics.js`**:
|
||||
- Removed `GET /analytics/forecast` endpoint that was using MySQL-specific syntax incompatible with the PostgreSQL database used elsewhere in the application.
|
||||
|
||||
3. **Renamed and Removed Redundant Endpoints**:
|
||||
- Renamed `csv.js` to `data-management.js` while maintaining the same `/csv/*` endpoint paths for consistency
|
||||
- Removed deprecated `/csv/update` endpoint (now fully replaced by `/csv/full-update`)
|
||||
- Removed deprecated `/csv/reset` endpoint (now fully replaced by `/csv/full-reset`)
|
||||
- Removed deprecated `/products/import` endpoint (now handled by `/csv/import`)
|
||||
- Removed deprecated `/products/:id/metrics` endpoint (now handled by `/metrics/:pid`)
|
||||
|
||||
4. **Fixed Data Integrity Issues**:
|
||||
- Improved `GET /products/brands` endpoint by removing the arbitrary filtering logic that was only showing brands with purchase orders that had a total cost of at least $500
|
||||
- The updated endpoint now returns all distinct brands from visible products, providing more complete data
|
||||
|
||||
5. **Optimized Database Connections**:
|
||||
- Created a new `dbConnection.js` utility file that encapsulates the optimized database connection management logic
|
||||
- Improved the `ai-validation.js` file to use this shared connection management, eliminating unnecessary repeated tunnel creation
|
||||
- Added proper connection pooling with timeout-based connection reuse, reducing the overhead of repeatedly creating SSH tunnels
|
||||
- Added query result caching for frequently accessed data to improve performance
|
||||
|
||||
These changes improve maintainability by removing duplicate code, enhance consistency by standardizing on the newer endpoint patterns, and optimize performance by reducing redundant database connections.
|
||||
|
||||
## Additional Improvements
|
||||
|
||||
1. **Further Database Connection Optimizations**:
|
||||
- Extended the use of the optimized database connection utility to additional endpoints in `ai-validation.js`
|
||||
- Updated the `/validate` endpoint and `/test-taxonomy` endpoint to use `getDbConnection`
|
||||
- Ensured consistent connection management across all routes that access the production database
|
||||
|
||||
2. **AI Prompts Data Integrity Verification**:
|
||||
- Confirmed proper uniqueness constraints are in place in the database schema for AI prompts
|
||||
- The schema includes:
|
||||
- `unique_company_prompt` constraint ensuring only one prompt per company
|
||||
- `idx_unique_general_prompt` index ensuring only one general prompt in the system
|
||||
- `idx_unique_system_prompt` index ensuring only one system prompt in the system
|
||||
- Endpoint handlers properly handle uniqueness constraint violations with appropriate 409 Conflict responses
|
||||
- Validation ensures company-specific prompts have company IDs, while general/system prompts do not
|
||||
|
||||
3. **AI Prompts Endpoint Consolidation**:
|
||||
- Added a new consolidated `/by-type` endpoint that handles all types of prompts (general, system, company_specific)
|
||||
- Marked the existing separate endpoints as deprecated with console warnings
|
||||
- Maintained backward compatibility while providing a cleaner API moving forward
|
||||
|
||||
## Completed Items
|
||||
|
||||
✅ Removed obsolete legacy endpoints in `config.js`
|
||||
✅ Removed MySQL syntax `/forecast` endpoint in `analytics.js`
|
||||
✅ Fixed `GET /products/brands` endpoint filtering logic
|
||||
✅ Created reusable database connection utility (`dbConnection.js`)
|
||||
✅ Optimized database connections in `ai-validation.js`
|
||||
✅ Verified data integrity in AI prompts handling
|
||||
✅ Consolidated AI prompts endpoints with a unified `/by-type` endpoint
|
||||
|
||||
## Remaining Items
|
||||
|
||||
- Consider adding additional error handling and logging for database connections
|
||||
- Perform load testing on the optimized database connections to ensure they handle high traffic properly
|
||||
@@ -0,0 +1,23 @@
|
||||
This portion of the application is going to be a read only chat archive. It will pull data from a rocketchat export converted to postgresql. This is a separate database than the rest of the inventory application uses, but it will still use users and permissions from the inventory database. Both databases are on the same postgres instance.
|
||||
|
||||
For now, let's add a select to the top of the page that allows me to "view as" any of the users in the rocketchat database. We'll connect this to the authorization in the main application later.
|
||||
|
||||
The db connection info is stored in the .env file in the inventory-server root. It contains these variables
|
||||
DB_HOST=localhost
|
||||
DB_USER=rocketchat_user
|
||||
DB_PASSWORD=password
|
||||
DB_NAME=rocketchat_converted
|
||||
DB_PORT=5432
|
||||
|
||||
Not all of the information in this database is relevant as it's a direct export from another app with more features. You can use the query tool to examine the structure and data available.
|
||||
|
||||
Server-side files should use similar conventions and the same technologies as the inventory-server (inventor-server root) and auth-server (inventory-server/auth). I will provide my current pm2 ecosystem file upon request for you to add the configuration for the new "chat-server". I use Caddy on the server and can provide my caddyfile to assist with configuring the api routes. All configuration and routes for the chat-server should go in the inventory-server/chat folder or subfolders you create.
|
||||
|
||||
The folder you see as inventory-server is actually a direct mount of the /var/www/inventory folder on the server. You can read and write files from there like usual, but any terminal commands for the server I will have to run myself.
|
||||
|
||||
The "Chat" page should be added to the main application sidebar and a similar page to the others should be created in inventory/src/pages. All other frontend pages should go in inventory/src/components/chat.
|
||||
|
||||
The application uses shadcn components and those should be used for all ui elements where possible (located in inventory/src/components/ui). The UI should match existing pages and components.
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
Okay, I understand completely now. The core issue is that the previous approaches tried too hard to reconcile every receipt back to a specific PO line within the `purchase_orders` table structure, which doesn't reflect the reality where receipts can be independent events. Your downstream scripts, especially `daily_snapshots` and `product_metrics`, rely on having a complete picture of *all* receivings.
|
||||
|
||||
Let's pivot to a model that respects both distinct data streams: **Orders (Intent)** and **Receivings (Actuals)**.
|
||||
|
||||
**Proposed Solution: Separate `purchase_orders` and `receivings` Tables**
|
||||
|
||||
This is the cleanest way to model the reality you've described.
|
||||
|
||||
1. **`purchase_orders` Table:**
|
||||
* **Purpose:** Tracks the status and details of purchase *orders* placed. Represents the *intent* to receive goods.
|
||||
* **Key Columns:** `po_id`, `pid`, `ordered` (quantity ordered), `po_cost_price`, `date` (order/created date), `expected_date`, `status` (PO lifecycle: 'ordered', 'canceled', 'done'), `vendor`, `notes`, etc.
|
||||
* **Crucially:** This table *does not* need a `received` column or a `receiving_history` column derived from complex allocations. It focuses solely on the PO itself.
|
||||
|
||||
2. **`receivings` Table (New or Refined):**
|
||||
* **Purpose:** Tracks every single line item received, regardless of whether it was linked to a PO during the receiving process. Represents the *actual* goods that arrived.
|
||||
* **Key Columns:**
|
||||
* `receiving_id` (Identifier for the overall receiving document/batch)
|
||||
* `pid` (Product ID received)
|
||||
* `received_qty` (Quantity received for this specific line)
|
||||
* `cost_each` (Actual cost paid for this item on this receiving)
|
||||
* `received_date` (Actual date the item was received)
|
||||
* `received_by` (Employee ID/Name)
|
||||
* `source_po_id` (The `po_id` entered on the receiving screen, *nullable*. Stores the original link attempt, even if it was wrong or missing)
|
||||
* `source_receiving_status` (The status from the source `receivings` table: 'partial_received', 'full_received', 'paid', 'canceled')
|
||||
|
||||
**How the Import Script Changes:**
|
||||
|
||||
1. **Fetch POs:** Fetch data from `po` and `po_products`.
|
||||
2. **Populate `purchase_orders`:**
|
||||
* Insert/Update rows into `purchase_orders` based directly on the fetched PO data.
|
||||
* Set `po_id`, `pid`, `ordered`, `po_cost_price`, `date` (`COALESCE(date_ordered, date_created)`), `expected_date`.
|
||||
* Set `status` by mapping the source `po.status` code directly ('ordered', 'canceled', 'done', etc.).
|
||||
* **No complex allocation needed here.**
|
||||
3. **Fetch Receivings:** Fetch data from `receivings` and `receivings_products`.
|
||||
4. **Populate `receivings`:**
|
||||
* For *every* line item fetched from `receivings_products`:
|
||||
* Perform necessary data validation (dates, numbers).
|
||||
* Insert a new row into `receivings` with all the relevant details (`receiving_id`, `pid`, `received_qty`, `cost_each`, `received_date`, `received_by`, `source_po_id`, `source_receiving_status`).
|
||||
* Use `ON CONFLICT (receiving_id, pid)` (or similar unique key based on your source data) `DO UPDATE SET ...` for incremental updates if necessary, or simply delete/re-insert based on `receiving_id` for simplicity if performance allows.
|
||||
|
||||
**Impact on Downstream Scripts (and how to adapt):**
|
||||
|
||||
* **Initial Query (Active POs):**
|
||||
* `SELECT ... FROM purchase_orders po WHERE po.status NOT IN ('canceled', 'done', 'paid_equivalent_status?') AND po.date >= ...`
|
||||
* `active_pos`: `COUNT(DISTINCT po.po_id)` based on the filtered POs.
|
||||
* `overdue_pos`: Add `AND po.expected_date < CURRENT_DATE`.
|
||||
* `total_units`: `SUM(po.ordered)`. Represents total units *ordered* on active POs.
|
||||
* `total_cost`: `SUM(po.ordered * po.po_cost_price)`. Cost of units *ordered*.
|
||||
* `total_retail`: `SUM(po.ordered * pm.current_price)`. Retail value of units *ordered*.
|
||||
* **Result:** This query now cleanly reports on the status of *orders* placed, which seems closer to its original intent. The filter `po.receiving_status NOT IN ('partial_received', 'full_received', 'paid')` is replaced by `po.status NOT IN ('canceled', 'done', 'paid_equivalent?')`. The 90% received check is removed as `received` is not reliably tracked *on the PO* anymore.
|
||||
|
||||
* **`daily_product_snapshots`:**
|
||||
* **`SalesData` CTE:** No change needed.
|
||||
* **`ReceivingData` CTE:** **Must be changed.** Query the **`receivings`** table instead of `purchase_orders`.
|
||||
```sql
|
||||
ReceivingData AS (
|
||||
SELECT
|
||||
rl.pid,
|
||||
COUNT(DISTINCT rl.receiving_id) as receiving_doc_count,
|
||||
SUM(rl.received_qty) AS units_received,
|
||||
SUM(rl.received_qty * rl.cost_each) AS cost_received
|
||||
FROM public.receivings rl
|
||||
WHERE rl.received_date::date = _date
|
||||
-- Optional: Filter out canceled receivings if needed
|
||||
-- AND rl.source_receiving_status <> 'canceled'
|
||||
GROUP BY rl.pid
|
||||
),
|
||||
```
|
||||
* **Result:** This now accurately reflects *all* units received on a given day from the definitive source.
|
||||
|
||||
* **`update_product_metrics`:**
|
||||
* **`CurrentInfo` CTE:** No change needed (pulls from `products`).
|
||||
* **`OnOrderInfo` CTE:** Needs re-evaluation. How do you want to define "On Order"?
|
||||
* **Option A (Strict PO View):** `SUM(po.ordered)` from `purchase_orders po WHERE po.status NOT IN ('canceled', 'done', 'paid_equivalent?')`. This is quantity on *open orders*, ignoring fulfillment state. Simple, but might overestimate if items arrived unlinked.
|
||||
* **Option B (Approximate Fulfillment):** `SUM(po.ordered)` from open POs MINUS `SUM(rl.received_qty)` from `receivings rl` where `rl.source_po_id = po.po_id` (summing only directly linked receivings). Better, but still misses fulfillment via unlinked receivings.
|
||||
* **Option C (Heuristic):** `SUM(po.ordered)` from open POs MINUS `SUM(rl.received_qty)` from `receivings rl` where `rl.pid = po.pid` and `rl.received_date >= po.date`. This *tries* to account for unlinked receivings but is imprecise.
|
||||
* **Recommendation:** Start with **Option A** for simplicity, clearly labeling it "Quantity on Open POs". You might need a separate process or metric for a more nuanced view of expected vs. actual pipeline.
|
||||
```sql
|
||||
-- Example for Option A
|
||||
OnOrderInfo AS (
|
||||
SELECT
|
||||
pid,
|
||||
SUM(ordered) AS on_order_qty, -- Total qty on open POs
|
||||
SUM(ordered * po_cost_price) AS on_order_cost -- Cost of qty on open POs
|
||||
FROM public.purchase_orders
|
||||
WHERE status NOT IN ('canceled', 'done', 'paid_equivalent?') -- Define your open statuses
|
||||
GROUP BY pid
|
||||
),
|
||||
```
|
||||
* **`HistoricalDates` CTE:**
|
||||
* `date_first_sold`, `max_order_date`: No change (queries `orders`).
|
||||
* `date_first_received_calc`, `date_last_received_calc`: **Must be changed.** Query `MIN(rl.received_date)` and `MAX(rl.received_date)` from the **`receivings`** table grouped by `pid`.
|
||||
* **`SnapshotAggregates` CTE:**
|
||||
* `received_qty_30d`, `received_cost_30d`: These are calculated from `daily_product_snapshots`, which are now correctly sourced from `receivings`, so this part is fine.
|
||||
* **Forecasting Calculations:** Will use the chosen definition of `on_order_qty`. Be aware of the implications of Option A (potentially inflated if unlinked receivings fulfill orders).
|
||||
* **Result:** Metrics are calculated based on distinct order data and complete receiving data. The definition of "on order" needs careful consideration.
|
||||
|
||||
**Summary of this Approach:**
|
||||
|
||||
* **Pros:**
|
||||
* Accurately models distinct order and receiving events.
|
||||
* Provides a definitive source (`receivings`) for all received inventory.
|
||||
* Simplifies the `purchase_orders` table and its import logic.
|
||||
* Avoids complex/potentially inaccurate allocation logic for unlinked receivings within the main tables.
|
||||
* Avoids synthetic records.
|
||||
* Fixes downstream reporting (`daily_snapshots` receiving data).
|
||||
* **Cons:**
|
||||
* Requires creating/managing the `receivings` table.
|
||||
* Requires modifying downstream queries (`ReceivingData`, `OnOrderInfo`, `HistoricalDates`).
|
||||
* Calculating a precise "net quantity still expected to arrive" (true on-order minus all relevant fulfillment) becomes more complex and may require specific business rules or heuristics outside the basic table structure if Option A for `OnOrderInfo` isn't sufficient.
|
||||
|
||||
This two-table approach (`purchase_orders` + `receivings`) seems the most robust and accurate way to handle your requirement for complete receiving records independent of potentially flawed PO linking. It directly addresses the shortcomings of the previous attempts.
|
||||
@@ -0,0 +1,239 @@
|
||||
# Validation Display Issue Implementation
|
||||
|
||||
## Issue Being Addressed
|
||||
|
||||
**Validation Display Issue**: Validation isn't happening beyond checking if a cell is required or not. All validation rules defined in import.tsx need to be respected.
|
||||
* Required fields correctly show a red border when empty (✅ ALREADY WORKING)
|
||||
* Non-empty fields with validation errors (regex, unique, etc.) should show a red border AND an alert circle icon with tooltip explaining the error (❌ NOT WORKING)
|
||||
|
||||
## Implementation Attempts
|
||||
|
||||
!!!!**NOTE** All previous attempts have been reverted and are no longer part of the code, please take this into account when trying a new solution. !!!!
|
||||
|
||||
### Attempt 1: Fix Validation Display Logic
|
||||
|
||||
**Approach**: Modified `processErrors` function to separate required errors from validation errors and show alert icons only for non-empty fields with validation errors.
|
||||
|
||||
**Changes Made**:
|
||||
```typescript
|
||||
function processErrors(value: any, errors: ErrorObject[]) {
|
||||
// ...existing code...
|
||||
|
||||
// Separate required errors from other validation errors
|
||||
const requiredErrors = errors.filter(error =>
|
||||
error.message?.toLowerCase().includes('required')
|
||||
);
|
||||
const validationErrors = errors.filter(error =>
|
||||
!error.message?.toLowerCase().includes('required')
|
||||
);
|
||||
|
||||
const isRequiredButEmpty = valueIsEmpty && requiredErrors.length > 0;
|
||||
const hasValidationErrors = validationErrors.length > 0;
|
||||
const shouldShowErrorIcon = hasValidationErrors && !valueIsEmpty;
|
||||
|
||||
// ...more code...
|
||||
}
|
||||
```
|
||||
|
||||
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
|
||||
|
||||
### Attempt 2: Comprehensive Fix for Validation Display
|
||||
|
||||
**Approach**: Completely rewrote `processErrors` function with consistent empty value detection, clear error separation, and improved error message extraction.
|
||||
|
||||
**Changes Made**:
|
||||
```typescript
|
||||
function processErrors(value: any, errors: ErrorObject[]) {
|
||||
if (!errors || errors.length === 0) {
|
||||
return { filteredErrors: [], hasError: false, isRequiredButEmpty: false,
|
||||
shouldShowErrorIcon: false, errorMessages: '' };
|
||||
}
|
||||
|
||||
const valueIsEmpty = isEmpty(value);
|
||||
const requiredErrors = errors.filter(error =>
|
||||
error.message?.toLowerCase().includes('required')
|
||||
);
|
||||
const validationErrors = errors.filter(error =>
|
||||
!error.message?.toLowerCase().includes('required')
|
||||
);
|
||||
|
||||
let filteredErrors = valueIsEmpty ? requiredErrors : validationErrors;
|
||||
|
||||
const isRequiredButEmpty = valueIsEmpty && requiredErrors.length > 0;
|
||||
const hasValidationErrors = validationErrors.length > 0;
|
||||
const hasError = isRequiredButEmpty || hasValidationErrors;
|
||||
const shouldShowErrorIcon = hasValidationErrors && !valueIsEmpty;
|
||||
|
||||
let errorMessages = '';
|
||||
if (shouldShowErrorIcon) {
|
||||
errorMessages = validationErrors.map(getErrorMessage).join('\n');
|
||||
}
|
||||
|
||||
return { filteredErrors, hasError, isRequiredButEmpty, shouldShowErrorIcon, errorMessages };
|
||||
}
|
||||
```
|
||||
|
||||
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
|
||||
|
||||
### Attempt 3: Simplified Error Processing Logic
|
||||
|
||||
**Approach**: Refactored `processErrors` to use shared `isEmpty` function, simplified error icon logic, and made error message extraction more direct.
|
||||
|
||||
**Changes Made**:
|
||||
```typescript
|
||||
function processErrors(value: any, errors: ErrorObject[]) {
|
||||
if (!errors || errors.length === 0) {
|
||||
return { filteredErrors: [], hasError: false, isRequiredButEmpty: false,
|
||||
shouldShowErrorIcon: false, errorMessages: '' };
|
||||
}
|
||||
|
||||
const valueIsEmpty = isEmpty(value);
|
||||
const requiredErrors = errors.filter(error =>
|
||||
error.message?.toLowerCase().includes('required')
|
||||
);
|
||||
const validationErrors = errors.filter(error =>
|
||||
!error.message?.toLowerCase().includes('required')
|
||||
);
|
||||
|
||||
let filteredErrors = valueIsEmpty ? requiredErrors : validationErrors;
|
||||
|
||||
const isRequiredButEmpty = valueIsEmpty && requiredErrors.length > 0;
|
||||
const hasValidationErrors = !valueIsEmpty && validationErrors.length > 0;
|
||||
const hasError = isRequiredButEmpty || hasValidationErrors;
|
||||
const shouldShowErrorIcon = hasValidationErrors;
|
||||
|
||||
let errorMessages = '';
|
||||
if (shouldShowErrorIcon) {
|
||||
errorMessages = validationErrors.map(getErrorMessage).join('\n');
|
||||
}
|
||||
|
||||
return { filteredErrors, hasError, isRequiredButEmpty, shouldShowErrorIcon, errorMessages };
|
||||
}
|
||||
```
|
||||
|
||||
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
|
||||
|
||||
### Attempt 4: Consistent Error Processing Across Components
|
||||
|
||||
**Approach**: Updated both `processErrors` function and `ValidationCell` component to ensure consistent error handling between components.
|
||||
|
||||
**Changes Made**:
|
||||
```typescript
|
||||
// In processErrors function
|
||||
function processErrors(value: any, errors: ErrorObject[]) {
|
||||
// Similar to Attempt 3 with consistent error handling
|
||||
}
|
||||
|
||||
// In ValidationCell component
|
||||
const ValidationCell = ({ field, value, onChange, errors, /* other props */ }) => {
|
||||
// ...existing code...
|
||||
|
||||
// Use the processErrors function to handle validation errors
|
||||
const { hasError, isRequiredButEmpty, shouldShowErrorIcon, errorMessages } =
|
||||
React.useMemo(() => processErrors(value, errors), [value, errors]);
|
||||
|
||||
// ...rest of the component...
|
||||
}
|
||||
```
|
||||
|
||||
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
|
||||
|
||||
### Attempt 5: Unified Error Processing with ItemNumberCell
|
||||
|
||||
**Approach**: Replaced custom error processing in `ValidationCell` with the same `processErrors` utility used by `ItemNumberCell`.
|
||||
|
||||
**Changes Made**:
|
||||
```typescript
|
||||
const ValidationCell = ({ field, value, onChange, errors, /* other props */ }) => {
|
||||
// State and context setup...
|
||||
|
||||
// For item_number fields, use the specialized component
|
||||
if (fieldKey === 'item_number') {
|
||||
return <ItemNumberCell {...props} />;
|
||||
}
|
||||
|
||||
// Use the same processErrors utility function that ItemNumberCell uses
|
||||
const { hasError, isRequiredButEmpty, shouldShowErrorIcon, errorMessages } =
|
||||
React.useMemo(() => processErrors(value, errors), [value, errors]);
|
||||
|
||||
// Rest of component...
|
||||
}
|
||||
```
|
||||
|
||||
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
|
||||
|
||||
### Attempt 6: Standardize Error Processing Across Cell Types
|
||||
|
||||
**Approach**: Standardized error handling across all cell types using the shared `processErrors` utility function.
|
||||
|
||||
**Changes Made**: Similar to Attempt 5, with focus on standardizing the approach for determining when to show validation error icons.
|
||||
|
||||
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
|
||||
|
||||
### Attempt 7: Replace Custom Error Processing with Shared Utility
|
||||
|
||||
**Approach**: Ensured consistent error handling between `ItemNumberCell` and regular `ValidationCell` components.
|
||||
|
||||
**Changes Made**: Similar to Attempts 5 and 6, with focus on using the shared utility function consistently.
|
||||
|
||||
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
|
||||
|
||||
### Attempt 8: Improved Error Normalization and Deep Comparison
|
||||
|
||||
**Approach**: Modified `MemoizedCell` in `ValidationTable.tsx` to use deep comparison for error objects and improved error normalization.
|
||||
|
||||
**Changes Made**:
|
||||
```typescript
|
||||
// Create a memoized cell component
|
||||
const MemoizedCell = React.memo(({ field, value, onChange, errors, /* other props */ }) => {
|
||||
return <ValidationCell {...props} />;
|
||||
}, (prev, next) => {
|
||||
// Basic prop comparison
|
||||
if (prev.value !== next.value) return false;
|
||||
if (prev.isValidating !== next.isValidating) return false;
|
||||
if (prev.itemNumber !== next.itemNumber) return false;
|
||||
|
||||
// Deep compare errors - critical for validation display
|
||||
if (!prev.errors && next.errors) return false;
|
||||
if (prev.errors && !next.errors) return false;
|
||||
if (prev.errors && next.errors) {
|
||||
if (prev.errors.length !== next.errors.length) return false;
|
||||
|
||||
// Compare each error object
|
||||
for (let i = 0; i < prev.errors.length; i++) {
|
||||
if (prev.errors[i].message !== next.errors[i].message) return false;
|
||||
if (prev.errors[i].level !== next.errors[i].level) return false;
|
||||
if (prev.errors[i].source !== next.errors[i].source) return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Compare options...
|
||||
|
||||
return true;
|
||||
});
|
||||
|
||||
// In the field columns definition:
|
||||
cell: ({ row }) => {
|
||||
const rowErrors = validationErrors.get(row.index);
|
||||
const cellErrors = rowErrors?.[fieldKey] || [];
|
||||
|
||||
// Ensure cellErrors is always an array
|
||||
const normalizedErrors = Array.isArray(cellErrors) ? cellErrors : [cellErrors];
|
||||
|
||||
return <MemoizedCell {...props} errors={normalizedErrors} />;
|
||||
}
|
||||
```
|
||||
|
||||
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
|
||||
|
||||
## Root Causes (Revised Hypothesis)
|
||||
|
||||
After multiple attempts, the issue appears more complex than initially thought. Possible root causes:
|
||||
|
||||
1. **Error Object Structure**: Error objects might not have the expected structure or properties
|
||||
2. **Error Propagation**: Errors might be getting filtered out before reaching cell components
|
||||
3. **Validation Rules Configuration**: Validation rules in import.tsx might be incorrectly configured
|
||||
4. **Error State Management**: Error state might not be properly updated or might be reset incorrectly
|
||||
5. **Component Rendering Logic**: Components might not re-render when validation state changes
|
||||
6. **CSS/Styling Issues**: Validation icons might be rendered but hidden due to styling issues
|
||||
7. **Validation Timing**: Validation might be happening at the wrong time or getting overridden
|
||||
@@ -0,0 +1,138 @@
|
||||
# Multiple Cell Edit Issue Implementation
|
||||
|
||||
## Issue Being Addressed
|
||||
|
||||
**Multiple Cell Edit Issue**: When you enter values in 2+ cells before validation finishes, contents from all edited cells get erased when validation finishes.
|
||||
|
||||
## Implementation Attempts
|
||||
|
||||
### Attempt 1: Fix Multiple Cell Edit Issue (First Approach)
|
||||
|
||||
**Approach**:
|
||||
- Added a tracking mechanism using a Set to keep track of cells that are currently being edited
|
||||
- Modified the `flushPendingUpdates` function to preserve values of cells being edited
|
||||
- Added cleanup of editing state after validation completes
|
||||
|
||||
**Changes Made**:
|
||||
```typescript
|
||||
// Add ref to track cells currently being edited
|
||||
const currentlyEditingCellsRef = useRef(new Set<string>());
|
||||
|
||||
// Update a row's field value
|
||||
const updateRow = useCallback((rowIndex: number, key: T, value: any) => {
|
||||
// Add this cell to currently editing cells
|
||||
const cellKey = `${rowIndex}-${key}`;
|
||||
currentlyEditingCellsRef.current.add(cellKey);
|
||||
|
||||
// ...existing code...
|
||||
|
||||
// After validation completes, remove this cell from currently editing list
|
||||
setTimeout(() => {
|
||||
currentlyEditingCellsRef.current.delete(cellKey);
|
||||
}, 100);
|
||||
}, []);
|
||||
|
||||
// Modify flushPendingUpdates to respect currently editing cells
|
||||
const flushPendingUpdates = useCallback(() => {
|
||||
// ...existing code...
|
||||
|
||||
if (dataUpdates.length > 0) {
|
||||
setData(prev => {
|
||||
// ...existing code...
|
||||
|
||||
dataUpdates.forEach((row, index) => {
|
||||
if (index < newData.length) {
|
||||
const updatedRow = { ...row };
|
||||
|
||||
// Check if any fields in this row are currently being edited
|
||||
// If so, preserve their current values in the previous data
|
||||
Object.keys(prev[index] || {}).forEach(key => {
|
||||
const cellKey = `${index}-${key}`;
|
||||
if (currentlyEditingCellsRef.current.has(cellKey)) {
|
||||
// Keep the value from the previous state for this field
|
||||
updatedRow[key] = prev[index][key];
|
||||
}
|
||||
});
|
||||
|
||||
newData[index] = updatedRow;
|
||||
}
|
||||
});
|
||||
|
||||
return newData;
|
||||
});
|
||||
}
|
||||
}, []);
|
||||
```
|
||||
|
||||
**Result**:
|
||||
- Slight improvement - the first value entered was saved, but any subsequent values still got erased
|
||||
|
||||
### Attempt 2: Fix Multiple Cell Edit Issue (Second Approach)
|
||||
|
||||
**Approach**:
|
||||
- Completely revised the cell editing tracking system
|
||||
- Used a Map with timestamps to track editing cells more accurately
|
||||
- Added proper Promise-based tracking for cell validation
|
||||
- Increased timeout from 100ms to 1000ms
|
||||
- Made cleanup more robust by checking if it's still the same editing session
|
||||
|
||||
**Changes Made**:
|
||||
```typescript
|
||||
// Add ref to track cells currently being edited with timestamps
|
||||
const currentlyEditingCellsRef = useRef(new Map<string, number>());
|
||||
|
||||
// Add ref to track validation promises
|
||||
const validationPromisesRef = useRef<Map<string, Promise<void>>>(new Map());
|
||||
|
||||
// Update a row's field value
|
||||
const updateRow = useCallback((rowIndex: number, key: T, value: any) => {
|
||||
// Mark this cell as being edited with the current timestamp
|
||||
const cellKey = `${rowIndex}-${key}`;
|
||||
currentlyEditingCellsRef.current.set(cellKey, Date.now());
|
||||
|
||||
// ...existing code...
|
||||
|
||||
// Create a validation promise
|
||||
const validationPromise = new Promise<void>((resolve) => {
|
||||
setTimeout(() => {
|
||||
try {
|
||||
validateRow(rowIndex);
|
||||
} finally {
|
||||
resolve();
|
||||
}
|
||||
}, 0);
|
||||
});
|
||||
|
||||
validationPromisesRef.current.set(cellKey, validationPromise);
|
||||
|
||||
// When validation is complete, remove from validating cells
|
||||
validationPromise.then(() => {
|
||||
// ...existing code...
|
||||
|
||||
// Keep this cell in the editing state for a longer time
|
||||
setTimeout(() => {
|
||||
if (currentlyEditingCellsRef.current.has(cellKey)) {
|
||||
currentlyEditingCellsRef.current.delete(cellKey);
|
||||
}
|
||||
}, 1000); // Keep as "editing" for 1 second
|
||||
});
|
||||
}, []);
|
||||
```
|
||||
|
||||
**Result**:
|
||||
- Worse than the first approach - now all values get erased, including the first one
|
||||
|
||||
## Root Causes (Hypothesized)
|
||||
|
||||
- The validation process might be updating the entire data state, causing race conditions with cell edits
|
||||
- The timing of validation completions might be problematic
|
||||
- State updates might be happening in a way that overwrites user changes
|
||||
- The cell state tracking system is not robust enough to prevent overwrites
|
||||
|
||||
## Next Steps
|
||||
|
||||
The issue requires a more fundamental approach than just tweaking the editing logic. We need to:
|
||||
|
||||
1. Implement a more robust state management system for cell edits that can survive validation cycles
|
||||
2. Consider disabling validation during active editing
|
||||
3. Implement a proper "dirty state" tracking system for cells
|
||||
@@ -0,0 +1,305 @@
|
||||
# Current Issues to Address
|
||||
4. Validation isn't happening beyond checking if a cell is required or not - needs to respect rules in import.tsx
|
||||
* Red cell outline if cell is required and it's empty
|
||||
* Red outline + alert circle icon with tooltip if cell is NOT empty and isn't valid
|
||||
8. When you enter a value in 2+ cells before validation finishes, contents from all edited cells get erased when validation finishes
|
||||
|
||||
## Do NOT change or edit
|
||||
* Anything related to AI validation
|
||||
* Anything about how templates or UPC validation work (only focus on specific issues described above)
|
||||
* Anything outside of the ValidationStepNew folder
|
||||
|
||||
## Issues already fixed - do not work on these
|
||||
✅FIXED 1. The red row background should go away when all cells in the row are valid and all required cells are populated
|
||||
✅FIXED 2. Columns alignment with header is slightly off, gets worse the further right you go
|
||||
✅FIXED 3. The copy down button is in the way of the validation error icon and the select open trigger - all three need to be in unique locations
|
||||
✅FIXED 5. Description column needs to have an expanded view of some sort, maybe a popover to allow for easier editing
|
||||
* Don't distort table to make it happen
|
||||
✅FIXED 6. Need to ensure all cell's contents don't overflow the input (truncate). COO does this currently, probably more
|
||||
✅FIXED 7. The template select cell is expanding, needs to be fixed size and truncate
|
||||
✅FIXED 9. Import dialog state not fully reset when closing? (validate data step appears scrolled to the middle of the table where I left it)
|
||||
✅FIXED 10. UPC column doesn't need to show loading state when Item Number is being processed, only show on item number column
|
||||
✅FIXED 11. Copy down needs to show a loading state on the cells that it will copy to
|
||||
✅FIXED 12. Shipping restrictions/tax category should default to ID 0 if we didn't get it elsewhere
|
||||
✅FIXED 13. Header row should be sticky (both up/down and left/right)
|
||||
✅FIXED 14. Need a way to scroll around table if user doesn't have mouse wheel for left/right
|
||||
✅FIXED 15. Enhance copy down feature by allowing user to choose the last cell to copy to, instead of going all the way to the bottom
|
||||
|
||||
---------
|
||||
|
||||
# Validation Step Components Overview
|
||||
|
||||
## Core Components
|
||||
|
||||
### ValidationContainer
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx`
|
||||
- Main wrapper component for the validation step
|
||||
- Manages global state and coordinates between subcomponents
|
||||
- Handles navigation events (next, back)
|
||||
- Manages template application and validation state
|
||||
- Coordinates UPC validation and product line loading
|
||||
- Manages row selection and filtering
|
||||
- Contains cache management for UPC validation results
|
||||
- Maintains item number references separate from main data
|
||||
|
||||
### ValidationTable
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx`
|
||||
- Handles data display and column configuration
|
||||
- Uses TanStack Table for core functionality
|
||||
- Features:
|
||||
- Sticky header (both vertical and horizontal) - currently doesn't work properly
|
||||
- Row selection with checkboxes
|
||||
- Template selection column
|
||||
- Dynamic column widths based on field types - specified in import.tsx component
|
||||
- Copy down functionality for cell values
|
||||
- Error highlighting for rows and cells
|
||||
- Loading states for cells being validated
|
||||
|
||||
### ValidationCell
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx`
|
||||
- Base cell component that renders different cell types based on field configuration
|
||||
- Handles error display with tooltips
|
||||
- Manages copy down button visibility
|
||||
- Supports loading states during validation
|
||||
- Cell Types:
|
||||
1. InputCell: For single-value text input
|
||||
2. SelectCell: For dropdown selection
|
||||
3. MultiInputCell: For multiple value inputs
|
||||
4. Template selection cells with SearchableTemplateSelect component
|
||||
|
||||
### SearchableTemplateSelect
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx`
|
||||
- Advanced template selection component with search functionality
|
||||
- Features:
|
||||
- Real-time search filtering of templates
|
||||
- Customizable display text for templates
|
||||
- Support for default brand selection
|
||||
- Accessible popover interface
|
||||
- Keyboard navigation support
|
||||
- Custom styling through className props
|
||||
- Scroll event handling for nested scrollable areas
|
||||
|
||||
### TemplateManager
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/TemplateManager.tsx`
|
||||
- Comprehensive template management interface
|
||||
- Features:
|
||||
- Template selection with search functionality
|
||||
- Save template dialog with name and type inputs
|
||||
- Batch template application to selected rows
|
||||
- Template count tracking
|
||||
- Toast notifications for user feedback
|
||||
- Dialog-based interface for template operations
|
||||
|
||||
### AiValidationDialogs
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx`
|
||||
- Manages AI-assisted validation dialogs and interactions
|
||||
|
||||
### SaveTemplateDialog
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SaveTemplateDialog.tsx`
|
||||
- Dialog component for saving new templates
|
||||
|
||||
## Cell Components
|
||||
|
||||
### InputCell
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx`
|
||||
- Handles single value text input
|
||||
- Features:
|
||||
- Inline/edit mode switching
|
||||
- Multiline support
|
||||
- Price formatting
|
||||
- Error state display
|
||||
- Loading state during validation
|
||||
- Width constraints
|
||||
- Automated cleanPriceFields processing for "$" formatting
|
||||
|
||||
### SelectCell
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx`
|
||||
- Handles dropdown selection
|
||||
- Features:
|
||||
- Searchable dropdown
|
||||
- Custom option rendering
|
||||
- Error state display
|
||||
- Loading state during validation
|
||||
- Width constraints
|
||||
- Disabled state support
|
||||
- Deferred search query handling for performance
|
||||
|
||||
### MultiInputCell
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiInputCell.tsx`
|
||||
- Handles multiple value inputs
|
||||
- Features:
|
||||
- Comma-separated input support
|
||||
- Multi-select dropdown for predefined options
|
||||
- Custom separators
|
||||
- Badge display for selected count
|
||||
- Truncation for long values
|
||||
- Width constraints
|
||||
- Price formatting support
|
||||
- Internal state management to avoid excessive re-renders
|
||||
|
||||
## Validation System
|
||||
|
||||
### useValidation Hook
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx`
|
||||
- Provides core validation logic
|
||||
- Validates at multiple levels:
|
||||
1. Field-level validation (required, regex, unique)
|
||||
2. Row-level validation (supplier, company fields)
|
||||
3. Table-level validation
|
||||
4. Custom validation hooks support
|
||||
- Error object structure includes message, level, and source properties
|
||||
- Handles debounced validation updates to avoid UI freezing
|
||||
|
||||
### useAiValidation Hook
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx`
|
||||
- Manages AI-assisted validation logic and state
|
||||
- Features:
|
||||
- Tracks detailed changes per product
|
||||
- Manages validation progress with estimated completion time
|
||||
- Handles warnings and change suggestions
|
||||
- Supports diff generation for changes
|
||||
- Progress tracking with step indicators
|
||||
- Prompt management for AI interactions
|
||||
- Timer management for long-running operations
|
||||
|
||||
### useTemplates Hook
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx`
|
||||
- Comprehensive template management system
|
||||
- Features:
|
||||
- Template CRUD operations
|
||||
- Template application logic
|
||||
- Default value handling
|
||||
- Template search and filtering
|
||||
- Batch template operations
|
||||
- Template validation
|
||||
|
||||
### useUpcValidation Hook
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx`
|
||||
- Dedicated UPC validation management
|
||||
- Features:
|
||||
- UPC format validation
|
||||
- Supplier data validation
|
||||
- Cache management for validation results
|
||||
- Batch processing of UPC validations
|
||||
- Item number generation logic
|
||||
- Loading state management
|
||||
|
||||
### useFilters Hook
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useFilters.tsx`
|
||||
- Advanced filtering system for table data
|
||||
- Features:
|
||||
- Multiple filter criteria support
|
||||
- Dynamic filter updates
|
||||
- Filter persistence
|
||||
- Filter combination logic
|
||||
- Performance optimized filtering
|
||||
|
||||
### useValidationState Hook
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx`
|
||||
- Manages global validation state
|
||||
- Handles:
|
||||
- Data updates
|
||||
- Template management
|
||||
- Error tracking using Map objects
|
||||
- Row selection
|
||||
- Filtering
|
||||
- UPC validation with caching to prevent duplicate API calls
|
||||
- Product line loading
|
||||
- Batch processing of updates
|
||||
- Default value application for tax_cat and ship_restrictions (defaulting to "0")
|
||||
- Price field auto-formatting to remove "$" symbols
|
||||
|
||||
### Utility Files
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts`
|
||||
- Core validation utility functions
|
||||
- Includes:
|
||||
- Field validation logic
|
||||
- Error message formatting
|
||||
- Validation rule processing
|
||||
- Type checking utilities
|
||||
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts`
|
||||
- Error handling and formatting utilities
|
||||
- Includes:
|
||||
- Error object creation
|
||||
- Error message formatting
|
||||
- Error source tracking
|
||||
- Error level management
|
||||
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts`
|
||||
- Data transformation and mutation utilities
|
||||
- Includes:
|
||||
- Row data updates
|
||||
- Batch data processing
|
||||
- Data structure conversions
|
||||
- Change tracking
|
||||
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js`
|
||||
- Helper functions for validation
|
||||
- Includes:
|
||||
- Common validation patterns
|
||||
- Validation state management
|
||||
- Validation result processing
|
||||
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts`
|
||||
- UPC-specific validation utilities
|
||||
- Includes:
|
||||
- UPC format checking
|
||||
- Checksum validation
|
||||
- Supplier data matching
|
||||
- Cache management
|
||||
|
||||
### Types
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts`
|
||||
- Core type definitions for the validation step
|
||||
|
||||
### Validation Types
|
||||
1. Required field validation
|
||||
2. Regex pattern validation
|
||||
3. Unique value validation
|
||||
4. Custom field validation
|
||||
5. Row-level validation
|
||||
6. Table-level validation
|
||||
|
||||
## State Management
|
||||
|
||||
### useValidationState Hook
|
||||
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx`
|
||||
- Manages global validation state
|
||||
- Handles:
|
||||
- Data updates
|
||||
- Template management
|
||||
- Error tracking using Map objects
|
||||
- Row selection
|
||||
- Filtering
|
||||
- UPC validation with caching to prevent duplicate API calls
|
||||
- Product line loading
|
||||
- Batch processing of updates
|
||||
- Default value application for tax_cat and ship_restrictions (defaulting to "0")
|
||||
- Price field auto-formatting to remove "$" symbols
|
||||
|
||||
## UPC Validation System
|
||||
|
||||
### UPC Processing
|
||||
- Validates UPCs against supplier data
|
||||
- Cache system for UPC validation results
|
||||
- Batch processing of UPC validation requests
|
||||
- Auto-generation of item numbers based on UPC
|
||||
- Special loading states for UPC/item number fields
|
||||
- Separate state tracking to avoid unnecessary data structure updates
|
||||
|
||||
## Template System
|
||||
|
||||
### Template Management
|
||||
- Supports saving and loading templates
|
||||
- Template application to single/multiple rows
|
||||
- Default template values
|
||||
- Template search and filtering
|
||||
|
||||
## Performance Optimizations
|
||||
1. Memoized components to prevent unnecessary renders
|
||||
2. Virtualized table for large datasets
|
||||
3. Deferred value updates for search inputs
|
||||
4. Efficient error state management
|
||||
5. Optimized cell update handling
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
|
||||
|
||||
# Refactoring Plan for Validation Code
|
||||
|
||||
## Current Structure Analysis
|
||||
- **useValidationState.tsx**: ~1650 lines - Core validation state management
|
||||
- **useValidation.tsx**: ~425 lines - Field/data validation utility
|
||||
- **useUpcValidation.tsx**: ~410 lines - UPC-specific validation
|
||||
|
||||
## Proposed New Structure
|
||||
|
||||
### 1. Core Types & Utilities (150-200 lines)
|
||||
**File: `validation/types.ts`**
|
||||
- All interfaces and types (RowData, ValidationError, FilterState, Template, etc.)
|
||||
- Shared utility functions (isEmpty, getCellKey, etc.)
|
||||
|
||||
**File: `validation/utils.ts`**
|
||||
- Generic validation utility functions
|
||||
- Caching mechanism and cache clearing helpers
|
||||
- API URL helpers
|
||||
|
||||
### 2. Field Validation (300-350 lines)
|
||||
**File: `validation/hooks/useFieldValidation.ts`**
|
||||
- `validateField` function
|
||||
- Field-level validation logic
|
||||
- Required, regex, and other field validations
|
||||
|
||||
### 3. Uniqueness Validation (250-300 lines)
|
||||
**File: `validation/hooks/useUniquenessValidation.ts`**
|
||||
- `validateUniqueField` function
|
||||
- `validateUniqueItemNumbers` function
|
||||
- All uniqueness checking logic
|
||||
|
||||
### 4. UPC Validation (300-350 lines)
|
||||
**File: `validation/hooks/useUpcValidation.ts`**
|
||||
- `fetchProductByUpc` function
|
||||
- `validateUpc` function
|
||||
- `applyItemNumbersToData` function
|
||||
- UPC validation state management
|
||||
|
||||
### 5. Validation Status Management (300-350 lines)
|
||||
**File: `validation/hooks/useValidationStatus.ts`**
|
||||
- Error state management
|
||||
- Row validation status tracking
|
||||
- Validation indicators and refs
|
||||
- Batch validation processing
|
||||
|
||||
### 6. Data Management (300-350 lines)
|
||||
**File: `validation/hooks/useValidationData.ts`**
|
||||
- Data state management
|
||||
- Row updates
|
||||
- Data filtering
|
||||
- Initial data processing
|
||||
|
||||
### 7. Template Management (250-300 lines)
|
||||
**File: `validation/hooks/useTemplateManagement.ts`**
|
||||
- Template saving
|
||||
- Template application
|
||||
- Template loading
|
||||
- Template display helpers
|
||||
|
||||
### 8. Main Validation Hook (300-350 lines)
|
||||
**File: `validation/hooks/useValidation.ts`**
|
||||
- Main hook that composes all other hooks
|
||||
- Public API export
|
||||
- Initialization logic
|
||||
- Core validation flow
|
||||
|
||||
## Function Distribution
|
||||
|
||||
### Core Types & Utilities
|
||||
- All interfaces (InfoWithSource, ValidationState, etc.)
|
||||
- `isEmpty` utility
|
||||
- `getApiUrl` helper
|
||||
|
||||
### Field Validation
|
||||
- `validateField`
|
||||
- `validateRow`
|
||||
- `validateData` (partial)
|
||||
- All validation result caching
|
||||
|
||||
### Uniqueness Validation
|
||||
- `validateUniqueField`
|
||||
- `validateUniqueItemNumbers`
|
||||
- Uniqueness caching mechanisms
|
||||
|
||||
### UPC Validation
|
||||
- `fetchProductByUpc`
|
||||
- `validateUpc`
|
||||
- `validateAllUPCs`
|
||||
- `applyItemNumbersToData`
|
||||
- UPC validation state tracking (cells, rows)
|
||||
|
||||
### Validation Status Management
|
||||
- `startValidatingCell`/`stopValidatingCell`
|
||||
- `startValidatingRow`/`stopValidatingRow`
|
||||
- `isValidatingCell`/`isRowValidatingUpc`
|
||||
- Error state management
|
||||
- `revalidateRows`
|
||||
|
||||
### Data Management
|
||||
- Initial data cleaning/processing
|
||||
- `updateRow`
|
||||
- `copyDown`
|
||||
- Search/filter functionality
|
||||
- `filteredData` calculation
|
||||
|
||||
### Template Management
|
||||
- `saveTemplate`
|
||||
- `applyTemplate`
|
||||
- `applyTemplateToSelected`
|
||||
- `getTemplateDisplayText`
|
||||
- `loadTemplates`/`refreshTemplates`
|
||||
|
||||
### Main Validation Hook
|
||||
- Composition of all other hooks
|
||||
- Initialization logic
|
||||
- Button/navigation handling
|
||||
- Field options management
|
||||
|
||||
## Implementation Approach
|
||||
|
||||
1. **Start with Types**: Create the types file first, as all other files will depend on it
|
||||
2. **Create Utility Functions**: Move shared utilities next
|
||||
3. **Build Core Validation**: Extract the field validation and uniqueness validation
|
||||
4. **Separate UPC Logic**: Move all UPC-specific code to its own module
|
||||
5. **Extract State Management**: Move data and status management to separate files
|
||||
6. **Move Template Logic**: Extract template functionality
|
||||
7. **Create Composition Hook**: Build the main hook that uses all other hooks
|
||||
|
||||
This approach will give you more maintainable code with clearer separation of concerns, making it easier to understand, test, and modify each component independently.
|
||||
@@ -0,0 +1,354 @@
|
||||
## 1. ✅ Error Filtering Logic Inconsistency (RESOLVED)
|
||||
|
||||
> **Note: This issue has been resolved by implementing a type-based error system.**
|
||||
|
||||
The filtering logic in `ValidationCell.tsx` previously relied on string matching, which was fragile:
|
||||
|
||||
```typescript
|
||||
// Old implementation (string-based matching)
|
||||
const filteredErrors = React.useMemo(() => {
|
||||
return !isEmpty(value)
|
||||
? errors.filter(error => !error.message?.toLowerCase().includes('required'))
|
||||
: errors;
|
||||
}, [value, errors]);
|
||||
|
||||
// New implementation (type-based filtering)
|
||||
const filteredErrors = React.useMemo(() => {
|
||||
return !isEmpty(value)
|
||||
? errors.filter(error => error.type !== ErrorType.Required)
|
||||
: errors;
|
||||
}, [value, errors]);
|
||||
```
|
||||
|
||||
The solution implemented:
|
||||
- Added an `ErrorType` enum in `types.ts` to standardize error categorization
|
||||
- Updated all error creation to include the appropriate error type
|
||||
- Modified error filtering to use the type property instead of string matching
|
||||
- Ensured consistent error handling across the application
|
||||
|
||||
**Guidelines for future development:**
|
||||
- Always use the `ErrorType` enum when creating errors
|
||||
- Never rely on string matching for error filtering
|
||||
- Ensure all error objects include the `type` property
|
||||
- Use the appropriate error type for each validation rule:
|
||||
- `ErrorType.Required` for required field validations
|
||||
- `ErrorType.Regex` for regex validations
|
||||
- `ErrorType.Unique` for uniqueness validations
|
||||
- `ErrorType.Custom` for custom validations
|
||||
- `ErrorType.Api` for API-based validations
|
||||
|
||||
## 2. ⚠️ Redundant Error Processing (PARTIALLY RESOLVED)
|
||||
|
||||
> **Note: This issue has been partially resolved by the re-rendering optimizations.**
|
||||
|
||||
The system still processes errors in multiple places:
|
||||
- In `ValidationCell.tsx`, errors are filtered by the optimized `processErrors` function
|
||||
- In `useValidation.tsx`, errors are generated at the field level
|
||||
- In `ValidationContainer.tsx`, errors are manipulated at the container level
|
||||
|
||||
While the error processing has been optimized to be more efficient, there is still some redundancy in how errors are handled across components. However, the current implementation has mitigated the performance impact.
|
||||
|
||||
**Improvements made:**
|
||||
- Created a central `processErrors` function in ValidationCell that efficiently handles error filtering
|
||||
- Implemented a batched update system to reduce redundant error processing
|
||||
- Added better memoization to avoid reprocessing errors when not needed
|
||||
|
||||
**Future improvement opportunities:**
|
||||
- Further consolidate error processing logic into a single location
|
||||
- Create a dedicated error handling service or hook
|
||||
- Implement a more declarative approach to error handling
|
||||
|
||||
## 3. Race Conditions in Async Validation
|
||||
|
||||
async validations could create race conditions:
|
||||
- If a user types quickly, multiple validation requests might be in flight
|
||||
- Later responses could overwrite more recent ones if they complete out of order
|
||||
- The debouncing helps but doesn't fully solve this issue
|
||||
|
||||
## 4. Memory Leaks in Timeout Management
|
||||
|
||||
The validation timeouts are stored in refs:
|
||||
```typescript
|
||||
const validationTimeoutsRef = useRef<Record<number, NodeJS.Timeout>>({});
|
||||
```
|
||||
|
||||
While there is cleanup on unmount, if rows are added/removed dynamically, timeouts for deleted rows might not be properly cleared.
|
||||
|
||||
## 5. ✅ Inefficient Error Storage (RESOLVED)
|
||||
|
||||
**Status: RESOLVED**
|
||||
|
||||
### Problem
|
||||
|
||||
Previously, validation errors were stored in multiple locations:
|
||||
- In the `validationErrors` Map in `useValidationState`
|
||||
- In the row data itself as `__errors`
|
||||
|
||||
This redundancy caused several issues:
|
||||
- Inconsistent error states between the two storage locations
|
||||
- Increased memory usage by storing the same information twice
|
||||
- Complex state management to keep both sources in sync
|
||||
- Difficulty reasoning about where errors should be accessed from
|
||||
|
||||
### Solution
|
||||
|
||||
We've implemented a unified error storage approach by:
|
||||
- Making the `validationErrors` Map in `useValidationState` the single source of truth for all validation errors
|
||||
- Removed the `__errors` property from row data
|
||||
- Updated all validation functions to interact with the central error store instead of modifying row data
|
||||
- Modified UPC validation to use the central error store
|
||||
- Updated all components to read errors from the `validationErrors` Map instead of row data
|
||||
|
||||
### Key Changes
|
||||
|
||||
1. Modified `dataMutations.ts` to stop storing errors in row data
|
||||
2. Updated the `Meta` type to remove the `__errors` property
|
||||
3. Modified the `RowData` type to remove the `__errors` property
|
||||
4. Updated the `useValidation` hook to return errors separately from row data
|
||||
5. Modified the `useAiValidation` hook to work with the central error store
|
||||
6. Updated the `useFilters` hook to check for errors in the `validationErrors` Map
|
||||
7. Modified the `ValidationTable` and `ValidationCell` components to read errors from the `validationErrors` Map
|
||||
|
||||
### Benefits
|
||||
|
||||
- **Single Source of Truth**: All validation errors are now stored in one place
|
||||
- **Reduced Memory Usage**: No duplicate storage of error information
|
||||
- **Simplified State Management**: Only one state to update when errors change
|
||||
- **Cleaner Data Structure**: Row data no longer contains validation metadata
|
||||
- **More Maintainable Code**: Clearer separation of concerns between data and validation
|
||||
|
||||
### Future Improvements
|
||||
|
||||
While this refactoring addresses the core issue of inefficient error storage, there are still opportunities for further optimization:
|
||||
|
||||
1. ✅ **Redundant Error Processing**: ~~The validation process still performs some redundant calculations that could be optimized.~~ This has been largely addressed by the re-rendering optimizations.
|
||||
2. **Race Conditions**: Async validation can lead to race conditions when multiple validations are triggered in quick succession.
|
||||
3. **Memory Leaks**: The timeout management for validation could be improved to prevent potential memory leaks.
|
||||
4. **Tight Coupling**: Components are still tightly coupled to the validation state structure.
|
||||
5. **Error Prioritization**: The system doesn't prioritize errors well, showing all errors at once rather than focusing on the most critical ones first.
|
||||
|
||||
### Validation Flow
|
||||
|
||||
The validation process now works as follows:
|
||||
|
||||
1. **Error Generation**:
|
||||
- Field-level validations generate errors based on validation rules
|
||||
- Row-level hooks add custom validation errors
|
||||
- Table-level validations (like uniqueness checks) add errors across rows
|
||||
|
||||
2. **Error Storage**:
|
||||
- All errors are stored in the `validationErrors` Map in `useValidationState`
|
||||
- The Map uses row indices as keys and objects of field errors as values
|
||||
|
||||
3. **Error Display**:
|
||||
- The `ValidationTable` component checks the `validationErrors` Map to highlight rows with errors
|
||||
- The `ValidationCell` component receives errors for specific fields from the `validationErrors` Map
|
||||
- Errors are filtered in the UI to avoid showing "required" errors for fields with values
|
||||
|
||||
This focused refactoring approach has successfully addressed a critical issue while keeping changes manageable and targeted.
|
||||
|
||||
## 6. ✅ Excessive Re-rendering (RESOLVED)
|
||||
|
||||
**Status: RESOLVED**
|
||||
|
||||
### Problem
|
||||
|
||||
The validation system was suffering from excessive re-renders due to several key issues:
|
||||
|
||||
- **Inefficient Error Filtering**: The ValidationCell component was filtering errors on every render
|
||||
- **Redundant Error Processing**: The same validation work was repeated in multiple components
|
||||
- **Poor Memoization**: Components were inadequately memoized, causing unnecessary re-renders
|
||||
- **Inefficient Batch Updates**: The state update system wasn't optimally batching changes
|
||||
|
||||
These issues led to performance problems, especially with large datasets, and affected the user experience.
|
||||
|
||||
### Solution
|
||||
|
||||
We've implemented a comprehensive optimization approach:
|
||||
|
||||
- **Optimized Error Processing**: Created an efficient `processErrors` function in ValidationCell that calculates all derived state in one pass
|
||||
- **Enhanced Memoization**: Improved memo comparison functions to avoid unnecessary rerenders
|
||||
- **Improved Batch Updates**: Redesigned the batching system to aggregate multiple changes before state updates
|
||||
- **Single Update Pattern**: Implemented a queue-based update mechanism that applies multiple state changes at once
|
||||
|
||||
### Key Changes
|
||||
|
||||
1. Added a more efficient error processing function in ValidationCell
|
||||
2. Created an enhanced error comparison function to properly compare error arrays
|
||||
3. Improved the memo comparison function in ValidationCell
|
||||
4. Added a batch update system in useValidationState
|
||||
5. Implemented a queue-based update mechanism for row modifications
|
||||
|
||||
### Benefits
|
||||
|
||||
- **Improved Performance**: Reduced render cycles = faster UI response
|
||||
- **Better User Experience**: Less lag when editing large datasets
|
||||
- **Reduced Memory Usage**: Fewer component instantiations and temporary objects
|
||||
- **Increased Scalability**: The application can now handle larger datasets without slowdown
|
||||
- **Maintainable Code**: More predictable update flow that's easier to debug and extend
|
||||
|
||||
### Guidelines for future development
|
||||
|
||||
- Use the `processErrors` function for error filtering and processing
|
||||
- Ensure React.memo components have proper comparison functions
|
||||
- Use the batched update system for state changes
|
||||
- Maintain stable references to objects and functions
|
||||
- Use appropriate React hooks (useMemo, useCallback) with correct dependencies
|
||||
- Avoid unnecessary recreations of arrays, objects, and functions
|
||||
|
||||
## 7. Complex Error Merging Logic
|
||||
|
||||
When merging errors from different sources, the logic is complex and potentially error-prone:
|
||||
```typescript
|
||||
// Merge field errors and row hook errors
|
||||
const mergedErrors: Record<string, InfoWithSource> = {}
|
||||
|
||||
// Convert field errors to InfoWithSource
|
||||
Object.entries(fieldErrors).forEach(([key, errors]) => {
|
||||
if (errors.length > 0) {
|
||||
mergedErrors[key] = {
|
||||
message: errors[0].message,
|
||||
level: errors[0].level,
|
||||
source: ErrorSources.Row,
|
||||
type: errors[0].type || ErrorType.Custom
|
||||
}
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
This only takes the first error for each field, potentially hiding important validation issues.
|
||||
|
||||
## 8. ✅ Inconsistent Error Handling for Empty Values (PARTIALLY RESOLVED)
|
||||
|
||||
> **Note: This issue has been partially resolved by standardizing the isEmpty function and error type system.**
|
||||
|
||||
The system previously had different approaches to handling empty values:
|
||||
- Some validations skipped empty values unless they're required
|
||||
- Others processed empty values differently
|
||||
- The `isEmpty` function was defined multiple times with slight variations
|
||||
|
||||
The solution implemented:
|
||||
- Standardized the `isEmpty` function implementation
|
||||
- Ensured consistent error type usage for required field validations
|
||||
- Made error filtering consistent across the application
|
||||
|
||||
**Guidelines for future development:**
|
||||
- Always use the shared `isEmpty` function for checking empty values
|
||||
- Ensure consistent handling of empty values across all validation rules
|
||||
- Use the `ErrorType.Required` type for all required field validations
|
||||
|
||||
## 9. Tight Coupling Between Components
|
||||
|
||||
The validation system is tightly coupled across components:
|
||||
- `ValidationCell` needs to understand the structure of errors
|
||||
- `ValidationTable` needs to extract and pass the right errors
|
||||
- `ValidationContainer` directly manipulates the error structure
|
||||
|
||||
This makes it harder to refactor or reuse components independently.
|
||||
|
||||
## 10. Limited Error Prioritization
|
||||
|
||||
There's no clear prioritization of errors:
|
||||
- When multiple errors exist for a field, which one should be shown first?
|
||||
- Are some errors more important than others?
|
||||
- The current system mostly shows the first error it finds
|
||||
|
||||
A more robust approach would be to have a consistent error source identification system and a clear prioritization strategy for displaying errors.
|
||||
|
||||
------------
|
||||
|
||||
Let me explain how these hooks fit together to create the validation errors that eventually get filtered in the `ValidationCell` component:
|
||||
|
||||
## The Validation Flow
|
||||
|
||||
1. **useValidationState Hook**:
|
||||
This is the main state management hook used by the `ValidationContainer` component. It:
|
||||
- Manages the core data state (`data`)
|
||||
- Tracks validation errors in a Map (`validationErrors`)
|
||||
- Provides functions to update and validate rows
|
||||
|
||||
2. **useValidation Hook**:
|
||||
This is a utility hook that provides the core validation logic:
|
||||
- `validateField`: Validates a single field against its validation rules
|
||||
- `validateRow`: Validates an entire row, field by field
|
||||
- `validateTable`: Runs table-level validations
|
||||
- `validateUnique`: Checks for uniqueness constraints
|
||||
- `validateData`: Orchestrates the complete validation process
|
||||
|
||||
## How Errors Are Generated
|
||||
|
||||
Validation errors come from multiple sources:
|
||||
|
||||
1. **Field-Level Validations**:
|
||||
In `useValidation.tsx`, the `validateField` function checks individual fields against rules like:
|
||||
- `required`: Field must have a value
|
||||
- `regex`: Value must match a pattern
|
||||
- `min`/`max`: Numeric constraints
|
||||
|
||||
2. **Row-Level Validations**:
|
||||
The `validateRow` function in `useValidation.tsx` runs:
|
||||
- Field validations for each field in the row
|
||||
- Special validations for required fields like supplier and company
|
||||
- Custom row hooks provided by the application
|
||||
|
||||
3. **Table-Level Validations**:
|
||||
- `validateUnique` checks for duplicate values in fields marked as unique
|
||||
- `validateTable` runs custom table hooks for cross-row validations
|
||||
|
||||
4. **API-Based Validations**:
|
||||
In `useValidationState.tsx` and `ValidationContainer.tsx`:
|
||||
- UPC validation via API calls
|
||||
- Item number uniqueness checks
|
||||
|
||||
## The Error Flow
|
||||
|
||||
1. Errors are collected in the `validationErrors` Map in `useValidationState`
|
||||
2. This Map is passed to `ValidationTable` as a prop
|
||||
3. `ValidationTable` extracts the relevant errors for each cell and passes them to `ValidationCell`
|
||||
4. In `ValidationCell`, the errors are filtered based on whether the cell has a value:
|
||||
```typescript
|
||||
// Updated implementation using type-based filtering
|
||||
const filteredErrors = React.useMemo(() => {
|
||||
return !isEmpty(value)
|
||||
? errors.filter(error => error.type !== ErrorType.Required)
|
||||
: errors;
|
||||
}, [value, errors]);
|
||||
```
|
||||
|
||||
## Key Insights
|
||||
|
||||
1. **Error Structure**:
|
||||
Errors now have a consistent structure with type information:
|
||||
```typescript
|
||||
type ErrorObject = {
|
||||
message: string;
|
||||
level: string; // 'error', 'warning', etc.
|
||||
source?: ErrorSources; // Where the error came from
|
||||
type: ErrorType; // The type of error (Required, Regex, Unique, etc.)
|
||||
}
|
||||
```
|
||||
|
||||
2. **Error Sources**:
|
||||
Errors can come from:
|
||||
- Field validations (required, regex, etc.)
|
||||
- Row validations (custom business logic)
|
||||
- Table validations (uniqueness checks)
|
||||
- API validations (UPC checks)
|
||||
|
||||
3. **Error Types**:
|
||||
Errors are now categorized by type:
|
||||
- `ErrorType.Required`: Field is required but empty
|
||||
- `ErrorType.Regex`: Value doesn't match the regex pattern
|
||||
- `ErrorType.Unique`: Value must be unique across rows
|
||||
- `ErrorType.Custom`: Custom validation errors
|
||||
- `ErrorType.Api`: Errors from API calls
|
||||
|
||||
4. **Error Filtering**:
|
||||
The filtering in `ValidationCell` is now more robust:
|
||||
- When a field has a value, errors of type `ErrorType.Required` are filtered out
|
||||
- When a field is empty, all errors are shown
|
||||
|
||||
5. **Performance Optimizations**:
|
||||
- Batch processing of validations
|
||||
- Debounced updates to avoid excessive re-renders
|
||||
- Memoization of computed values
|
||||
@@ -0,0 +1,538 @@
|
||||
# ValidationTable Scroll Position Issue
|
||||
|
||||
## Problem Description
|
||||
|
||||
The `ValidationTable` component in the inventory application suffers from a persistent scroll position issue. When the table content updates or re-renders, the scroll position resets to the top left corner. This creates a poor user experience, especially when users are working with large datasets and need to maintain their position while making edits or filtering data.
|
||||
|
||||
Specific behaviors:
|
||||
- Scroll position resets to the top left corner during re-renders
|
||||
- User loses their place in the table when data is updated
|
||||
- The table does not preserve vertical or horizontal scroll position
|
||||
|
||||
## Relevant Files
|
||||
|
||||
- **`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx`**
|
||||
- Main component that renders the validation table
|
||||
- Handles scroll position management
|
||||
|
||||
- **`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx`**
|
||||
- Parent component that wraps ValidationTable
|
||||
- Creates an EnhancedValidationTable wrapper component
|
||||
- Manages data and state for the validation table
|
||||
|
||||
- **`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx`**
|
||||
- Provides state management and data manipulation functions
|
||||
- Contains scroll-related code in the `updateRow` function
|
||||
|
||||
- **`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx`**
|
||||
- Renders individual cells in the table
|
||||
- May influence re-renders that affect scroll position
|
||||
|
||||
## Failed Attempts
|
||||
|
||||
We've tried multiple approaches to fix the scroll position issue, none of which have been successful:
|
||||
|
||||
### 1. Using Refs for Scroll Position
|
||||
|
||||
```typescript
|
||||
const scrollPosition = useRef({
|
||||
left: 0,
|
||||
top: 0
|
||||
});
|
||||
|
||||
// Capture position on scroll
|
||||
const handleScroll = useCallback(() => {
|
||||
if (tableContainerRef.current) {
|
||||
scrollPosition.current = {
|
||||
left: tableContainerRef.current.scrollLeft,
|
||||
top: tableContainerRef.current.scrollTop
|
||||
};
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Restore in useLayoutEffect
|
||||
useLayoutEffect(() => {
|
||||
const container = tableContainerRef.current;
|
||||
if (container) {
|
||||
const { left, top } = scrollPosition.current;
|
||||
if (left || top) {
|
||||
container.scrollLeft = left;
|
||||
container.scrollTop = top;
|
||||
}
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
Result: Scroll position was still lost during updates.
|
||||
|
||||
### 2. Multiple Restoration Attempts with Timeouts
|
||||
|
||||
```typescript
|
||||
// Multiple timeouts at different intervals
|
||||
setTimeout(() => {
|
||||
if (tableContainerRef.current) {
|
||||
tableContainerRef.current.scrollTop = savedPosition.top;
|
||||
tableContainerRef.current.scrollLeft = savedPosition.left;
|
||||
}
|
||||
}, 0);
|
||||
|
||||
setTimeout(() => {
|
||||
if (tableContainerRef.current) {
|
||||
tableContainerRef.current.scrollTop = savedPosition.top;
|
||||
tableContainerRef.current.scrollLeft = savedPosition.left;
|
||||
}
|
||||
}, 50);
|
||||
|
||||
// Additional timeouts at 100ms, 300ms
|
||||
```
|
||||
|
||||
Result: Still not reliable, scroll position would reset between timeouts or after all timeouts completed.
|
||||
|
||||
### 3. Using MutationObserver and ResizeObserver
|
||||
|
||||
```typescript
|
||||
// Create a mutation observer to detect DOM changes
|
||||
const mutationObserver = new MutationObserver(() => {
|
||||
if (shouldPreserveScroll) {
|
||||
restoreScrollPosition();
|
||||
}
|
||||
});
|
||||
|
||||
// Start observing the table for DOM changes
|
||||
mutationObserver.observe(scrollableContainer, {
|
||||
childList: true,
|
||||
subtree: true,
|
||||
attributes: true,
|
||||
attributeFilter: ['style', 'class']
|
||||
});
|
||||
|
||||
// Create a resize observer
|
||||
const resizeObserver = new ResizeObserver(() => {
|
||||
if (shouldPreserveScroll) {
|
||||
restoreScrollPosition();
|
||||
}
|
||||
});
|
||||
|
||||
// Observe the table container
|
||||
resizeObserver.observe(scrollableContainer);
|
||||
```
|
||||
|
||||
Result: Did not reliably maintain scroll position, and sometimes caused other rendering issues.
|
||||
|
||||
### 4. Recursive Restoration Approach
|
||||
|
||||
```typescript
|
||||
let attempts = 0;
|
||||
const maxAttempts = 5;
|
||||
|
||||
const restore = () => {
|
||||
if (tableContainerRef.current) {
|
||||
tableContainerRef.current.scrollTop = y;
|
||||
tableContainerRef.current.scrollLeft = x;
|
||||
|
||||
attempts++;
|
||||
if (attempts < maxAttempts) {
|
||||
setTimeout(restore, 50 * attempts);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
restore();
|
||||
```
|
||||
|
||||
Result: No improvement, scroll position still reset.
|
||||
|
||||
### 5. Using React State for Scroll Position
|
||||
|
||||
```typescript
|
||||
const [scrollPos, setScrollPos] = useState<{top: number; left: number}>({top: 0, left: 0});
|
||||
|
||||
// Track the scroll event
|
||||
useEffect(() => {
|
||||
const handleScroll = () => {
|
||||
if (scrollContainerRef.current) {
|
||||
setScrollPos({
|
||||
top: scrollContainerRef.current.scrollTop,
|
||||
left: scrollContainerRef.current.scrollLeft
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
// Add scroll listener...
|
||||
}, []);
|
||||
|
||||
// Restore scroll position
|
||||
useLayoutEffect(() => {
|
||||
const container = scrollContainerRef.current;
|
||||
const { top, left } = scrollPos;
|
||||
|
||||
if (top > 0 || left > 0) {
|
||||
requestAnimationFrame(() => {
|
||||
if (container) {
|
||||
container.scrollTop = top;
|
||||
container.scrollLeft = left;
|
||||
}
|
||||
});
|
||||
}
|
||||
}, [scrollPos, data]);
|
||||
```
|
||||
|
||||
Result: Caused the screen to shake violently when scrolling and did not preserve position.
|
||||
|
||||
### 6. Using Key Attribute for Stability
|
||||
|
||||
```typescript
|
||||
return (
|
||||
<div
|
||||
key="validation-table-container"
|
||||
ref={scrollContainerRef}
|
||||
className="overflow-auto max-h-[calc(100vh-300px)]"
|
||||
>
|
||||
{/* Table content */}
|
||||
</div>
|
||||
);
|
||||
```
|
||||
|
||||
Result: Did not resolve the issue and may have contributed to rendering instability.
|
||||
|
||||
### 7. Removing Scroll Management from Other Components
|
||||
|
||||
We removed scroll position management code from:
|
||||
- `useValidationState.tsx` (in the updateRow function)
|
||||
- `ValidationContainer.tsx` (in the enhancedUpdateRow function)
|
||||
|
||||
Result: This did not fix the issue either.
|
||||
|
||||
### 8. Simple Scroll Position Management with Event Listeners
|
||||
|
||||
```typescript
|
||||
// Create a ref to store scroll position
|
||||
const scrollPosition = useRef({ left: 0, top: 0 });
|
||||
const tableContainerRef = useRef<HTMLDivElement>(null);
|
||||
|
||||
// Save scroll position when scrolling
|
||||
const handleScroll = useCallback(() => {
|
||||
if (tableContainerRef.current) {
|
||||
scrollPosition.current = {
|
||||
left: tableContainerRef.current.scrollLeft,
|
||||
top: tableContainerRef.current.scrollTop
|
||||
};
|
||||
}
|
||||
}, []);
|
||||
|
||||
// Add scroll listener
|
||||
useEffect(() => {
|
||||
const container = tableContainerRef.current;
|
||||
if (container) {
|
||||
container.addEventListener('scroll', handleScroll);
|
||||
return () => container.removeEventListener('scroll', handleScroll);
|
||||
}
|
||||
}, [handleScroll]);
|
||||
|
||||
// Restore scroll position after data changes
|
||||
useLayoutEffect(() => {
|
||||
const container = tableContainerRef.current;
|
||||
if (container) {
|
||||
const { left, top } = scrollPosition.current;
|
||||
if (left > 0 || top > 0) {
|
||||
container.scrollLeft = left;
|
||||
container.scrollTop = top;
|
||||
}
|
||||
}
|
||||
}, [data]);
|
||||
```
|
||||
|
||||
Result: Still did not maintain scroll position during updates.
|
||||
|
||||
### 9. Memoized Scroll Container Component
|
||||
|
||||
```typescript
|
||||
// Create a stable scroll container that won't re-render with the table
|
||||
const ScrollContainer = React.memo(({ children }: { children: React.ReactNode }) => {
|
||||
const containerRef = useRef<HTMLDivElement>(null);
|
||||
const scrollPosition = useRef({ left: 0, top: 0 });
|
||||
|
||||
const handleScroll = useCallback(() => {
|
||||
if (containerRef.current) {
|
||||
scrollPosition.current = {
|
||||
left: containerRef.current.scrollLeft,
|
||||
top: containerRef.current.scrollTop
|
||||
};
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const container = containerRef.current;
|
||||
if (container) {
|
||||
// Set initial scroll position if it exists
|
||||
if (scrollPosition.current.left > 0 || scrollPosition.current.top > 0) {
|
||||
container.scrollLeft = scrollPosition.current.left;
|
||||
container.scrollTop = scrollPosition.current.top;
|
||||
}
|
||||
container.addEventListener('scroll', handleScroll);
|
||||
return () => container.removeEventListener('scroll', handleScroll);
|
||||
}
|
||||
}, [handleScroll]);
|
||||
|
||||
return (
|
||||
<div ref={containerRef} className="overflow-auto max-h-[calc(100vh-300px)]">
|
||||
{children}
|
||||
</div>
|
||||
);
|
||||
});
|
||||
```
|
||||
|
||||
Result: Still did not maintain scroll position during updates, even with a memoized container.
|
||||
|
||||
### 10. Using TanStack Table State Management
|
||||
|
||||
```typescript
|
||||
// Track scroll state in the table instance
|
||||
const [scrollState, setScrollState] = useState({ scrollLeft: 0, scrollTop: 0 });
|
||||
|
||||
const table = useReactTable({
|
||||
data,
|
||||
columns,
|
||||
getCoreRowModel: getCoreRowModel(),
|
||||
state: {
|
||||
rowSelection,
|
||||
// Include scroll position in table state
|
||||
scrollLeft: scrollState.scrollLeft,
|
||||
scrollTop: scrollState.scrollTop
|
||||
},
|
||||
onStateChange: (updater) => {
|
||||
if (typeof updater === 'function') {
|
||||
const newState = updater({
|
||||
rowSelection,
|
||||
scrollLeft: scrollState.scrollLeft,
|
||||
scrollTop: scrollState.scrollTop
|
||||
});
|
||||
if ('scrollLeft' in newState || 'scrollTop' in newState) {
|
||||
setScrollState({
|
||||
scrollLeft: newState.scrollLeft ?? scrollState.scrollLeft,
|
||||
scrollTop: newState.scrollTop ?? scrollState.scrollTop
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Handle scroll events
|
||||
const handleScroll = useCallback((event: React.UIEvent<HTMLDivElement>) => {
|
||||
const target = event.target as HTMLDivElement;
|
||||
setScrollState({
|
||||
scrollLeft: target.scrollLeft,
|
||||
scrollTop: target.scrollTop
|
||||
});
|
||||
}, []);
|
||||
|
||||
// Restore scroll position after updates
|
||||
useLayoutEffect(() => {
|
||||
if (tableContainerRef.current) {
|
||||
tableContainerRef.current.scrollLeft = scrollState.scrollLeft;
|
||||
tableContainerRef.current.scrollTop = scrollState.scrollTop;
|
||||
}
|
||||
}, [data, scrollState]);
|
||||
```
|
||||
|
||||
Result: Still did not maintain scroll position during updates, even with table state management.
|
||||
|
||||
### 11. Using CSS Sticky Positioning
|
||||
|
||||
```typescript
|
||||
return (
|
||||
<div className="relative max-h-[calc(100vh-300px)] overflow-auto">
|
||||
<Table>
|
||||
<TableHeader className="sticky top-0 z-10 bg-background">
|
||||
<TableRow>
|
||||
{table.getFlatHeaders().map((header) => (
|
||||
<TableHead
|
||||
key={header.id}
|
||||
style={{
|
||||
width: `${header.getSize()}px`,
|
||||
minWidth: `${header.getSize()}px`,
|
||||
position: 'sticky',
|
||||
top: 0,
|
||||
backgroundColor: 'inherit'
|
||||
}}
|
||||
>
|
||||
{/* Header content */}
|
||||
</TableHead>
|
||||
))}
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{/* Table body content */}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
);
|
||||
```
|
||||
|
||||
Result: Still did not maintain scroll position during updates, even with native CSS scrolling.
|
||||
|
||||
### 12. Optimized Memoization with Object.is
|
||||
|
||||
```typescript
|
||||
// Memoize data structures to prevent unnecessary re-renders
|
||||
const memoizedData = useMemo(() => data, [data]);
|
||||
const memoizedValidationErrors = useMemo(() => validationErrors, [validationErrors]);
|
||||
const memoizedValidatingCells = useMemo(() => validatingCells, [validatingCells]);
|
||||
const memoizedItemNumbers = useMemo(() => itemNumbers, [itemNumbers]);
|
||||
|
||||
// Use Object.is for more efficient comparisons
|
||||
export default React.memo(ValidationTable, (prev, next) => {
|
||||
if (!Object.is(prev.data.length, next.data.length)) return false;
|
||||
|
||||
if (prev.validationErrors.size !== next.validationErrors.size) return false;
|
||||
for (const [key, value] of prev.validationErrors) {
|
||||
if (!next.validationErrors.has(key)) return false;
|
||||
if (!Object.is(value, next.validationErrors.get(key))) return false;
|
||||
}
|
||||
|
||||
// ... more optimized comparisons ...
|
||||
});
|
||||
```
|
||||
|
||||
Result: Caused the page to crash with "TypeError: undefined has no properties" in the MemoizedCell component.
|
||||
|
||||
### 13. Simplified Component Structure
|
||||
|
||||
```typescript
|
||||
const ValidationTable = <T extends string>({
|
||||
data,
|
||||
fields,
|
||||
rowSelection,
|
||||
setRowSelection,
|
||||
updateRow,
|
||||
validationErrors,
|
||||
// ... other props
|
||||
}) => {
|
||||
const tableContainerRef = useRef<HTMLDivElement>(null);
|
||||
const lastScrollPosition = useRef({ left: 0, top: 0 });
|
||||
|
||||
// Simple scroll position management
|
||||
const handleScroll = useCallback(() => {
|
||||
if (tableContainerRef.current) {
|
||||
lastScrollPosition.current = {
|
||||
left: tableContainerRef.current.scrollLeft,
|
||||
top: tableContainerRef.current.scrollTop
|
||||
};
|
||||
}
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const container = tableContainerRef.current;
|
||||
if (container) {
|
||||
container.addEventListener('scroll', handleScroll);
|
||||
return () => container.removeEventListener('scroll', handleScroll);
|
||||
}
|
||||
}, [handleScroll]);
|
||||
|
||||
useLayoutEffect(() => {
|
||||
const container = tableContainerRef.current;
|
||||
if (container) {
|
||||
const { left, top } = lastScrollPosition.current;
|
||||
if (left > 0 || top > 0) {
|
||||
requestAnimationFrame(() => {
|
||||
if (container) {
|
||||
container.scrollLeft = left;
|
||||
container.scrollTop = top;
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}, [data]);
|
||||
|
||||
return (
|
||||
<div ref={tableContainerRef} className="overflow-auto max-h-[calc(100vh-300px)]">
|
||||
<Table>
|
||||
{/* ... table content ... */}
|
||||
<TableBody>
|
||||
{table.getRowModel().rows.map((row) => (
|
||||
<TableRow
|
||||
key={row.id}
|
||||
className={cn(
|
||||
"hover:bg-muted/50",
|
||||
row.getIsSelected() ? "bg-muted/50" : "",
|
||||
validationErrors.get(data.indexOf(row.original)) ? "bg-red-50/40" : ""
|
||||
)}
|
||||
>
|
||||
{/* ... row content ... */}
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
```
|
||||
|
||||
Result: Still did not maintain scroll position during updates. However, this implementation restored the subtle red highlight on rows with validation errors, which is a useful visual indicator that should be preserved in future attempts.
|
||||
|
||||
### 14. Portal-Based Scroll Container
|
||||
|
||||
```typescript
|
||||
// Create a stable container outside of React's control
|
||||
const createStableContainer = () => {
|
||||
const containerId = 'validation-table-container';
|
||||
let container = document.getElementById(containerId);
|
||||
|
||||
if (!container) {
|
||||
container = document.createElement('div');
|
||||
container.id = containerId;
|
||||
container.className = 'overflow-auto';
|
||||
container.style.maxHeight = 'calc(100vh - 300px)';
|
||||
document.body.appendChild(container);
|
||||
}
|
||||
|
||||
return container;
|
||||
};
|
||||
|
||||
const ValidationTable = <T extends string>({...props}) => {
|
||||
const [container] = useState(createStableContainer);
|
||||
const [mounted, setMounted] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
setMounted(true);
|
||||
return () => {
|
||||
if (container && container.parentNode) {
|
||||
container.parentNode.removeChild(container);
|
||||
}
|
||||
};
|
||||
}, [container]);
|
||||
|
||||
// ... table configuration ...
|
||||
|
||||
return createPortal(content, container);
|
||||
};
|
||||
```
|
||||
|
||||
Result: The table contents failed to render at all. The portal-based approach to maintain scroll position by moving the scroll container outside of React's control was unsuccessful.
|
||||
|
||||
## Current Understanding
|
||||
|
||||
The scroll position issue appears to be complex and likely stems from multiple factors:
|
||||
|
||||
1. React's virtual DOM reconciliation may be replacing the scroll container element during updates
|
||||
2. The table uses complex memo patterns with custom equality checks that may not be working as expected
|
||||
3. The data structure may be changing in ways that cause complete re-renders
|
||||
4. The component hierarchy (with EnhancedValidationTable wrapper) may be affecting DOM stability
|
||||
|
||||
## Next Steps to Consider
|
||||
|
||||
At this point, we have tried multiple approaches without success:
|
||||
1. Various scroll position management techniques
|
||||
2. Memoization and optimization strategies
|
||||
3. Different component structures
|
||||
4. Portal-based rendering
|
||||
|
||||
Given that none of these approaches have fully resolved the issue, it may be worth:
|
||||
1. Investigating if there are any parent component updates forcing re-renders
|
||||
2. Profiling the application to identify the exact timing of scroll position resets
|
||||
3. Considering if the current table implementation could be simplified
|
||||
4. Exploring if the data update patterns could be optimized to reduce re-renders
|
||||
|
||||
## Conclusion
|
||||
|
||||
The scroll position issue has proven resistant to multiple solution attempts. Each approach has either failed to maintain scroll position, introduced new issues, or in some cases (like the portal-based approach) prevented the table from rendering entirely. A deeper investigation into the component lifecycle and data flow may be necessary to identify the root cause.
|
||||
@@ -1,22 +0,0 @@
|
||||
module.exports = {
|
||||
apps: [
|
||||
{
|
||||
...commonSettings,
|
||||
name: 'new-auth-server',
|
||||
script: './inventory-server/auth/server.js',
|
||||
env: {
|
||||
NODE_ENV: 'production',
|
||||
AUTH_PORT: 3011,
|
||||
JWT_SECRET: process.env.JWT_SECRET
|
||||
},
|
||||
error_file: 'inventory-server/auth/logs/pm2/err.log',
|
||||
out_file: 'inventory-server/auth/logs/pm2/out.log',
|
||||
log_file: 'inventory-server/auth/logs/pm2/combined.log',
|
||||
env_production: {
|
||||
NODE_ENV: 'production',
|
||||
AUTH_PORT: 3011,
|
||||
JWT_SECRET: process.env.JWT_SECRET
|
||||
}
|
||||
}
|
||||
]
|
||||
};
|
||||
@@ -0,0 +1,75 @@
|
||||
import bcrypt from 'bcrypt';
|
||||
import pg from 'pg';
|
||||
import inquirer from 'inquirer';
|
||||
|
||||
const { Pool } = pg;
|
||||
import { config as loadEnv } from 'dotenv';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, resolve as resolvePath } from 'node:path';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
loadEnv({ path: resolvePath(__dirname, '../.env') });
|
||||
|
||||
const pool = new Pool({
|
||||
host: process.env.DB_HOST,
|
||||
user: process.env.DB_USER,
|
||||
password: process.env.DB_PASSWORD,
|
||||
database: process.env.DB_NAME,
|
||||
port: Number(process.env.DB_PORT) || 5432,
|
||||
});
|
||||
|
||||
async function promptUser() {
|
||||
return inquirer.prompt([
|
||||
{
|
||||
type: 'input',
|
||||
name: 'username',
|
||||
message: 'Enter username:',
|
||||
validate: (input) => input.length >= 3 || 'Username must be at least 3 characters long',
|
||||
},
|
||||
{
|
||||
type: 'password',
|
||||
name: 'password',
|
||||
message: 'Enter password:',
|
||||
mask: '*',
|
||||
validate: (input) => input.length >= 8 || 'Password must be at least 8 characters long',
|
||||
},
|
||||
{
|
||||
type: 'password',
|
||||
name: 'confirmPassword',
|
||||
message: 'Confirm password:',
|
||||
mask: '*',
|
||||
validate: (input, answers) => input === answers.password || 'Passwords do not match',
|
||||
},
|
||||
]);
|
||||
}
|
||||
|
||||
async function addUser() {
|
||||
try {
|
||||
const { username, password } = await promptUser();
|
||||
const hashedPassword = await bcrypt.hash(password, 10);
|
||||
|
||||
const checkResult = await pool.query(
|
||||
'SELECT id FROM users WHERE username = $1',
|
||||
[username]
|
||||
);
|
||||
if (checkResult.rows.length > 0) {
|
||||
console.error('Error: Username already exists');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const result = await pool.query(
|
||||
'INSERT INTO users (username, password) VALUES ($1, $2) RETURNING id',
|
||||
[username, hashedPassword]
|
||||
);
|
||||
console.log(`User ${username} created successfully with id ${result.rows[0].id}`);
|
||||
} catch (error) {
|
||||
console.error('Error creating user:', error);
|
||||
if (error.code) console.error('Error code:', error.code);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
addUser();
|
||||
@@ -1,41 +0,0 @@
|
||||
const bcrypt = require('bcrypt');
|
||||
const mysql = require('mysql2/promise');
|
||||
const readline = require('readline').createInterface({
|
||||
input: process.stdin,
|
||||
output: process.stdout,
|
||||
});
|
||||
require('dotenv').config({ path: '../.env' });
|
||||
|
||||
const dbConfig = {
|
||||
host: process.env.DB_HOST,
|
||||
user: process.env.DB_USER,
|
||||
password: process.env.DB_PASSWORD,
|
||||
database: process.env.DB_NAME,
|
||||
};
|
||||
|
||||
async function addUser() {
|
||||
const username = await askQuestion('Enter username: ');
|
||||
const password = await askQuestion('Enter password: ');
|
||||
|
||||
const hashedPassword = await bcrypt.hash(password, 10);
|
||||
|
||||
const connection = await mysql.createConnection(dbConfig);
|
||||
|
||||
try {
|
||||
await connection.query('INSERT INTO users (username, password) VALUES (?, ?)', [username, hashedPassword]);
|
||||
console.log(`User ${username} added successfully.`);
|
||||
} catch (error) {
|
||||
console.error('Error adding user:', error);
|
||||
} finally {
|
||||
connection.end();
|
||||
readline.close();
|
||||
}
|
||||
}
|
||||
|
||||
function askQuestion(query) {
|
||||
return new Promise(resolve => readline.question(query, ans => {
|
||||
resolve(ans);
|
||||
}));
|
||||
}
|
||||
|
||||
addUser();
|
||||
Generated
+628
-268
File diff suppressed because it is too large
Load Diff
@@ -1,21 +1,23 @@
|
||||
{
|
||||
"name": "auth-server",
|
||||
"name": "inventory-auth-server",
|
||||
"version": "1.0.0",
|
||||
"description": "Authentication server for inventory management",
|
||||
"description": "Authentication server for inventory management system",
|
||||
"type": "module",
|
||||
"main": "server.js",
|
||||
"scripts": {
|
||||
"start": "node server.js",
|
||||
"dev": "nodemon server.js",
|
||||
"add_user": "node add_user.js"
|
||||
"add-user": "node add-user.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"bcrypt": "^5.1.1",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^16.4.5",
|
||||
"dotenv": "^16.4.7",
|
||||
"express": "^4.18.2",
|
||||
"jsonwebtoken": "^9.0.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nodemon": "^3.1.0"
|
||||
"express-rate-limit": "^7.4.0",
|
||||
"inquirer": "^8.2.6",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"pg": "^8.11.3",
|
||||
"pino": "^9.5.0",
|
||||
"pino-http": "^10.3.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
export function createPermissionHelpers({ pool }) {
|
||||
async function checkPermission(userId, permissionCode) {
|
||||
try {
|
||||
const adminResult = await pool.query(
|
||||
'SELECT is_admin FROM users WHERE id = $1',
|
||||
[userId]
|
||||
);
|
||||
if (adminResult.rows.length > 0 && adminResult.rows[0].is_admin) return true;
|
||||
|
||||
const result = await pool.query(
|
||||
`SELECT COUNT(*) AS has_permission
|
||||
FROM user_permissions up
|
||||
JOIN permissions p ON up.permission_id = p.id
|
||||
WHERE up.user_id = $1 AND p.code = $2`,
|
||||
[userId, permissionCode]
|
||||
);
|
||||
return Number(result.rows[0].has_permission) > 0;
|
||||
} catch (error) {
|
||||
console.error('Error checking permission:', error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function requirePermission(permissionCode) {
|
||||
return async (req, res, next) => {
|
||||
try {
|
||||
if (!req.user?.id) {
|
||||
return res.status(401).json({ error: 'Authentication required' });
|
||||
}
|
||||
const hasPermission = await checkPermission(req.user.id, permissionCode);
|
||||
if (!hasPermission) {
|
||||
return res.status(403).json({
|
||||
error: 'Insufficient permissions',
|
||||
requiredPermission: permissionCode,
|
||||
});
|
||||
}
|
||||
next();
|
||||
} catch (error) {
|
||||
console.error('Permission middleware error:', error);
|
||||
res.status(500).json({ error: 'Server error checking permissions' });
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function getUserPermissions(userId) {
|
||||
try {
|
||||
const adminResult = await pool.query(
|
||||
'SELECT is_admin FROM users WHERE id = $1',
|
||||
[userId]
|
||||
);
|
||||
if (adminResult.rows.length === 0) return [];
|
||||
|
||||
if (adminResult.rows[0].is_admin) {
|
||||
const allPermissions = await pool.query('SELECT code FROM permissions');
|
||||
return allPermissions.rows.map((p) => p.code);
|
||||
}
|
||||
|
||||
const permissions = await pool.query(
|
||||
`SELECT p.code
|
||||
FROM permissions p
|
||||
JOIN user_permissions up ON p.id = up.permission_id
|
||||
WHERE up.user_id = $1`,
|
||||
[userId]
|
||||
);
|
||||
return permissions.rows.map((p) => p.code);
|
||||
} catch (error) {
|
||||
console.error('Error getting user permissions:', error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
return { checkPermission, requirePermission, getUserPermissions };
|
||||
}
|
||||
@@ -0,0 +1,317 @@
|
||||
import express from 'express';
|
||||
import bcrypt from 'bcrypt';
|
||||
import jwt from 'jsonwebtoken';
|
||||
import { createPermissionHelpers } from './permissions.js';
|
||||
|
||||
export function createAuthRoutes({ pool }) {
|
||||
const router = express.Router();
|
||||
const { requirePermission, getUserPermissions } = createPermissionHelpers({ pool });
|
||||
|
||||
// Local authenticate(): used by user-management endpoints that need req.user populated
|
||||
// with id/username/email/is_admin. NOT the per-service authenticate() — that lives in
|
||||
// shared/auth/middleware.js and is used by downstream services. Auth-server's surface is
|
||||
// small enough that a local copy is fine; the security boundary is the JWT verify step.
|
||||
async function authenticate(req, res, next) {
|
||||
try {
|
||||
const authHeader = req.headers.authorization;
|
||||
if (!authHeader || !authHeader.startsWith('Bearer ')) {
|
||||
return res.status(401).json({ error: 'Authentication required' });
|
||||
}
|
||||
const token = authHeader.split(' ')[1];
|
||||
const decoded = jwt.verify(token, process.env.JWT_SECRET);
|
||||
const result = await pool.query(
|
||||
'SELECT id, username, email, is_admin, rocket_chat_user_id FROM users WHERE id = $1',
|
||||
[decoded.userId]
|
||||
);
|
||||
if (result.rows.length === 0) {
|
||||
return res.status(401).json({ error: 'User not found' });
|
||||
}
|
||||
req.user = result.rows[0];
|
||||
next();
|
||||
} catch (error) {
|
||||
res.status(401).json({ error: 'Invalid token' });
|
||||
}
|
||||
}
|
||||
|
||||
router.post('/login', async (req, res) => {
|
||||
try {
|
||||
const { username, password } = req.body;
|
||||
const result = await pool.query(
|
||||
'SELECT id, username, password, is_admin, is_active, rocket_chat_user_id FROM users WHERE username = $1',
|
||||
[username]
|
||||
);
|
||||
if (result.rows.length === 0) {
|
||||
return res.status(401).json({ error: 'Invalid username or password' });
|
||||
}
|
||||
const user = result.rows[0];
|
||||
if (!user.is_active) {
|
||||
return res.status(403).json({ error: 'Account is inactive' });
|
||||
}
|
||||
const validPassword = await bcrypt.compare(password, user.password);
|
||||
if (!validPassword) {
|
||||
return res.status(401).json({ error: 'Invalid username or password' });
|
||||
}
|
||||
await pool.query(
|
||||
'UPDATE users SET last_login = CURRENT_TIMESTAMP WHERE id = $1',
|
||||
[user.id]
|
||||
);
|
||||
const token = jwt.sign(
|
||||
{ userId: user.id, username: user.username },
|
||||
process.env.JWT_SECRET,
|
||||
{ expiresIn: '8h' }
|
||||
);
|
||||
const permissions = await getUserPermissions(user.id);
|
||||
res.json({
|
||||
token,
|
||||
user: {
|
||||
id: user.id,
|
||||
username: user.username,
|
||||
is_admin: user.is_admin,
|
||||
rocket_chat_user_id: user.rocket_chat_user_id,
|
||||
permissions,
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Login error:', error);
|
||||
res.status(500).json({ error: 'Server error' });
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/me', authenticate, async (req, res) => {
|
||||
try {
|
||||
const permissions = await getUserPermissions(req.user.id);
|
||||
res.json({
|
||||
id: req.user.id,
|
||||
username: req.user.username,
|
||||
email: req.user.email,
|
||||
is_admin: req.user.is_admin,
|
||||
rocket_chat_user_id: req.user.rocket_chat_user_id,
|
||||
permissions,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error getting current user:', error);
|
||||
res.status(500).json({ error: 'Server error' });
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/users', authenticate, requirePermission('view:users'), async (req, res) => {
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
SELECT id, username, email, is_admin, is_active, rocket_chat_user_id, created_at, last_login
|
||||
FROM users
|
||||
ORDER BY username
|
||||
`);
|
||||
res.json(result.rows);
|
||||
} catch (error) {
|
||||
console.error('Error getting users:', error);
|
||||
res.status(500).json({ error: 'Server error' });
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/users/:id', authenticate, requirePermission('view:users'), async (req, res) => {
|
||||
try {
|
||||
const userId = req.params.id;
|
||||
const userResult = await pool.query(`
|
||||
SELECT id, username, email, is_admin, is_active, rocket_chat_user_id, created_at, last_login
|
||||
FROM users
|
||||
WHERE id = $1
|
||||
`, [userId]);
|
||||
if (userResult.rows.length === 0) {
|
||||
return res.status(404).json({ error: 'User not found' });
|
||||
}
|
||||
const permissionsResult = await pool.query(`
|
||||
SELECT p.id, p.name, p.code, p.category, p.description
|
||||
FROM permissions p
|
||||
JOIN user_permissions up ON p.id = up.permission_id
|
||||
WHERE up.user_id = $1
|
||||
ORDER BY p.category, p.name
|
||||
`, [userId]);
|
||||
res.json({
|
||||
...userResult.rows[0],
|
||||
permissions: permissionsResult.rows,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error getting user:', error);
|
||||
res.status(500).json({ error: 'Server error' });
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/users', authenticate, requirePermission('create:users'), async (req, res) => {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
const { username, email, password, is_admin, is_active, rocket_chat_user_id, permissions } = req.body;
|
||||
if (!username || !password) {
|
||||
return res.status(400).json({ error: 'Username and password are required' });
|
||||
}
|
||||
const existingUser = await client.query(
|
||||
'SELECT id FROM users WHERE username = $1',
|
||||
[username]
|
||||
);
|
||||
if (existingUser.rows.length > 0) {
|
||||
return res.status(400).json({ error: 'Username already exists' });
|
||||
}
|
||||
await client.query('BEGIN');
|
||||
const hashedPassword = await bcrypt.hash(password, 10);
|
||||
const rcUserId = rocket_chat_user_id ? parseInt(rocket_chat_user_id, 10) : null;
|
||||
const userResult = await client.query(`
|
||||
INSERT INTO users (username, email, password, is_admin, is_active, rocket_chat_user_id, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, CURRENT_TIMESTAMP)
|
||||
RETURNING id
|
||||
`, [username, email || null, hashedPassword, !!is_admin, is_active !== false, rcUserId]);
|
||||
const userId = userResult.rows[0].id;
|
||||
|
||||
if (!is_admin && Array.isArray(permissions) && permissions.length > 0) {
|
||||
const permissionIds = normalizePermissionIds(permissions);
|
||||
if (permissionIds.length > 0) {
|
||||
await client.query(
|
||||
`INSERT INTO user_permissions (user_id, permission_id)
|
||||
SELECT $1, unnest($2::int[])
|
||||
ON CONFLICT DO NOTHING`,
|
||||
[userId, permissionIds]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
await client.query('COMMIT');
|
||||
res.status(201).json({ id: userId, message: 'User created successfully' });
|
||||
} catch (error) {
|
||||
await client.query('ROLLBACK');
|
||||
console.error('Error creating user:', error);
|
||||
res.status(500).json({ error: 'Server error' });
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
});
|
||||
|
||||
router.put('/users/:id', authenticate, requirePermission('edit:users'), async (req, res) => {
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
const userId = req.params.id;
|
||||
const { username, email, password, is_admin, is_active, rocket_chat_user_id, permissions } = req.body;
|
||||
|
||||
const userExists = await client.query('SELECT id FROM users WHERE id = $1', [userId]);
|
||||
if (userExists.rows.length === 0) {
|
||||
return res.status(404).json({ error: 'User not found' });
|
||||
}
|
||||
|
||||
await client.query('BEGIN');
|
||||
|
||||
const updateFields = [];
|
||||
const updateValues = [userId];
|
||||
let paramIndex = 2;
|
||||
|
||||
if (username !== undefined) { updateFields.push(`username = $${paramIndex++}`); updateValues.push(username); }
|
||||
if (email !== undefined) { updateFields.push(`email = $${paramIndex++}`); updateValues.push(email || null); }
|
||||
if (is_admin !== undefined) { updateFields.push(`is_admin = $${paramIndex++}`); updateValues.push(!!is_admin); }
|
||||
if (is_active !== undefined) { updateFields.push(`is_active = $${paramIndex++}`); updateValues.push(!!is_active); }
|
||||
if (rocket_chat_user_id !== undefined) {
|
||||
updateFields.push(`rocket_chat_user_id = $${paramIndex++}`);
|
||||
updateValues.push(rocket_chat_user_id ? parseInt(rocket_chat_user_id, 10) : null);
|
||||
}
|
||||
if (password) {
|
||||
const hashedPassword = await bcrypt.hash(password, 10);
|
||||
updateFields.push(`password = $${paramIndex++}`);
|
||||
updateValues.push(hashedPassword);
|
||||
}
|
||||
|
||||
if (updateFields.length > 0) {
|
||||
updateFields.push(`updated_at = CURRENT_TIMESTAMP`);
|
||||
await client.query(`
|
||||
UPDATE users SET ${updateFields.join(', ')} WHERE id = $1
|
||||
`, updateValues);
|
||||
}
|
||||
|
||||
if (Array.isArray(permissions)) {
|
||||
await client.query('DELETE FROM user_permissions WHERE user_id = $1', [userId]);
|
||||
const newIsAdmin = is_admin !== undefined
|
||||
? is_admin
|
||||
: (await client.query('SELECT is_admin FROM users WHERE id = $1', [userId])).rows[0].is_admin;
|
||||
|
||||
if (!newIsAdmin && permissions.length > 0) {
|
||||
const permissionIds = normalizePermissionIds(permissions);
|
||||
if (permissionIds.length > 0) {
|
||||
await client.query(
|
||||
`INSERT INTO user_permissions (user_id, permission_id)
|
||||
SELECT $1, unnest($2::int[])
|
||||
ON CONFLICT DO NOTHING`,
|
||||
[userId, permissionIds]
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await client.query('COMMIT');
|
||||
res.json({ message: 'User updated successfully' });
|
||||
} catch (error) {
|
||||
await client.query('ROLLBACK');
|
||||
console.error('Error updating user:', error);
|
||||
res.status(500).json({ error: 'Server error' });
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
});
|
||||
|
||||
router.delete('/users/:id', authenticate, requirePermission('delete:users'), async (req, res) => {
|
||||
try {
|
||||
const userId = req.params.id;
|
||||
if (req.user.id === parseInt(userId, 10)) {
|
||||
return res.status(400).json({ error: 'Cannot delete your own account' });
|
||||
}
|
||||
const result = await pool.query(
|
||||
'DELETE FROM users WHERE id = $1 RETURNING id',
|
||||
[userId]
|
||||
);
|
||||
if (result.rows.length === 0) {
|
||||
return res.status(404).json({ error: 'User not found' });
|
||||
}
|
||||
res.json({ message: 'User deleted successfully' });
|
||||
} catch (error) {
|
||||
console.error('Error deleting user:', error);
|
||||
res.status(500).json({ error: 'Server error' });
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/permissions/categories', authenticate, requirePermission('view:users'), async (req, res) => {
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
SELECT category, json_agg(
|
||||
json_build_object(
|
||||
'id', id, 'name', name, 'code', code, 'description', description
|
||||
) ORDER BY name
|
||||
) as permissions
|
||||
FROM permissions
|
||||
GROUP BY category
|
||||
ORDER BY category
|
||||
`);
|
||||
res.json(result.rows);
|
||||
} catch (error) {
|
||||
console.error('Error getting permissions:', error);
|
||||
res.status(500).json({ error: 'Server error' });
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/permissions', authenticate, requirePermission('view:users'), async (req, res) => {
|
||||
try {
|
||||
const result = await pool.query(`
|
||||
SELECT * FROM permissions ORDER BY category, name
|
||||
`);
|
||||
res.json(result.rows);
|
||||
} catch (error) {
|
||||
console.error('Error getting permissions:', error);
|
||||
res.status(500).json({ error: 'Server error' });
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
function normalizePermissionIds(permissions) {
|
||||
return permissions
|
||||
.map((p) => {
|
||||
if (typeof p === 'object' && p?.id) return parseInt(p.id, 10);
|
||||
if (typeof p === 'number') return p;
|
||||
if (typeof p === 'string' && !Number.isNaN(parseInt(p, 10))) return parseInt(p, 10);
|
||||
return null;
|
||||
})
|
||||
.filter((id) => id !== null && !Number.isNaN(id));
|
||||
}
|
||||
@@ -1,6 +1,89 @@
|
||||
CREATE TABLE `users` (
|
||||
`id` INT AUTO_INCREMENT PRIMARY KEY,
|
||||
`username` VARCHAR(255) NOT NULL UNIQUE,
|
||||
`password` VARCHAR(255) NOT NULL,
|
||||
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
CREATE TABLE users (
|
||||
id SERIAL PRIMARY KEY,
|
||||
username VARCHAR(255) NOT NULL UNIQUE,
|
||||
password VARCHAR(255) NOT NULL,
|
||||
email VARCHAR UNIQUE,
|
||||
is_admin BOOLEAN DEFAULT FALSE,
|
||||
is_active BOOLEAN DEFAULT TRUE,
|
||||
last_login TIMESTAMP WITH TIME ZONE,
|
||||
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Function to update the updated_at timestamp
|
||||
CREATE OR REPLACE FUNCTION update_updated_at_column()
|
||||
RETURNS TRIGGER AS $$
|
||||
BEGIN
|
||||
NEW.updated_at = CURRENT_TIMESTAMP;
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ language 'plpgsql';
|
||||
|
||||
-- Sequence and defined type for users table if not exists
|
||||
CREATE SEQUENCE IF NOT EXISTS users_id_seq;
|
||||
|
||||
-- Create permissions table
|
||||
CREATE TABLE IF NOT EXISTS "public"."permissions" (
|
||||
"id" SERIAL PRIMARY KEY,
|
||||
"name" varchar NOT NULL UNIQUE,
|
||||
"code" varchar NOT NULL UNIQUE,
|
||||
"description" text,
|
||||
"category" varchar NOT NULL,
|
||||
"created_at" timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
|
||||
"updated_at" timestamp with time zone DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Create user_permissions junction table
|
||||
CREATE TABLE IF NOT EXISTS "public"."user_permissions" (
|
||||
"user_id" int4 NOT NULL REFERENCES "public"."users"("id") ON DELETE CASCADE,
|
||||
"permission_id" int4 NOT NULL REFERENCES "public"."permissions"("id") ON DELETE CASCADE,
|
||||
"created_at" timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY ("user_id", "permission_id")
|
||||
);
|
||||
|
||||
-- Add triggers for updated_at on users and permissions
|
||||
DROP TRIGGER IF EXISTS update_users_updated_at ON users;
|
||||
CREATE TRIGGER update_users_updated_at
|
||||
BEFORE UPDATE ON users
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
DROP TRIGGER IF EXISTS update_permissions_updated_at ON permissions;
|
||||
CREATE TRIGGER update_permissions_updated_at
|
||||
BEFORE UPDATE ON permissions
|
||||
FOR EACH ROW
|
||||
EXECUTE FUNCTION update_updated_at_column();
|
||||
|
||||
-- Insert default permissions by page - only the ones used in application
|
||||
INSERT INTO permissions (name, code, description, category) VALUES
|
||||
('Dashboard Access', 'access:dashboard', 'Can access the Dashboard page', 'Pages'),
|
||||
('Products Access', 'access:products', 'Can access the Products page', 'Pages'),
|
||||
('Categories Access', 'access:categories', 'Can access the Categories page', 'Pages'),
|
||||
('Vendors Access', 'access:vendors', 'Can access the Vendors page', 'Pages'),
|
||||
('Analytics Access', 'access:analytics', 'Can access the Analytics page', 'Pages'),
|
||||
('Forecasting Access', 'access:forecasting', 'Can access the Forecasting page', 'Pages'),
|
||||
('Purchase Orders Access', 'access:purchase_orders', 'Can access the Purchase Orders page', 'Pages'),
|
||||
('Import Access', 'access:import', 'Can access the Import page', 'Pages'),
|
||||
('Settings Access', 'access:settings', 'Can access the Settings page', 'Pages'),
|
||||
('AI Validation Debug Access', 'access:ai_validation_debug', 'Can access the AI Validation Debug page', 'Pages')
|
||||
ON CONFLICT (code) DO NOTHING;
|
||||
|
||||
-- Settings section permissions
|
||||
INSERT INTO permissions (name, code, description, category) VALUES
|
||||
('Data Management', 'settings:data_management', 'Access to the Data Management settings section', 'Settings'),
|
||||
('Stock Management', 'settings:stock_management', 'Access to the Stock Management settings section', 'Settings'),
|
||||
('Performance Metrics', 'settings:performance_metrics', 'Access to the Performance Metrics settings section', 'Settings'),
|
||||
('Calculation Settings', 'settings:calculation_settings', 'Access to the Calculation Settings section', 'Settings'),
|
||||
('Template Management', 'settings:templates', 'Access to the Template Management settings section', 'Settings'),
|
||||
('User Management', 'settings:user_management', 'Access to the User Management settings section', 'Settings')
|
||||
ON CONFLICT (code) DO NOTHING;
|
||||
|
||||
-- Set any existing users as admin
|
||||
UPDATE users SET is_admin = TRUE WHERE is_admin IS NULL;
|
||||
|
||||
-- Grant all permissions to admin users
|
||||
INSERT INTO user_permissions (user_id, permission_id)
|
||||
SELECT u.id, p.id
|
||||
FROM users u, permissions p
|
||||
WHERE u.is_admin = TRUE
|
||||
ON CONFLICT DO NOTHING;
|
||||
+67
-118
@@ -1,135 +1,84 @@
|
||||
const express = require('express');
|
||||
const bcrypt = require('bcrypt');
|
||||
const jwt = require('jsonwebtoken');
|
||||
const cors = require('cors');
|
||||
const mysql = require('mysql2/promise');
|
||||
require('dotenv').config({ path: '../.env' });
|
||||
import 'dotenv/config';
|
||||
import express from 'express';
|
||||
import cors from 'cors';
|
||||
import pg from 'pg';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const { Pool } = pg;
|
||||
import { dirname, resolve as resolvePath } from 'node:path';
|
||||
import { config as loadEnv } from 'dotenv';
|
||||
|
||||
import { corsOptions } from '../shared/cors/policy.js';
|
||||
import { requestLog } from '../shared/logging/request-log.js';
|
||||
import { logger } from '../shared/logging/logger.js';
|
||||
import { errorHandler } from '../shared/errors/handler.js';
|
||||
import { loginLimiter, verifyLimiter } from '../shared/rate-limit/login.js';
|
||||
import { extractBearerToken, verifyToken, TokenError } from '../shared/auth/verify.js';
|
||||
|
||||
import { createAuthRoutes } from './routes.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// auth/ lives at inventory-server/auth/, so .env one level up
|
||||
loadEnv({ path: resolvePath(__dirname, '../.env') });
|
||||
|
||||
if (!process.env.JWT_SECRET) {
|
||||
logger.error('JWT_SECRET is not set; refusing to start');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
logger.info({
|
||||
host: process.env.DB_HOST,
|
||||
user: process.env.DB_USER,
|
||||
database: process.env.DB_NAME,
|
||||
port: process.env.DB_PORT,
|
||||
auth_port: process.env.AUTH_PORT,
|
||||
}, 'starting auth server');
|
||||
|
||||
const app = express();
|
||||
const PORT = process.env.AUTH_PORT || 3011;
|
||||
const port = Number(process.env.AUTH_PORT) || 3011;
|
||||
|
||||
// Database configuration
|
||||
const dbConfig = {
|
||||
const pool = new Pool({
|
||||
host: process.env.DB_HOST,
|
||||
user: process.env.DB_USER,
|
||||
password: process.env.DB_PASSWORD,
|
||||
database: process.env.DB_NAME,
|
||||
};
|
||||
|
||||
// Create a connection pool
|
||||
const pool = mysql.createPool(dbConfig);
|
||||
|
||||
app.use(cors({
|
||||
origin: [
|
||||
'https://inventory.kent.pw',
|
||||
'http://localhost:5173',
|
||||
'http://127.0.0.1:5173',
|
||||
/^http:\/\/192\.168\.\d+\.\d+(:\d+)?$/,
|
||||
/^http:\/\/10\.\d+\.\d+\.\d+(:\d+)?$/
|
||||
],
|
||||
methods: ['GET', 'POST', 'OPTIONS'],
|
||||
allowedHeaders: ['Content-Type', 'Authorization', 'X-Requested-With'],
|
||||
credentials: true,
|
||||
exposedHeaders: ['set-cookie']
|
||||
}));
|
||||
app.use(express.json());
|
||||
|
||||
// Debug middleware to log request details
|
||||
app.use((req, res, next) => {
|
||||
console.log('Request details:', {
|
||||
method: req.method,
|
||||
url: req.url,
|
||||
origin: req.get('Origin'),
|
||||
headers: req.headers,
|
||||
body: req.body,
|
||||
});
|
||||
next();
|
||||
port: Number(process.env.DB_PORT) || 5432,
|
||||
});
|
||||
|
||||
// Registration endpoint
|
||||
app.post('/register', async (req, res) => {
|
||||
app.use(requestLog());
|
||||
app.use(express.json({ limit: '1mb' }));
|
||||
app.use(cors(corsOptions));
|
||||
|
||||
// Caddy forward_auth target: JWT signature check only, no DB hit.
|
||||
// Returns 200 with X-User-Id / X-User-Username on success; 401 otherwise.
|
||||
// Per-service middleware re-verifies independently; these headers are informational.
|
||||
app.all('/verify', verifyLimiter, (req, res) => {
|
||||
try {
|
||||
const { username, password } = req.body;
|
||||
const hashedPassword = await bcrypt.hash(password, 10);
|
||||
|
||||
const connection = await pool.getConnection();
|
||||
await connection.query('INSERT INTO users (username, password) VALUES (?, ?)', [username, hashedPassword]);
|
||||
connection.release();
|
||||
|
||||
res.status(201).json({ message: 'User registered successfully' });
|
||||
} catch (error) {
|
||||
console.error('Registration error:', error);
|
||||
res.status(500).json({ error: 'Registration failed' });
|
||||
const token = extractBearerToken(req.headers.authorization);
|
||||
const decoded = verifyToken(token, process.env.JWT_SECRET);
|
||||
res.set('X-User-Id', String(decoded.userId));
|
||||
if (decoded.username) res.set('X-User-Username', decoded.username);
|
||||
res.status(200).end();
|
||||
} catch (err) {
|
||||
if (err instanceof TokenError) {
|
||||
return res.status(401).json({ error: err.message });
|
||||
}
|
||||
res.status(401).json({ error: 'Invalid token' });
|
||||
}
|
||||
});
|
||||
|
||||
// Login endpoint
|
||||
app.post('/login', async (req, res) => {
|
||||
try {
|
||||
const { username, password } = req.body;
|
||||
console.log(`Login attempt for user: ${username}`);
|
||||
// Login route gets its own rate limiter to slow credential stuffing.
|
||||
app.use('/login', loginLimiter);
|
||||
|
||||
const connection = await pool.getConnection();
|
||||
const [rows] = await connection.query(
|
||||
'SELECT * FROM users WHERE username = ?',
|
||||
[username],
|
||||
);
|
||||
connection.release();
|
||||
// Mount user-management + /login + /me from routes.js
|
||||
app.use('/', createAuthRoutes({ pool }));
|
||||
|
||||
if (rows.length === 1) {
|
||||
const user = rows[0];
|
||||
const passwordMatch = await bcrypt.compare(password, user.password);
|
||||
app.get('/health', (req, res) => res.json({ status: 'healthy' }));
|
||||
|
||||
if (passwordMatch) {
|
||||
console.log(`User ${username} authenticated successfully`);
|
||||
const token = jwt.sign(
|
||||
{ username: user.username },
|
||||
process.env.JWT_SECRET,
|
||||
{ expiresIn: '1h' },
|
||||
);
|
||||
res.json({ token });
|
||||
} else {
|
||||
console.error(`Invalid password for user: ${username}`);
|
||||
res.status(401).json({ error: 'Invalid credentials' });
|
||||
}
|
||||
} else {
|
||||
console.error(`User not found: ${username}`);
|
||||
res.status(401).json({ error: 'Invalid credentials' });
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Login error:', error);
|
||||
res.status(500).json({ error: 'Login failed' });
|
||||
}
|
||||
});
|
||||
|
||||
// Protected endpoint example
|
||||
app.get('/protected', async (req, res) => {
|
||||
const authHeader = req.headers.authorization;
|
||||
if (!authHeader) {
|
||||
return res.status(401).json({ error: 'Unauthorized' });
|
||||
}
|
||||
|
||||
const token = authHeader.split(' ')[1];
|
||||
try {
|
||||
const decoded = jwt.verify(token, process.env.JWT_SECRET);
|
||||
|
||||
// Optionally, you can fetch the user from the database here
|
||||
// to verify that the user still exists or to get more user information
|
||||
const connection = await pool.getConnection();
|
||||
const [rows] = await connection.query('SELECT * FROM users WHERE username = ?', [decoded.username]);
|
||||
connection.release();
|
||||
|
||||
if (rows.length === 0) {
|
||||
return res.status(401).json({ error: 'User not found' });
|
||||
}
|
||||
|
||||
res.json({ message: 'Protected resource accessed', user: decoded });
|
||||
} catch (error) {
|
||||
console.error('Protected endpoint error:', error);
|
||||
res.status(403).json({ error: 'Invalid token' });
|
||||
}
|
||||
});
|
||||
|
||||
app.listen(PORT, "0.0.0.0", () => {
|
||||
console.log(`Auth server running on port ${PORT}`);
|
||||
app.use(errorHandler);
|
||||
|
||||
app.listen(port, () => {
|
||||
logger.info({ port }, 'auth server listening');
|
||||
});
|
||||
@@ -0,0 +1,45 @@
|
||||
-- PostgreSQL Database Creation Script for New Server
|
||||
-- Run as: sudo -u postgres psql -f create-new-database.sql
|
||||
|
||||
-- Terminate all connections to the database (if it exists)
|
||||
SELECT pg_terminate_backend(pid)
|
||||
FROM pg_stat_activity
|
||||
WHERE datname = 'rocketchat_converted' AND pid <> pg_backend_pid();
|
||||
|
||||
-- Drop the database if it exists
|
||||
DROP DATABASE IF EXISTS rocketchat_converted;
|
||||
|
||||
-- Create fresh database
|
||||
CREATE DATABASE rocketchat_converted;
|
||||
|
||||
-- Create user (if not exists) - UPDATE PASSWORD BEFORE RUNNING!
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_user WHERE usename = 'rocketchat_user') THEN
|
||||
CREATE USER rocketchat_user WITH PASSWORD 'HKjLgt23gWuPXzEAn3rW';
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- Grant database privileges
|
||||
GRANT CONNECT ON DATABASE rocketchat_converted TO rocketchat_user;
|
||||
GRANT CREATE ON DATABASE rocketchat_converted TO rocketchat_user;
|
||||
|
||||
-- Connect to the new database
|
||||
\c rocketchat_converted;
|
||||
|
||||
-- Grant schema privileges
|
||||
GRANT CREATE ON SCHEMA public TO rocketchat_user;
|
||||
GRANT USAGE ON SCHEMA public TO rocketchat_user;
|
||||
|
||||
-- Grant privileges on all future tables and sequences
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO rocketchat_user;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO rocketchat_user;
|
||||
|
||||
-- Display success message
|
||||
\echo 'Database created successfully!'
|
||||
\echo 'IMPORTANT: Update the password for rocketchat_user before proceeding'
|
||||
\echo 'Next steps:'
|
||||
\echo '1. Update the password in this file'
|
||||
\echo '2. Run export-chat-data.sh on your current server'
|
||||
\echo '3. Transfer the exported files to this server'
|
||||
\echo '4. Run import-chat-data.sh on this server'
|
||||
@@ -0,0 +1,881 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MongoDB to PostgreSQL Converter for Rocket.Chat
|
||||
Converts MongoDB BSON export files to PostgreSQL database
|
||||
|
||||
Usage:
|
||||
python3 mongo_to_postgres_converter.py \
|
||||
--mongo-path db/database/62df06d44234d20001289144 \
|
||||
--pg-database rocketchat_converted \
|
||||
--pg-user rocketchat_user \
|
||||
--pg-password your_password \
|
||||
--debug
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import struct
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Optional
|
||||
import argparse
|
||||
import traceback
|
||||
|
||||
# Auto-install dependencies if needed
|
||||
try:
|
||||
import bson
|
||||
import psycopg2
|
||||
except ImportError:
|
||||
print("Installing required packages...")
|
||||
subprocess.check_call([sys.executable, "-m", "pip", "install", "pymongo", "psycopg2-binary"])
|
||||
import bson
|
||||
import psycopg2
|
||||
|
||||
class MongoToPostgresConverter:
|
||||
def __init__(self, mongo_db_path: str, postgres_config: Dict[str, str], debug_mode: bool = False, debug_collections: List[str] = None):
|
||||
self.mongo_db_path = Path(mongo_db_path)
|
||||
self.postgres_config = postgres_config
|
||||
self.debug_mode = debug_mode
|
||||
self.debug_collections = debug_collections or []
|
||||
self.collections = {}
|
||||
self.schema_info = {}
|
||||
self.error_log = {}
|
||||
|
||||
def log_debug(self, message: str, collection: str = None):
|
||||
"""Log debug messages if debug mode is enabled and collection is in debug list"""
|
||||
if self.debug_mode and (not self.debug_collections or collection in self.debug_collections):
|
||||
print(f"DEBUG: {message}")
|
||||
|
||||
def log_error(self, collection: str, error_type: str, details: str):
|
||||
"""Log detailed error information"""
|
||||
if collection not in self.error_log:
|
||||
self.error_log[collection] = []
|
||||
self.error_log[collection].append({
|
||||
'type': error_type,
|
||||
'details': details,
|
||||
'timestamp': datetime.now().isoformat()
|
||||
})
|
||||
|
||||
def sample_documents(self, collection_name: str, max_samples: int = 3) -> List[Dict]:
|
||||
"""Sample documents from a collection for debugging"""
|
||||
if not self.debug_mode or (self.debug_collections and collection_name not in self.debug_collections):
|
||||
return []
|
||||
|
||||
print(f"\n🔍 Sampling documents from {collection_name}:")
|
||||
|
||||
bson_file = self.collections[collection_name]['bson_file']
|
||||
if bson_file.stat().st_size == 0:
|
||||
print(" Collection is empty")
|
||||
return []
|
||||
|
||||
samples = []
|
||||
|
||||
try:
|
||||
with open(bson_file, 'rb') as f:
|
||||
sample_count = 0
|
||||
while sample_count < max_samples:
|
||||
try:
|
||||
doc_size = int.from_bytes(f.read(4), byteorder='little')
|
||||
if doc_size <= 0:
|
||||
break
|
||||
f.seek(-4, 1)
|
||||
doc_bytes = f.read(doc_size)
|
||||
if len(doc_bytes) != doc_size:
|
||||
break
|
||||
|
||||
doc = bson.decode(doc_bytes)
|
||||
samples.append(doc)
|
||||
sample_count += 1
|
||||
|
||||
print(f" Sample {sample_count} - Keys: {list(doc.keys())}")
|
||||
# Show a few key fields with their types and truncated values
|
||||
for key, value in list(doc.items())[:3]:
|
||||
value_preview = str(value)[:50] + "..." if len(str(value)) > 50 else str(value)
|
||||
print(f" {key}: {type(value).__name__} = {value_preview}")
|
||||
if len(doc) > 3:
|
||||
print(f" ... and {len(doc) - 3} more fields")
|
||||
print()
|
||||
|
||||
except (bson.InvalidBSON, struct.error, OSError) as e:
|
||||
self.log_error(collection_name, 'document_parsing', str(e))
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
self.log_error(collection_name, 'file_reading', str(e))
|
||||
print(f" Error reading collection: {e}")
|
||||
|
||||
return samples
|
||||
|
||||
def discover_collections(self):
|
||||
"""Discover all BSON files and their metadata"""
|
||||
print("Discovering MongoDB collections...")
|
||||
|
||||
for bson_file in self.mongo_db_path.glob("*.bson"):
|
||||
collection_name = bson_file.stem
|
||||
metadata_file = bson_file.with_suffix(".metadata.json")
|
||||
|
||||
# Read metadata if available
|
||||
metadata = {}
|
||||
if metadata_file.exists():
|
||||
try:
|
||||
with open(metadata_file, 'r', encoding='utf-8') as f:
|
||||
metadata = json.load(f)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as e:
|
||||
print(f"Warning: Could not read metadata for {collection_name}: {e}")
|
||||
metadata = {}
|
||||
|
||||
# Get file size and document count estimate
|
||||
file_size = bson_file.stat().st_size
|
||||
doc_count = self._estimate_document_count(bson_file)
|
||||
|
||||
self.collections[collection_name] = {
|
||||
'bson_file': bson_file,
|
||||
'metadata': metadata,
|
||||
'file_size': file_size,
|
||||
'estimated_docs': doc_count
|
||||
}
|
||||
|
||||
print(f"Found {len(self.collections)} collections")
|
||||
for name, info in self.collections.items():
|
||||
print(f" - {name}: {info['file_size']/1024/1024:.1f}MB (~{info['estimated_docs']} docs)")
|
||||
|
||||
def _estimate_document_count(self, bson_file: Path) -> int:
|
||||
"""Estimate document count by reading first few documents"""
|
||||
if bson_file.stat().st_size == 0:
|
||||
return 0
|
||||
|
||||
try:
|
||||
with open(bson_file, 'rb') as f:
|
||||
docs_sampled = 0
|
||||
bytes_sampled = 0
|
||||
max_sample_size = min(1024 * 1024, bson_file.stat().st_size) # 1MB or file size
|
||||
|
||||
while bytes_sampled < max_sample_size:
|
||||
try:
|
||||
doc_size = int.from_bytes(f.read(4), byteorder='little')
|
||||
if doc_size <= 0 or doc_size > 16 * 1024 * 1024: # MongoDB doc size limit
|
||||
break
|
||||
f.seek(-4, 1) # Go back
|
||||
doc_bytes = f.read(doc_size)
|
||||
if len(doc_bytes) != doc_size:
|
||||
break
|
||||
bson.decode(doc_bytes) # Validate it's a valid BSON document
|
||||
docs_sampled += 1
|
||||
bytes_sampled += doc_size
|
||||
except (bson.InvalidBSON, struct.error, OSError):
|
||||
break
|
||||
|
||||
if docs_sampled > 0 and bytes_sampled > 0:
|
||||
avg_doc_size = bytes_sampled / docs_sampled
|
||||
return int(bson_file.stat().st_size / avg_doc_size)
|
||||
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return 0
|
||||
|
||||
def analyze_schema(self, collection_name: str, sample_size: int = 100) -> Dict[str, Any]:
|
||||
"""Analyze collection schema by sampling documents"""
|
||||
print(f"Analyzing schema for {collection_name}...")
|
||||
|
||||
bson_file = self.collections[collection_name]['bson_file']
|
||||
if bson_file.stat().st_size == 0:
|
||||
return {}
|
||||
|
||||
schema = {}
|
||||
docs_analyzed = 0
|
||||
|
||||
try:
|
||||
with open(bson_file, 'rb') as f:
|
||||
while docs_analyzed < sample_size:
|
||||
try:
|
||||
doc_size = int.from_bytes(f.read(4), byteorder='little')
|
||||
if doc_size <= 0:
|
||||
break
|
||||
f.seek(-4, 1)
|
||||
doc_bytes = f.read(doc_size)
|
||||
if len(doc_bytes) != doc_size:
|
||||
break
|
||||
|
||||
doc = bson.decode(doc_bytes)
|
||||
self._analyze_document_schema(doc, schema)
|
||||
docs_analyzed += 1
|
||||
|
||||
except (bson.InvalidBSON, struct.error, OSError):
|
||||
break
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error analyzing {collection_name}: {e}")
|
||||
|
||||
self.schema_info[collection_name] = schema
|
||||
return schema
|
||||
|
||||
def _analyze_document_schema(self, doc: Dict[str, Any], schema: Dict[str, Any], prefix: str = ""):
|
||||
"""Recursively analyze document structure"""
|
||||
for key, value in doc.items():
|
||||
full_key = f"{prefix}.{key}" if prefix else key
|
||||
|
||||
if full_key not in schema:
|
||||
schema[full_key] = {
|
||||
'types': set(),
|
||||
'null_count': 0,
|
||||
'total_count': 0,
|
||||
'is_array': False,
|
||||
'nested_schema': {}
|
||||
}
|
||||
|
||||
schema[full_key]['total_count'] += 1
|
||||
|
||||
if value is None:
|
||||
schema[full_key]['null_count'] += 1
|
||||
schema[full_key]['types'].add('null')
|
||||
elif isinstance(value, dict):
|
||||
schema[full_key]['types'].add('object')
|
||||
if 'nested_schema' not in schema[full_key]:
|
||||
schema[full_key]['nested_schema'] = {}
|
||||
self._analyze_document_schema(value, schema[full_key]['nested_schema'])
|
||||
elif isinstance(value, list):
|
||||
schema[full_key]['types'].add('array')
|
||||
schema[full_key]['is_array'] = True
|
||||
if value and isinstance(value[0], dict):
|
||||
if 'array_item_schema' not in schema[full_key]:
|
||||
schema[full_key]['array_item_schema'] = {}
|
||||
for item in value[:5]: # Sample first 5 items
|
||||
if isinstance(item, dict):
|
||||
self._analyze_document_schema(item, schema[full_key]['array_item_schema'])
|
||||
else:
|
||||
schema[full_key]['types'].add(type(value).__name__)
|
||||
|
||||
def generate_postgres_schema(self) -> Dict[str, str]:
|
||||
"""Generate PostgreSQL CREATE TABLE statements"""
|
||||
print("Generating PostgreSQL schema...")
|
||||
|
||||
table_definitions = {}
|
||||
|
||||
for collection_name, schema in self.schema_info.items():
|
||||
if not schema: # Empty collection
|
||||
continue
|
||||
|
||||
table_name = self._sanitize_table_name(collection_name)
|
||||
columns = []
|
||||
|
||||
# Always add an id column (PostgreSQL doesn't use _id like MongoDB)
|
||||
columns.append("id SERIAL PRIMARY KEY")
|
||||
|
||||
for field_name, field_info in schema.items():
|
||||
if field_name == '_id':
|
||||
columns.append("mongo_id TEXT") # Always allow NULL for mongo_id
|
||||
continue
|
||||
|
||||
col_name = self._sanitize_column_name(field_name)
|
||||
|
||||
# Handle conflicts with PostgreSQL auto-generated columns
|
||||
if col_name in ['id', 'mongo_id', 'created_at', 'updated_at']:
|
||||
col_name = f"field_{col_name}"
|
||||
|
||||
col_type = self._determine_postgres_type(field_info)
|
||||
|
||||
# Make all fields nullable by default to avoid constraint violations
|
||||
columns.append(f"{col_name} {col_type}")
|
||||
|
||||
# Add metadata columns
|
||||
columns.extend([
|
||||
"created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP",
|
||||
"updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP"
|
||||
])
|
||||
|
||||
column_definitions = ',\n '.join(columns)
|
||||
table_sql = f"""
|
||||
CREATE TABLE IF NOT EXISTS {table_name} (
|
||||
{column_definitions}
|
||||
);
|
||||
|
||||
-- Create indexes based on MongoDB indexes
|
||||
"""
|
||||
|
||||
# Get list of actual columns that will exist in the table
|
||||
existing_columns = set(['id', 'mongo_id', 'created_at', 'updated_at'])
|
||||
for field_name in schema.keys():
|
||||
if field_name != '_id':
|
||||
col_name = self._sanitize_column_name(field_name)
|
||||
# Handle conflicts with PostgreSQL auto-generated columns
|
||||
if col_name in ['id', 'mongo_id', 'created_at', 'updated_at']:
|
||||
col_name = f"field_{col_name}"
|
||||
existing_columns.add(col_name)
|
||||
|
||||
# Add indexes from MongoDB metadata
|
||||
metadata = self.collections[collection_name].get('metadata', {})
|
||||
indexes = metadata.get('indexes', [])
|
||||
|
||||
for index in indexes:
|
||||
if index['name'] != '_id_': # Skip the default _id index
|
||||
# Sanitize index name - remove special characters
|
||||
sanitized_index_name = re.sub(r'[^a-zA-Z0-9_]', '_', index['name'])
|
||||
index_name = f"idx_{table_name}_{sanitized_index_name}"
|
||||
index_keys = list(index['key'].keys())
|
||||
if index_keys:
|
||||
sanitized_keys = []
|
||||
for key in index_keys:
|
||||
if key != '_id':
|
||||
sanitized_key = self._sanitize_column_name(key)
|
||||
# Handle conflicts with PostgreSQL auto-generated columns
|
||||
if sanitized_key in ['id', 'mongo_id', 'created_at', 'updated_at']:
|
||||
sanitized_key = f"field_{sanitized_key}"
|
||||
# Only add if the column actually exists in our table
|
||||
if sanitized_key in existing_columns:
|
||||
sanitized_keys.append(sanitized_key)
|
||||
|
||||
if sanitized_keys:
|
||||
table_sql += f"CREATE INDEX IF NOT EXISTS {index_name} ON {table_name} ({', '.join(sanitized_keys)});\n"
|
||||
|
||||
table_definitions[collection_name] = table_sql
|
||||
|
||||
return table_definitions
|
||||
|
||||
def _sanitize_table_name(self, name: str) -> str:
|
||||
"""Convert MongoDB collection name to PostgreSQL table name"""
|
||||
# Remove rocketchat_ prefix if present
|
||||
if name.startswith('rocketchat_'):
|
||||
name = name[11:]
|
||||
|
||||
# Replace special characters with underscores
|
||||
name = re.sub(r'[^a-zA-Z0-9_]', '_', name)
|
||||
|
||||
# Ensure it starts with a letter
|
||||
if name and name[0].isdigit():
|
||||
name = 'table_' + name
|
||||
|
||||
return name.lower()
|
||||
|
||||
def _sanitize_column_name(self, name: str) -> str:
|
||||
"""Convert MongoDB field name to PostgreSQL column name"""
|
||||
# Handle nested field names (convert dots to underscores)
|
||||
name = name.replace('.', '_')
|
||||
|
||||
# Replace special characters with underscores
|
||||
name = re.sub(r'[^a-zA-Z0-9_]', '_', name)
|
||||
|
||||
# Ensure it starts with a letter or underscore
|
||||
if name and name[0].isdigit():
|
||||
name = 'col_' + name
|
||||
|
||||
# Handle PostgreSQL reserved words
|
||||
reserved = {
|
||||
'user', 'order', 'group', 'table', 'index', 'key', 'value', 'date', 'time', 'timestamp',
|
||||
'default', 'select', 'from', 'where', 'insert', 'update', 'delete', 'create', 'drop',
|
||||
'alter', 'grant', 'revoke', 'commit', 'rollback', 'begin', 'end', 'case', 'when',
|
||||
'then', 'else', 'if', 'null', 'not', 'and', 'or', 'in', 'exists', 'between',
|
||||
'like', 'limit', 'offset', 'union', 'join', 'inner', 'outer', 'left', 'right',
|
||||
'full', 'cross', 'natural', 'on', 'using', 'distinct', 'all', 'any', 'some',
|
||||
'desc', 'asc', 'primary', 'foreign', 'references', 'constraint', 'unique',
|
||||
'check', 'cascade', 'restrict', 'action', 'match', 'partial', 'full'
|
||||
}
|
||||
if name.lower() in reserved:
|
||||
name = name + '_col'
|
||||
|
||||
return name.lower()
|
||||
|
||||
def _determine_postgres_type(self, field_info: Dict[str, Any]) -> str:
|
||||
"""Determine PostgreSQL column type from MongoDB field analysis with improved logic"""
|
||||
types = field_info['types']
|
||||
|
||||
# Convert set to list for easier checking
|
||||
type_list = list(types)
|
||||
|
||||
# If there's only one type (excluding null), use specific typing
|
||||
non_null_types = [t for t in type_list if t != 'null']
|
||||
|
||||
if len(non_null_types) == 1:
|
||||
single_type = non_null_types[0]
|
||||
if single_type == 'bool':
|
||||
return 'BOOLEAN'
|
||||
elif single_type == 'int':
|
||||
return 'INTEGER'
|
||||
elif single_type == 'float':
|
||||
return 'NUMERIC'
|
||||
elif single_type == 'str':
|
||||
return 'TEXT'
|
||||
elif single_type == 'datetime':
|
||||
return 'TIMESTAMP'
|
||||
elif single_type == 'ObjectId':
|
||||
return 'TEXT'
|
||||
|
||||
# Handle mixed types more conservatively
|
||||
if 'array' in types or field_info.get('is_array', False):
|
||||
return 'JSONB' # Arrays always go to JSONB
|
||||
elif 'object' in types:
|
||||
return 'JSONB' # Objects always go to JSONB
|
||||
elif len(non_null_types) > 1:
|
||||
# Multiple non-null types - check for common combinations
|
||||
if set(non_null_types) <= {'int', 'float'}:
|
||||
return 'NUMERIC' # Can handle both int and float
|
||||
elif set(non_null_types) <= {'bool', 'str'}:
|
||||
return 'TEXT' # Convert everything to text
|
||||
elif set(non_null_types) <= {'str', 'ObjectId'}:
|
||||
return 'TEXT' # Both are string-like
|
||||
else:
|
||||
return 'JSONB' # Complex mixed types go to JSONB
|
||||
elif 'ObjectId' in types:
|
||||
return 'TEXT'
|
||||
elif 'datetime' in types:
|
||||
return 'TIMESTAMP'
|
||||
elif 'bool' in types:
|
||||
return 'BOOLEAN'
|
||||
elif 'int' in types:
|
||||
return 'INTEGER'
|
||||
elif 'float' in types:
|
||||
return 'NUMERIC'
|
||||
elif 'str' in types:
|
||||
return 'TEXT'
|
||||
else:
|
||||
return 'TEXT' # Default fallback
|
||||
|
||||
def create_postgres_database(self, table_definitions: Dict[str, str]):
|
||||
"""Create PostgreSQL database and tables"""
|
||||
print("Creating PostgreSQL database schema...")
|
||||
|
||||
try:
|
||||
# Connect to PostgreSQL
|
||||
conn = psycopg2.connect(**self.postgres_config)
|
||||
conn.autocommit = True
|
||||
cursor = conn.cursor()
|
||||
|
||||
# Create tables
|
||||
for collection_name, table_sql in table_definitions.items():
|
||||
print(f"Creating table for {collection_name}...")
|
||||
cursor.execute(table_sql)
|
||||
|
||||
cursor.close()
|
||||
conn.close()
|
||||
print("Database schema created successfully!")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error creating database schema: {e}")
|
||||
raise
|
||||
|
||||
def convert_and_insert_data(self, batch_size: int = 1000):
|
||||
"""Convert BSON data and insert into PostgreSQL"""
|
||||
print("Converting and inserting data...")
|
||||
|
||||
try:
|
||||
conn = psycopg2.connect(**self.postgres_config)
|
||||
conn.autocommit = False
|
||||
|
||||
for collection_name in self.collections:
|
||||
print(f"Processing {collection_name}...")
|
||||
self._convert_collection(conn, collection_name, batch_size)
|
||||
|
||||
conn.close()
|
||||
print("Data conversion completed successfully!")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error converting data: {e}")
|
||||
raise
|
||||
|
||||
def _convert_collection(self, conn, collection_name: str, batch_size: int):
|
||||
"""Convert a single collection"""
|
||||
bson_file = self.collections[collection_name]['bson_file']
|
||||
|
||||
if bson_file.stat().st_size == 0:
|
||||
print(f" Skipping empty collection {collection_name}")
|
||||
return
|
||||
|
||||
table_name = self._sanitize_table_name(collection_name)
|
||||
cursor = conn.cursor()
|
||||
|
||||
batch = []
|
||||
total_inserted = 0
|
||||
errors = 0
|
||||
|
||||
try:
|
||||
with open(bson_file, 'rb') as f:
|
||||
while True:
|
||||
try:
|
||||
doc_size = int.from_bytes(f.read(4), byteorder='little')
|
||||
if doc_size <= 0:
|
||||
break
|
||||
f.seek(-4, 1)
|
||||
doc_bytes = f.read(doc_size)
|
||||
if len(doc_bytes) != doc_size:
|
||||
break
|
||||
|
||||
doc = bson.decode(doc_bytes)
|
||||
batch.append(doc)
|
||||
|
||||
if len(batch) >= batch_size:
|
||||
inserted, batch_errors = self._insert_batch(cursor, table_name, batch, collection_name)
|
||||
total_inserted += inserted
|
||||
errors += batch_errors
|
||||
batch = []
|
||||
conn.commit()
|
||||
if total_inserted % 5000 == 0: # Less frequent progress updates
|
||||
print(f" Inserted {total_inserted} documents...")
|
||||
|
||||
except (bson.InvalidBSON, struct.error, OSError):
|
||||
break
|
||||
|
||||
# Insert remaining documents
|
||||
if batch:
|
||||
inserted, batch_errors = self._insert_batch(cursor, table_name, batch, collection_name)
|
||||
total_inserted += inserted
|
||||
errors += batch_errors
|
||||
conn.commit()
|
||||
|
||||
if errors > 0:
|
||||
print(f" Completed {collection_name}: {total_inserted} documents inserted ({errors} errors)")
|
||||
else:
|
||||
print(f" Completed {collection_name}: {total_inserted} documents inserted")
|
||||
|
||||
except Exception as e:
|
||||
print(f" Error processing {collection_name}: {e}")
|
||||
conn.rollback()
|
||||
finally:
|
||||
cursor.close()
|
||||
|
||||
def _insert_batch(self, cursor, table_name: str, documents: List[Dict], collection_name: str):
|
||||
"""Insert a batch of documents with proper transaction handling"""
|
||||
if not documents:
|
||||
return 0, 0
|
||||
|
||||
# Get schema info for this collection
|
||||
schema = self.schema_info.get(collection_name, {})
|
||||
|
||||
# Build column list
|
||||
columns = ['mongo_id']
|
||||
for field_name in schema.keys():
|
||||
if field_name != '_id':
|
||||
col_name = self._sanitize_column_name(field_name)
|
||||
# Handle conflicts with PostgreSQL auto-generated columns
|
||||
if col_name in ['id', 'mongo_id', 'created_at', 'updated_at']:
|
||||
col_name = f"field_{col_name}"
|
||||
columns.append(col_name)
|
||||
|
||||
# Build INSERT statement
|
||||
placeholders = ', '.join(['%s'] * len(columns))
|
||||
sql = f"INSERT INTO {table_name} ({', '.join(columns)}) VALUES ({placeholders})"
|
||||
|
||||
self.log_debug(f"SQL: {sql}", collection_name)
|
||||
|
||||
# Convert documents to tuples
|
||||
rows = []
|
||||
errors = 0
|
||||
|
||||
for doc_idx, doc in enumerate(documents):
|
||||
try:
|
||||
row = []
|
||||
|
||||
# Add mongo_id
|
||||
row.append(str(doc.get('_id', '')))
|
||||
|
||||
# Add other fields
|
||||
for field_name in schema.keys():
|
||||
if field_name != '_id':
|
||||
try:
|
||||
value = self._get_nested_value(doc, field_name)
|
||||
converted_value = self._convert_value_for_postgres(value, field_name, schema)
|
||||
row.append(converted_value)
|
||||
except Exception as e:
|
||||
self.log_error(collection_name, 'field_conversion',
|
||||
f"Field '{field_name}' in doc {doc_idx}: {str(e)}")
|
||||
# Only show debug for collections we're focusing on
|
||||
if collection_name in self.debug_collections:
|
||||
print(f" ⚠️ Error converting field '{field_name}': {e}")
|
||||
row.append(None) # Use NULL for problematic fields
|
||||
|
||||
rows.append(tuple(row))
|
||||
|
||||
except Exception as e:
|
||||
self.log_error(collection_name, 'document_conversion', f"Document {doc_idx}: {str(e)}")
|
||||
errors += 1
|
||||
continue
|
||||
|
||||
# Execute batch insert
|
||||
if rows:
|
||||
try:
|
||||
cursor.executemany(sql, rows)
|
||||
return len(rows), errors
|
||||
except Exception as batch_error:
|
||||
self.log_error(collection_name, 'batch_insert', str(batch_error))
|
||||
|
||||
# Only show detailed debugging for targeted collections
|
||||
if collection_name in self.debug_collections:
|
||||
print(f" 🔴 Batch insert failed for {collection_name}: {batch_error}")
|
||||
print(" Trying individual inserts with rollback handling...")
|
||||
|
||||
# Rollback the failed transaction
|
||||
cursor.connection.rollback()
|
||||
|
||||
# Try inserting one by one in individual transactions
|
||||
success_count = 0
|
||||
for row_idx, row in enumerate(rows):
|
||||
try:
|
||||
cursor.execute(sql, row)
|
||||
cursor.connection.commit() # Commit each successful insert
|
||||
success_count += 1
|
||||
except Exception as row_error:
|
||||
cursor.connection.rollback() # Rollback failed insert
|
||||
self.log_error(collection_name, 'row_insert', f"Row {row_idx}: {str(row_error)}")
|
||||
|
||||
# Show detailed error only for the first few failures and only for targeted collections
|
||||
if collection_name in self.debug_collections and errors < 3:
|
||||
print(f" Row {row_idx} failed: {row_error}")
|
||||
print(f" Row data: {len(row)} values, expected {len(columns)} columns")
|
||||
|
||||
errors += 1
|
||||
continue
|
||||
return success_count, errors
|
||||
|
||||
return 0, errors
|
||||
|
||||
def _get_nested_value(self, doc: Dict, field_path: str):
|
||||
"""Get value from nested document using dot notation"""
|
||||
keys = field_path.split('.')
|
||||
value = doc
|
||||
|
||||
for key in keys:
|
||||
if isinstance(value, dict) and key in value:
|
||||
value = value[key]
|
||||
else:
|
||||
return None
|
||||
|
||||
return value
|
||||
|
||||
def _convert_value_for_postgres(self, value, field_name: str = None, schema: Dict = None):
|
||||
"""Convert MongoDB value to PostgreSQL compatible value with schema-aware conversion"""
|
||||
if value is None:
|
||||
return None
|
||||
|
||||
# Get the expected PostgreSQL type for this field if available
|
||||
expected_type = None
|
||||
if schema and field_name and field_name in schema:
|
||||
field_info = schema[field_name]
|
||||
expected_type = self._determine_postgres_type(field_info)
|
||||
|
||||
# Handle conversion based on expected type
|
||||
if expected_type == 'BOOLEAN':
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
elif isinstance(value, str):
|
||||
return value.lower() in ('true', '1', 'yes', 'on')
|
||||
elif isinstance(value, (int, float)):
|
||||
return bool(value)
|
||||
else:
|
||||
return None
|
||||
elif expected_type == 'INTEGER':
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
elif isinstance(value, float):
|
||||
return int(value)
|
||||
elif isinstance(value, str) and value.isdigit():
|
||||
return int(value)
|
||||
elif isinstance(value, bool):
|
||||
return int(value)
|
||||
else:
|
||||
return None
|
||||
elif expected_type == 'NUMERIC':
|
||||
if isinstance(value, (int, float)):
|
||||
return value
|
||||
elif isinstance(value, str):
|
||||
try:
|
||||
return float(value)
|
||||
except ValueError:
|
||||
return None
|
||||
elif isinstance(value, bool):
|
||||
return float(value)
|
||||
else:
|
||||
return None
|
||||
elif expected_type == 'TEXT':
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
elif value is not None:
|
||||
str_value = str(value)
|
||||
# Handle very long strings
|
||||
if len(str_value) > 65535:
|
||||
return str_value[:65535]
|
||||
return str_value
|
||||
else:
|
||||
return None
|
||||
elif expected_type == 'TIMESTAMP':
|
||||
if hasattr(value, 'isoformat'):
|
||||
return value.isoformat()
|
||||
elif isinstance(value, str):
|
||||
return value
|
||||
else:
|
||||
return str(value) if value is not None else None
|
||||
elif expected_type == 'JSONB':
|
||||
if isinstance(value, (dict, list)):
|
||||
return json.dumps(value, default=self._json_serializer)
|
||||
elif isinstance(value, str):
|
||||
# Check if it's already valid JSON
|
||||
try:
|
||||
json.loads(value)
|
||||
return value
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
# Not valid JSON, wrap it
|
||||
return json.dumps(value)
|
||||
else:
|
||||
return json.dumps(value, default=self._json_serializer)
|
||||
|
||||
# Fallback to original logic if no expected type or type not recognized
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
elif isinstance(value, (int, float)):
|
||||
return value
|
||||
elif isinstance(value, str):
|
||||
return value
|
||||
elif isinstance(value, (dict, list)):
|
||||
return json.dumps(value, default=self._json_serializer)
|
||||
elif hasattr(value, 'isoformat'): # datetime
|
||||
return value.isoformat()
|
||||
elif hasattr(value, '__str__'):
|
||||
str_value = str(value)
|
||||
if len(str_value) > 65535:
|
||||
return str_value[:65535]
|
||||
return str_value
|
||||
else:
|
||||
return str(value)
|
||||
|
||||
def _json_serializer(self, obj):
|
||||
"""Custom JSON serializer for complex objects with better error handling"""
|
||||
try:
|
||||
if hasattr(obj, 'isoformat'): # datetime
|
||||
return obj.isoformat()
|
||||
elif hasattr(obj, '__str__'):
|
||||
return str(obj)
|
||||
else:
|
||||
return None
|
||||
except Exception as e:
|
||||
self.log_debug(f"JSON serialization error: {e}")
|
||||
return str(obj)
|
||||
|
||||
def run_conversion(self, sample_size: int = 100, batch_size: int = 1000):
|
||||
"""Run the full conversion process with focused debugging"""
|
||||
print("Starting MongoDB to PostgreSQL conversion...")
|
||||
print("This will convert your Rocket.Chat database from MongoDB to PostgreSQL")
|
||||
if self.debug_mode:
|
||||
if self.debug_collections:
|
||||
print(f"🐛 DEBUG MODE: Focusing on collections: {', '.join(self.debug_collections)}")
|
||||
else:
|
||||
print("🐛 DEBUG MODE: All collections")
|
||||
print("=" * 70)
|
||||
|
||||
# Step 1: Discover collections
|
||||
self.discover_collections()
|
||||
|
||||
# Step 2: Analyze schemas
|
||||
print("\nAnalyzing collection schemas...")
|
||||
for collection_name in self.collections:
|
||||
self.analyze_schema(collection_name, sample_size)
|
||||
|
||||
# Sample problematic collections if debugging
|
||||
if self.debug_mode and self.debug_collections:
|
||||
for coll in self.debug_collections:
|
||||
if coll in self.collections:
|
||||
self.sample_documents(coll, 2)
|
||||
|
||||
# Step 3: Generate PostgreSQL schema
|
||||
table_definitions = self.generate_postgres_schema()
|
||||
|
||||
# Step 4: Create database schema
|
||||
self.create_postgres_database(table_definitions)
|
||||
|
||||
# Step 5: Convert and insert data
|
||||
self.convert_and_insert_data(batch_size)
|
||||
|
||||
# Step 6: Show error summary
|
||||
self._print_error_summary()
|
||||
|
||||
print("=" * 70)
|
||||
print("✅ Conversion completed!")
|
||||
print(f" Database: {self.postgres_config['database']}")
|
||||
print(f" Tables created: {len(table_definitions)}")
|
||||
|
||||
def _print_error_summary(self):
|
||||
"""Print a focused summary of errors"""
|
||||
if not self.error_log:
|
||||
print("\n✅ No errors encountered during conversion!")
|
||||
return
|
||||
|
||||
print("\n⚠️ ERROR SUMMARY:")
|
||||
print("=" * 50)
|
||||
|
||||
# Sort by error count descending
|
||||
sorted_collections = sorted(self.error_log.items(),
|
||||
key=lambda x: len(x[1]), reverse=True)
|
||||
|
||||
for collection, errors in sorted_collections:
|
||||
error_types = {}
|
||||
for error in errors:
|
||||
error_type = error['type']
|
||||
if error_type not in error_types:
|
||||
error_types[error_type] = []
|
||||
error_types[error_type].append(error['details'])
|
||||
|
||||
print(f"\n🔴 {collection} ({len(errors)} total errors):")
|
||||
for error_type, details_list in error_types.items():
|
||||
print(f" {error_type}: {len(details_list)} errors")
|
||||
|
||||
# Show sample errors for critical collections
|
||||
if collection in ['rocketchat_settings', 'rocketchat_room'] and len(details_list) > 0:
|
||||
print(f" Sample: {details_list[0][:100]}...")
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Convert MongoDB BSON export to PostgreSQL',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
# Basic usage
|
||||
python3 mongo_to_postgres_converter.py \\
|
||||
--mongo-path db/database/62df06d44234d20001289144 \\
|
||||
--pg-database rocketchat_converted \\
|
||||
--pg-user rocketchat_user \\
|
||||
--pg-password mypassword
|
||||
|
||||
# Debug specific failing collections
|
||||
python3 mongo_to_postgres_converter.py \\
|
||||
--mongo-path db/database/62df06d44234d20001289144 \\
|
||||
--pg-database rocketchat_converted \\
|
||||
--pg-user rocketchat_user \\
|
||||
--pg-password mypassword \\
|
||||
--debug-collections rocketchat_settings rocketchat_room
|
||||
|
||||
Before running this script:
|
||||
1. Run: sudo -u postgres psql -f reset_database.sql
|
||||
2. Update the password in reset_database.sql
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument('--mongo-path', required=True, help='Path to MongoDB export directory')
|
||||
parser.add_argument('--pg-host', default='localhost', help='PostgreSQL host (default: localhost)')
|
||||
parser.add_argument('--pg-port', default='5432', help='PostgreSQL port (default: 5432)')
|
||||
parser.add_argument('--pg-database', required=True, help='PostgreSQL database name')
|
||||
parser.add_argument('--pg-user', required=True, help='PostgreSQL username')
|
||||
parser.add_argument('--pg-password', required=True, help='PostgreSQL password')
|
||||
parser.add_argument('--sample-size', type=int, default=100, help='Number of documents to sample for schema analysis (default: 100)')
|
||||
parser.add_argument('--batch-size', type=int, default=1000, help='Batch size for data insertion (default: 1000)')
|
||||
parser.add_argument('--debug', action='store_true', help='Enable debug mode with detailed error logging')
|
||||
parser.add_argument('--debug-collections', nargs='*', help='Specific collections to debug (e.g., rocketchat_settings rocketchat_room)')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
postgres_config = {
|
||||
'host': args.pg_host,
|
||||
'port': args.pg_port,
|
||||
'database': args.pg_database,
|
||||
'user': args.pg_user,
|
||||
'password': args.pg_password
|
||||
}
|
||||
|
||||
# Enable debug mode if debug collections are specified
|
||||
debug_mode = args.debug or (args.debug_collections is not None)
|
||||
|
||||
converter = MongoToPostgresConverter(args.mongo_path, postgres_config, debug_mode, args.debug_collections)
|
||||
converter.run_conversion(args.sample_size, args.batch_size)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,41 @@
|
||||
-- PostgreSQL Database Reset Script for Rocket.Chat Import
|
||||
-- Run as: sudo -u postgres psql -f reset_database.sql
|
||||
|
||||
-- Terminate all connections to the database (force disconnect users)
|
||||
SELECT pg_terminate_backend(pid)
|
||||
FROM pg_stat_activity
|
||||
WHERE datname = 'rocketchat_converted' AND pid <> pg_backend_pid();
|
||||
|
||||
-- Drop the database if it exists
|
||||
DROP DATABASE IF EXISTS rocketchat_converted;
|
||||
|
||||
-- Create fresh database
|
||||
CREATE DATABASE rocketchat_converted;
|
||||
|
||||
-- Create user (if not exists)
|
||||
DO $$
|
||||
BEGIN
|
||||
IF NOT EXISTS (SELECT FROM pg_user WHERE usename = 'rocketchat_user') THEN
|
||||
CREATE USER rocketchat_user WITH PASSWORD 'HKjLgt23gWuPXzEAn3rW';
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- Grant database privileges
|
||||
GRANT CONNECT ON DATABASE rocketchat_converted TO rocketchat_user;
|
||||
GRANT CREATE ON DATABASE rocketchat_converted TO rocketchat_user;
|
||||
|
||||
-- Connect to the new database
|
||||
\c rocketchat_converted;
|
||||
|
||||
-- Grant schema privileges
|
||||
GRANT CREATE ON SCHEMA public TO rocketchat_user;
|
||||
GRANT USAGE ON SCHEMA public TO rocketchat_user;
|
||||
|
||||
-- Grant privileges on all future tables and sequences
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO rocketchat_user;
|
||||
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT USAGE, SELECT ON SEQUENCES TO rocketchat_user;
|
||||
|
||||
-- Display success message
|
||||
\echo 'Database reset completed successfully!'
|
||||
\echo 'You can now run the converter with:'
|
||||
\echo 'python3 mongo_to_postgres_converter.py --mongo-path db/database/62df06d44234d20001289144 --pg-database rocketchat_converted --pg-user rocketchat_user --pg-password your_password'
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Quick test script to verify the converter fixes work for problematic collections
|
||||
"""
|
||||
|
||||
from mongo_to_postgres_converter import MongoToPostgresConverter
|
||||
|
||||
def test_problematic_collections():
|
||||
print("🧪 Testing converter fixes for problematic collections...")
|
||||
|
||||
postgres_config = {
|
||||
'host': 'localhost',
|
||||
'port': '5432',
|
||||
'database': 'rocketchat_test',
|
||||
'user': 'rocketchat_user',
|
||||
'password': 'password123'
|
||||
}
|
||||
|
||||
converter = MongoToPostgresConverter(
|
||||
'db/database/62df06d44234d20001289144',
|
||||
postgres_config,
|
||||
debug_mode=True,
|
||||
debug_collections=['rocketchat_settings', 'rocketchat_room']
|
||||
)
|
||||
|
||||
# Test just discovery and schema analysis
|
||||
print("\n1. Testing collection discovery...")
|
||||
converter.discover_collections()
|
||||
|
||||
print("\n2. Testing schema analysis...")
|
||||
if 'rocketchat_settings' in converter.collections:
|
||||
settings_schema = converter.analyze_schema('rocketchat_settings', 10)
|
||||
print(f"Settings schema fields: {len(settings_schema)}")
|
||||
|
||||
# Check specific problematic fields
|
||||
if 'packageValue' in settings_schema:
|
||||
packagevalue_info = settings_schema['packageValue']
|
||||
pg_type = converter._determine_postgres_type(packagevalue_info)
|
||||
print(f"packageValue types: {packagevalue_info['types']} -> PostgreSQL: {pg_type}")
|
||||
|
||||
if 'rocketchat_room' in converter.collections:
|
||||
room_schema = converter.analyze_schema('rocketchat_room', 10)
|
||||
print(f"Room schema fields: {len(room_schema)}")
|
||||
|
||||
# Check specific problematic fields
|
||||
if 'sysMes' in room_schema:
|
||||
sysmes_info = room_schema['sysMes']
|
||||
pg_type = converter._determine_postgres_type(sysmes_info)
|
||||
print(f"sysMes types: {sysmes_info['types']} -> PostgreSQL: {pg_type}")
|
||||
|
||||
print("\n✅ Test completed - check the type mappings above!")
|
||||
|
||||
if __name__ == '__main__':
|
||||
test_problematic_collections()
|
||||
@@ -0,0 +1,147 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Chat Database Export Script
|
||||
# This script exports the chat database schema and data for migration
|
||||
|
||||
set -e # Exit on any error
|
||||
|
||||
echo "🚀 Starting chat database export..."
|
||||
|
||||
# Configuration - Update these values for your setup
|
||||
DB_HOST="${CHAT_DB_HOST:-localhost}"
|
||||
DB_PORT="${CHAT_DB_PORT:-5432}"
|
||||
DB_NAME="${CHAT_DB_NAME:-rocketchat_converted}"
|
||||
DB_USER="${CHAT_DB_USER:-rocketchat_user}"
|
||||
|
||||
# Check if database connection info is available
|
||||
if [ -z "$CHAT_DB_PASSWORD" ]; then
|
||||
echo "⚠️ CHAT_DB_PASSWORD environment variable not set"
|
||||
echo "Please set it with: export CHAT_DB_PASSWORD='your_password'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "📊 Database: $DB_NAME on $DB_HOST:$DB_PORT"
|
||||
|
||||
# Create export directory
|
||||
EXPORT_DIR="chat-migration-$(date +%Y%m%d-%H%M%S)"
|
||||
mkdir -p "$EXPORT_DIR"
|
||||
|
||||
echo "📁 Export directory: $EXPORT_DIR"
|
||||
|
||||
# Export database schema
|
||||
echo "📋 Exporting database schema..."
|
||||
PGPASSWORD="$CHAT_DB_PASSWORD" pg_dump \
|
||||
-h "$DB_HOST" \
|
||||
-p "$DB_PORT" \
|
||||
-U "$DB_USER" \
|
||||
-d "$DB_NAME" \
|
||||
--schema-only \
|
||||
--no-owner \
|
||||
--no-privileges \
|
||||
-f "$EXPORT_DIR/chat-schema.sql"
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "✅ Schema exported successfully"
|
||||
else
|
||||
echo "❌ Schema export failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Export database data
|
||||
echo "💾 Exporting database data..."
|
||||
PGPASSWORD="$CHAT_DB_PASSWORD" pg_dump \
|
||||
-h "$DB_HOST" \
|
||||
-p "$DB_PORT" \
|
||||
-U "$DB_USER" \
|
||||
-d "$DB_NAME" \
|
||||
--data-only \
|
||||
--no-owner \
|
||||
--no-privileges \
|
||||
--disable-triggers \
|
||||
--column-inserts \
|
||||
-f "$EXPORT_DIR/chat-data.sql"
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "✅ Data exported successfully"
|
||||
else
|
||||
echo "❌ Data export failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Export file uploads and avatars
|
||||
echo "📎 Exporting chat files (uploads and avatars)..."
|
||||
if [ -d "db-convert/db/files" ]; then
|
||||
cd db-convert/db
|
||||
tar -czf "../../$EXPORT_DIR/chat-files.tar.gz" files/
|
||||
cd ../..
|
||||
echo "✅ Files exported successfully"
|
||||
else
|
||||
echo "⚠️ No files directory found at db-convert/db/files"
|
||||
echo " This is normal if you have no file uploads"
|
||||
touch "$EXPORT_DIR/chat-files.tar.gz"
|
||||
fi
|
||||
|
||||
# Get table statistics for verification
|
||||
echo "📈 Generating export statistics..."
|
||||
PGPASSWORD="$CHAT_DB_PASSWORD" psql \
|
||||
-h "$DB_HOST" \
|
||||
-p "$DB_PORT" \
|
||||
-U "$DB_USER" \
|
||||
-d "$DB_NAME" \
|
||||
-c "
|
||||
SELECT
|
||||
schemaname,
|
||||
tablename,
|
||||
n_tup_ins as inserted_rows,
|
||||
n_tup_upd as updated_rows,
|
||||
n_tup_del as deleted_rows,
|
||||
n_live_tup as live_rows,
|
||||
n_dead_tup as dead_rows
|
||||
FROM pg_stat_user_tables
|
||||
ORDER BY n_live_tup DESC;
|
||||
" > "$EXPORT_DIR/table-stats.txt"
|
||||
|
||||
# Create export summary
|
||||
cat > "$EXPORT_DIR/export-summary.txt" << EOF
|
||||
Chat Database Export Summary
|
||||
===========================
|
||||
|
||||
Export Date: $(date)
|
||||
Database: $DB_NAME
|
||||
Host: $DB_HOST:$DB_PORT
|
||||
User: $DB_USER
|
||||
|
||||
Files Generated:
|
||||
- chat-schema.sql: Database schema (tables, indexes, constraints)
|
||||
- chat-data.sql: All table data
|
||||
- chat-files.tar.gz: Uploaded files and avatars
|
||||
- table-stats.txt: Database statistics
|
||||
- export-summary.txt: This summary
|
||||
|
||||
Next Steps:
|
||||
1. Transfer these files to your new server
|
||||
2. Run create-new-database.sql on the new server first
|
||||
3. Run import-chat-data.sh on the new server
|
||||
4. Update your application configuration
|
||||
5. Run verify-migration.js to validate the migration
|
||||
|
||||
Important Notes:
|
||||
- Keep these files secure as they contain your chat data
|
||||
- Ensure the new server has enough disk space
|
||||
- Plan for application downtime during the migration
|
||||
EOF
|
||||
|
||||
echo ""
|
||||
echo "🎉 Export completed successfully!"
|
||||
echo "📁 Files are in: $EXPORT_DIR/"
|
||||
echo ""
|
||||
echo "📋 Export Summary:"
|
||||
ls -lh "$EXPORT_DIR/"
|
||||
echo ""
|
||||
echo "🚚 Next steps:"
|
||||
echo "1. Transfer the $EXPORT_DIR/ directory to your new server"
|
||||
echo "2. Run create-new-database.sql on the new server (update password first!)"
|
||||
echo "3. Run import-chat-data.sh on the new server"
|
||||
echo ""
|
||||
echo "💡 To transfer files to new server:"
|
||||
echo " scp -r $EXPORT_DIR/ user@new-server:/tmp/"
|
||||
@@ -0,0 +1,167 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Chat Database Import Script
|
||||
# This script imports the chat database schema and data on the new server
|
||||
|
||||
set -e # Exit on any error
|
||||
|
||||
echo "🚀 Starting chat database import..."
|
||||
|
||||
# Configuration - Update these values for your new server
|
||||
DB_HOST="${CHAT_DB_HOST:-localhost}"
|
||||
DB_PORT="${CHAT_DB_PORT:-5432}"
|
||||
DB_NAME="${CHAT_DB_NAME:-rocketchat_converted}"
|
||||
DB_USER="${CHAT_DB_USER:-rocketchat_user}"
|
||||
|
||||
# Check if database connection info is available
|
||||
if [ -z "$CHAT_DB_PASSWORD" ]; then
|
||||
echo "⚠️ CHAT_DB_PASSWORD environment variable not set"
|
||||
echo "Please set it with: export CHAT_DB_PASSWORD='your_password'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Find the migration directory
|
||||
MIGRATION_DIR=""
|
||||
if [ -d "/tmp" ]; then
|
||||
MIGRATION_DIR=$(find /tmp -maxdepth 1 -name "chat-migration-*" -type d | head -1)
|
||||
fi
|
||||
|
||||
if [ -z "$MIGRATION_DIR" ]; then
|
||||
echo "❌ No migration directory found in /tmp/"
|
||||
echo "Please specify the migration directory:"
|
||||
read -p "Enter full path to migration directory: " MIGRATION_DIR
|
||||
fi
|
||||
|
||||
if [ ! -d "$MIGRATION_DIR" ]; then
|
||||
echo "❌ Migration directory not found: $MIGRATION_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "📁 Using migration directory: $MIGRATION_DIR"
|
||||
echo "📊 Target database: $DB_NAME on $DB_HOST:$DB_PORT"
|
||||
|
||||
# Verify required files exist
|
||||
REQUIRED_FILES=("chat-schema.sql" "chat-data.sql" "chat-files.tar.gz")
|
||||
for file in "${REQUIRED_FILES[@]}"; do
|
||||
if [ ! -f "$MIGRATION_DIR/$file" ]; then
|
||||
echo "❌ Required file not found: $MIGRATION_DIR/$file"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
echo "✅ All required files found"
|
||||
|
||||
# Test database connection
|
||||
echo "🔗 Testing database connection..."
|
||||
PGPASSWORD="$CHAT_DB_PASSWORD" psql \
|
||||
-h "$DB_HOST" \
|
||||
-p "$DB_PORT" \
|
||||
-U "$DB_USER" \
|
||||
-d "$DB_NAME" \
|
||||
-c "SELECT version();" > /dev/null
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "✅ Database connection successful"
|
||||
else
|
||||
echo "❌ Database connection failed"
|
||||
echo "Please ensure:"
|
||||
echo " 1. PostgreSQL is running"
|
||||
echo " 2. Database '$DB_NAME' exists"
|
||||
echo " 3. User '$DB_USER' has access"
|
||||
echo " 4. Password is correct"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Import database schema
|
||||
echo "📋 Importing database schema..."
|
||||
PGPASSWORD="$CHAT_DB_PASSWORD" psql \
|
||||
-h "$DB_HOST" \
|
||||
-p "$DB_PORT" \
|
||||
-U "$DB_USER" \
|
||||
-d "$DB_NAME" \
|
||||
-f "$MIGRATION_DIR/chat-schema.sql"
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "✅ Schema imported successfully"
|
||||
else
|
||||
echo "❌ Schema import failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Import database data
|
||||
echo "💾 Importing database data..."
|
||||
echo " This may take a while depending on data size..."
|
||||
|
||||
PGPASSWORD="$CHAT_DB_PASSWORD" psql \
|
||||
-h "$DB_HOST" \
|
||||
-p "$DB_PORT" \
|
||||
-U "$DB_USER" \
|
||||
-d "$DB_NAME" \
|
||||
-f "$MIGRATION_DIR/chat-data.sql"
|
||||
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "✅ Data imported successfully"
|
||||
else
|
||||
echo "❌ Data import failed"
|
||||
echo "Check the error messages above for details"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create files directory and import files
|
||||
echo "📎 Setting up files directory..."
|
||||
mkdir -p "db-convert/db"
|
||||
|
||||
if [ -s "$MIGRATION_DIR/chat-files.tar.gz" ]; then
|
||||
echo "📂 Extracting chat files..."
|
||||
cd db-convert/db
|
||||
tar -xzf "$MIGRATION_DIR/chat-files.tar.gz"
|
||||
cd ../..
|
||||
|
||||
# Set proper permissions
|
||||
if [ -d "db-convert/db/files" ]; then
|
||||
chmod -R 755 db-convert/db/files
|
||||
echo "✅ Files imported and permissions set"
|
||||
else
|
||||
echo "⚠️ Files directory not created properly"
|
||||
fi
|
||||
else
|
||||
echo "ℹ️ No files to import (empty archive)"
|
||||
mkdir -p "db-convert/db/files/uploads"
|
||||
mkdir -p "db-convert/db/files/avatars"
|
||||
fi
|
||||
|
||||
# Get final table statistics
|
||||
echo "📈 Generating import statistics..."
|
||||
PGPASSWORD="$CHAT_DB_PASSWORD" psql \
|
||||
-h "$DB_HOST" \
|
||||
-p "$DB_PORT" \
|
||||
-U "$DB_USER" \
|
||||
-d "$DB_NAME" \
|
||||
-c "
|
||||
SELECT
|
||||
tablename,
|
||||
n_live_tup as row_count
|
||||
FROM pg_stat_user_tables
|
||||
WHERE schemaname = 'public'
|
||||
ORDER BY n_live_tup DESC;
|
||||
"
|
||||
|
||||
# Create import summary
|
||||
echo ""
|
||||
echo "🎉 Import completed successfully!"
|
||||
echo ""
|
||||
echo "📋 Import Summary:"
|
||||
echo " Database: $DB_NAME"
|
||||
echo " Host: $DB_HOST:$DB_PORT"
|
||||
echo " Files location: $(pwd)/db-convert/db/files/"
|
||||
echo ""
|
||||
echo "🔍 Next steps:"
|
||||
echo "1. Update your application configuration to use this database"
|
||||
echo "2. Run verify-migration.js to validate the migration"
|
||||
echo "3. Test your application thoroughly"
|
||||
echo "4. Update DNS/load balancer to point to new server"
|
||||
echo ""
|
||||
echo "⚠️ Important:"
|
||||
echo "- Keep the original data as backup until migration is fully validated"
|
||||
echo "- Monitor the application closely after switching"
|
||||
echo "- Have a rollback plan ready"
|
||||
@@ -0,0 +1,86 @@
|
||||
# Chat Database Migration Guide
|
||||
|
||||
This guide will help you migrate your chat database from the current server to a new PostgreSQL server.
|
||||
|
||||
## Overview
|
||||
Your chat system uses:
|
||||
- Database: `rocketchat_converted` (PostgreSQL)
|
||||
- Main tables: users, message, room, uploads, avatars, subscription
|
||||
- File storage: db-convert/db/files/ directory with uploads and avatars
|
||||
- Environment configuration for database connection
|
||||
|
||||
## Migration Steps
|
||||
|
||||
### 1. Pre-Migration Setup
|
||||
|
||||
On your **new server**, ensure PostgreSQL is installed and running:
|
||||
```bash
|
||||
# Install PostgreSQL (if not already done)
|
||||
sudo apt update
|
||||
sudo apt install postgresql postgresql-contrib
|
||||
|
||||
# Start PostgreSQL service
|
||||
sudo systemctl start postgresql
|
||||
sudo systemctl enable postgresql
|
||||
```
|
||||
|
||||
### 2. Create Database Schema on New Server
|
||||
|
||||
Run the provided migration script:
|
||||
```bash
|
||||
# On new server
|
||||
sudo -u postgres psql -f create-new-database.sql
|
||||
```
|
||||
|
||||
### 3. Export Data from Current Server
|
||||
|
||||
Run the export script:
|
||||
```bash
|
||||
# On current server
|
||||
./export-chat-data.sh
|
||||
```
|
||||
|
||||
This will create:
|
||||
- `chat-schema.sql` - Database schema
|
||||
- `chat-data.sql` - All table data
|
||||
- `chat-files.tar.gz` - All uploaded files and avatars
|
||||
|
||||
### 4. Transfer Data to New Server
|
||||
|
||||
```bash
|
||||
# Copy files to new server
|
||||
scp chat-schema.sql chat-data.sql chat-files.tar.gz user@new-server:/tmp/
|
||||
```
|
||||
|
||||
### 5. Import Data on New Server
|
||||
|
||||
```bash
|
||||
# On new server
|
||||
./import-chat-data.sh
|
||||
```
|
||||
|
||||
### 6. Update Configuration
|
||||
|
||||
Update your environment variables to point to the new database server.
|
||||
|
||||
### 7. Verify Migration
|
||||
|
||||
Run the verification script to ensure everything transferred correctly:
|
||||
```bash
|
||||
node verify-migration.js
|
||||
```
|
||||
|
||||
## Files Provided
|
||||
|
||||
1. `create-new-database.sql` - Creates database and user on new server
|
||||
2. `export-chat-data.sh` - Exports data from current server
|
||||
3. `import-chat-data.sh` - Imports data to new server
|
||||
4. `verify-migration.js` - Verifies data integrity
|
||||
5. `update-config-template.env` - Template for new configuration
|
||||
|
||||
## Important Notes
|
||||
|
||||
- **Backup first**: Always backup your current database before migration
|
||||
- **Downtime**: Plan for application downtime during migration
|
||||
- **File permissions**: Ensure file permissions are preserved during transfer
|
||||
- **Network access**: Ensure new server can accept connections from your application
|
||||
Generated
+1446
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "chat-server",
|
||||
"version": "1.0.0",
|
||||
"description": "Chat archive server for Rocket.Chat data",
|
||||
"main": "server.js",
|
||||
"scripts": {
|
||||
"start": "node server.js",
|
||||
"dev": "nodemon server.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"express": "^4.18.2",
|
||||
"cors": "^2.8.5",
|
||||
"pg": "^8.11.0",
|
||||
"dotenv": "^16.0.3",
|
||||
"morgan": "^1.10.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nodemon": "^2.0.22"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,649 @@
|
||||
const express = require('express');
|
||||
const path = require('path');
|
||||
const router = express.Router();
|
||||
|
||||
// Serve uploaded files with proper mapping from database paths to actual file locations
|
||||
router.get('/files/uploads/*', async (req, res) => {
|
||||
try {
|
||||
// Extract the path from the URL (everything after /files/uploads/)
|
||||
const requestPath = req.params[0];
|
||||
|
||||
// The URL path will be like: ufs/AmazonS3:Uploads/274Mf9CyHNG72oF86/filename.jpg
|
||||
// We need to extract the mongo_id (274Mf9CyHNG72oF86) from this path
|
||||
const pathParts = requestPath.split('/');
|
||||
let mongoId = null;
|
||||
|
||||
// Find the mongo_id in the path structure
|
||||
for (let i = 0; i < pathParts.length; i++) {
|
||||
if (pathParts[i].includes('AmazonS3:Uploads') && i + 1 < pathParts.length) {
|
||||
mongoId = pathParts[i + 1];
|
||||
break;
|
||||
}
|
||||
// Sometimes the mongo_id might be the last part of ufs/AmazonS3:Uploads/mongoId
|
||||
if (pathParts[i] === 'AmazonS3:Uploads' && i + 1 < pathParts.length) {
|
||||
mongoId = pathParts[i + 1];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!mongoId) {
|
||||
// Try to get mongo_id from database by matching the full path
|
||||
const result = await global.pool.query(`
|
||||
SELECT mongo_id, name, type
|
||||
FROM uploads
|
||||
WHERE path = $1 OR url = $1
|
||||
LIMIT 1
|
||||
`, [`/ufs/AmazonS3:Uploads/${requestPath}`, `/ufs/AmazonS3:Uploads/${requestPath}`]);
|
||||
|
||||
if (result.rows.length > 0) {
|
||||
mongoId = result.rows[0].mongo_id;
|
||||
}
|
||||
}
|
||||
|
||||
if (!mongoId) {
|
||||
return res.status(404).json({ error: 'File not found' });
|
||||
}
|
||||
|
||||
// The actual file is stored with just the mongo_id as filename
|
||||
const filePath = path.join(__dirname, 'db-convert/db/files/uploads', mongoId);
|
||||
|
||||
// Get file info from database for proper content-type
|
||||
const fileInfo = await global.pool.query(`
|
||||
SELECT name, type
|
||||
FROM uploads
|
||||
WHERE mongo_id = $1
|
||||
LIMIT 1
|
||||
`, [mongoId]);
|
||||
|
||||
if (fileInfo.rows.length === 0) {
|
||||
return res.status(404).json({ error: 'File metadata not found' });
|
||||
}
|
||||
|
||||
const { name, type } = fileInfo.rows[0];
|
||||
|
||||
// Set proper content type
|
||||
if (type) {
|
||||
res.set('Content-Type', type);
|
||||
}
|
||||
|
||||
// Set content disposition with original filename
|
||||
if (name) {
|
||||
res.set('Content-Disposition', `inline; filename="${name}"`);
|
||||
}
|
||||
|
||||
// Send the file
|
||||
res.sendFile(filePath, (err) => {
|
||||
if (err) {
|
||||
console.error('Error serving file:', err);
|
||||
if (!res.headersSent) {
|
||||
res.status(404).json({ error: 'File not found on disk' });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error serving upload:', error);
|
||||
res.status(500).json({ error: 'Server error' });
|
||||
}
|
||||
});
|
||||
|
||||
// Also serve files directly by mongo_id for simpler access
|
||||
router.get('/files/by-id/:mongoId', async (req, res) => {
|
||||
try {
|
||||
const { mongoId } = req.params;
|
||||
|
||||
// Get file info from database
|
||||
const fileInfo = await global.pool.query(`
|
||||
SELECT name, type
|
||||
FROM uploads
|
||||
WHERE mongo_id = $1
|
||||
LIMIT 1
|
||||
`, [mongoId]);
|
||||
|
||||
if (fileInfo.rows.length === 0) {
|
||||
return res.status(404).json({ error: 'File not found' });
|
||||
}
|
||||
|
||||
const { name, type } = fileInfo.rows[0];
|
||||
const filePath = path.join(__dirname, 'db-convert/db/files/uploads', mongoId);
|
||||
|
||||
// Set proper content type and filename
|
||||
if (type) {
|
||||
res.set('Content-Type', type);
|
||||
}
|
||||
|
||||
if (name) {
|
||||
res.set('Content-Disposition', `inline; filename="${name}"`);
|
||||
}
|
||||
|
||||
// Send the file
|
||||
res.sendFile(filePath, (err) => {
|
||||
if (err) {
|
||||
console.error('Error serving file:', err);
|
||||
if (!res.headersSent) {
|
||||
res.status(404).json({ error: 'File not found on disk' });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error serving upload by ID:', error);
|
||||
res.status(500).json({ error: 'Server error' });
|
||||
}
|
||||
});
|
||||
|
||||
// Serve user avatars by mongo_id
|
||||
router.get('/avatar/:mongoId', async (req, res) => {
|
||||
try {
|
||||
const { mongoId } = req.params;
|
||||
|
||||
console.log(`[Avatar Debug] Looking up avatar for user mongo_id: ${mongoId}`);
|
||||
|
||||
// First try to find avatar by user's avataretag
|
||||
const userResult = await global.pool.query(`
|
||||
SELECT avataretag, username FROM users WHERE mongo_id = $1
|
||||
`, [mongoId]);
|
||||
|
||||
let avatarPath = null;
|
||||
|
||||
if (userResult.rows.length > 0) {
|
||||
const username = userResult.rows[0].username;
|
||||
const avataretag = userResult.rows[0].avataretag;
|
||||
|
||||
// Try method 1: Look up by avataretag -> etag (for users with avataretag set)
|
||||
if (avataretag) {
|
||||
console.log(`[Avatar Debug] Found user ${username} with avataretag: ${avataretag}`);
|
||||
|
||||
const avatarResult = await global.pool.query(`
|
||||
SELECT url, path FROM avatars WHERE etag = $1
|
||||
`, [avataretag]);
|
||||
|
||||
if (avatarResult.rows.length > 0) {
|
||||
const dbPath = avatarResult.rows[0].path || avatarResult.rows[0].url;
|
||||
console.log(`[Avatar Debug] Found avatar record with path: ${dbPath}`);
|
||||
|
||||
if (dbPath) {
|
||||
const pathParts = dbPath.split('/');
|
||||
for (let i = 0; i < pathParts.length; i++) {
|
||||
if (pathParts[i].includes('AmazonS3:Avatars') && i + 1 < pathParts.length) {
|
||||
const avatarMongoId = pathParts[i + 1];
|
||||
avatarPath = path.join(__dirname, 'db-convert/db/files/avatars', avatarMongoId);
|
||||
console.log(`[Avatar Debug] Extracted avatar mongo_id: ${avatarMongoId}, full path: ${avatarPath}`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
console.log(`[Avatar Debug] No avatar record found for etag: ${avataretag}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Try method 2: Look up by userid directly (for users without avataretag)
|
||||
if (!avatarPath) {
|
||||
console.log(`[Avatar Debug] Trying direct userid lookup for user ${username} (${mongoId})`);
|
||||
|
||||
const avatarResult = await global.pool.query(`
|
||||
SELECT url, path FROM avatars WHERE userid = $1
|
||||
`, [mongoId]);
|
||||
|
||||
if (avatarResult.rows.length > 0) {
|
||||
const dbPath = avatarResult.rows[0].path || avatarResult.rows[0].url;
|
||||
console.log(`[Avatar Debug] Found avatar record by userid with path: ${dbPath}`);
|
||||
|
||||
if (dbPath) {
|
||||
const pathParts = dbPath.split('/');
|
||||
for (let i = 0; i < pathParts.length; i++) {
|
||||
if (pathParts[i].includes('AmazonS3:Avatars') && i + 1 < pathParts.length) {
|
||||
const avatarMongoId = pathParts[i + 1];
|
||||
avatarPath = path.join(__dirname, 'db-convert/db/files/avatars', avatarMongoId);
|
||||
console.log(`[Avatar Debug] Extracted avatar mongo_id: ${avatarMongoId}, full path: ${avatarPath}`);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
console.log(`[Avatar Debug] No avatar record found for userid: ${mongoId}`);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
console.log(`[Avatar Debug] No user found for mongo_id: ${mongoId}`);
|
||||
}
|
||||
|
||||
// Fallback: try direct lookup by user mongo_id
|
||||
if (!avatarPath) {
|
||||
avatarPath = path.join(__dirname, 'db-convert/db/files/avatars', mongoId);
|
||||
console.log(`[Avatar Debug] Using fallback path: ${avatarPath}`);
|
||||
}
|
||||
|
||||
// Set proper content type for images
|
||||
res.set('Content-Type', 'image/jpeg'); // Most avatars are likely JPEG
|
||||
|
||||
// Send the file
|
||||
res.sendFile(avatarPath, (err) => {
|
||||
if (err) {
|
||||
// If avatar doesn't exist, send a default 404 or generate initials
|
||||
console.log(`[Avatar Debug] Avatar file not found at path: ${avatarPath}, error:`, err.message);
|
||||
if (!res.headersSent) {
|
||||
res.status(404).json({ error: 'Avatar not found' });
|
||||
}
|
||||
} else {
|
||||
console.log(`[Avatar Debug] Successfully served avatar from: ${avatarPath}`);
|
||||
}
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error serving avatar:', error);
|
||||
res.status(500).json({ error: 'Server error' });
|
||||
}
|
||||
});
|
||||
|
||||
// Serve avatars statically as fallback
|
||||
router.use('/files/avatars', express.static(path.join(__dirname, 'db-convert/db/files/avatars')));
|
||||
|
||||
// Get all users for the "view as" dropdown (active and inactive)
|
||||
router.get('/users', async (req, res) => {
|
||||
try {
|
||||
const result = await global.pool.query(`
|
||||
SELECT id, username, name, type, active, status, lastlogin,
|
||||
statustext, utcoffset, statusconnection, mongo_id, avataretag
|
||||
FROM users
|
||||
WHERE type = 'user'
|
||||
ORDER BY
|
||||
active DESC, -- Active users first
|
||||
CASE
|
||||
WHEN status = 'online' THEN 1
|
||||
WHEN status = 'away' THEN 2
|
||||
WHEN status = 'busy' THEN 3
|
||||
ELSE 4
|
||||
END,
|
||||
name ASC
|
||||
`);
|
||||
|
||||
res.json({
|
||||
status: 'success',
|
||||
users: result.rows
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error fetching users:', error);
|
||||
res.status(500).json({
|
||||
status: 'error',
|
||||
error: 'Failed to fetch users',
|
||||
details: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Get rooms for a specific user with enhanced room names for direct messages
|
||||
router.get('/users/:userId/rooms', async (req, res) => {
|
||||
const { userId } = req.params;
|
||||
|
||||
try {
|
||||
// Get the current user's mongo_id for filtering
|
||||
const userResult = await global.pool.query(`
|
||||
SELECT mongo_id, username FROM users WHERE id = $1
|
||||
`, [userId]);
|
||||
|
||||
if (userResult.rows.length === 0) {
|
||||
return res.status(404).json({
|
||||
status: 'error',
|
||||
error: 'User not found'
|
||||
});
|
||||
}
|
||||
|
||||
const currentUserMongoId = userResult.rows[0].mongo_id;
|
||||
const currentUsername = userResult.rows[0].username;
|
||||
|
||||
// Get rooms where the user is a member with proper naming from subscription table
|
||||
// Include archived and closed rooms but sort them at the bottom
|
||||
const result = await global.pool.query(`
|
||||
SELECT DISTINCT
|
||||
r.id,
|
||||
r.mongo_id as room_mongo_id,
|
||||
r.name,
|
||||
r.fname,
|
||||
r.t as type,
|
||||
r.msgs,
|
||||
r.lm as last_message_date,
|
||||
r.usernames,
|
||||
r.uids,
|
||||
r.userscount,
|
||||
r.description,
|
||||
r.teamid,
|
||||
r.archived,
|
||||
s.open,
|
||||
-- Use the subscription's name for direct messages (excludes current user)
|
||||
-- For channels/groups, use room's fname or name
|
||||
CASE
|
||||
WHEN r.t = 'd' THEN COALESCE(s.fname, s.name, 'Unknown User')
|
||||
ELSE COALESCE(r.fname, r.name, 'Unnamed Room')
|
||||
END as display_name
|
||||
FROM room r
|
||||
JOIN subscription s ON s.rid = r.mongo_id
|
||||
WHERE s.u->>'_id' = $1
|
||||
ORDER BY
|
||||
s.open DESC NULLS LAST, -- Open rooms first
|
||||
r.archived NULLS FIRST, -- Non-archived first (nulls treated as false)
|
||||
r.lm DESC NULLS LAST
|
||||
LIMIT 50
|
||||
`, [currentUserMongoId]);
|
||||
|
||||
// Enhance rooms with participant information for direct messages
|
||||
const enhancedRooms = await Promise.all(result.rows.map(async (room) => {
|
||||
if (room.type === 'd' && room.uids) {
|
||||
// Get participant info (excluding current user) for direct messages
|
||||
const participantResult = await global.pool.query(`
|
||||
SELECT u.username, u.name, u.mongo_id, u.avataretag
|
||||
FROM users u
|
||||
WHERE u.mongo_id = ANY($1::text[])
|
||||
AND u.mongo_id != $2
|
||||
`, [room.uids, currentUserMongoId]);
|
||||
|
||||
room.participants = participantResult.rows;
|
||||
}
|
||||
return room;
|
||||
}));
|
||||
|
||||
res.json({
|
||||
status: 'success',
|
||||
rooms: enhancedRooms
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error fetching user rooms:', error);
|
||||
res.status(500).json({
|
||||
status: 'error',
|
||||
error: 'Failed to fetch user rooms',
|
||||
details: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Get room details including participants
|
||||
router.get('/rooms/:roomId', async (req, res) => {
|
||||
const { roomId } = req.params;
|
||||
const { userId } = req.query; // Accept current user ID as query parameter
|
||||
|
||||
try {
|
||||
const result = await global.pool.query(`
|
||||
SELECT r.id, r.name, r.fname, r.t as type, r.msgs, r.description,
|
||||
r.lm as last_message_date, r.usernames, r.uids, r.userscount, r.teamid
|
||||
FROM room r
|
||||
WHERE r.id = $1
|
||||
`, [roomId]);
|
||||
|
||||
if (result.rows.length === 0) {
|
||||
return res.status(404).json({
|
||||
status: 'error',
|
||||
error: 'Room not found'
|
||||
});
|
||||
}
|
||||
|
||||
const room = result.rows[0];
|
||||
|
||||
// For direct messages, get the proper display name based on current user
|
||||
if (room.type === 'd' && room.uids && userId) {
|
||||
// Get current user's mongo_id
|
||||
const userResult = await global.pool.query(`
|
||||
SELECT mongo_id FROM users WHERE id = $1
|
||||
`, [userId]);
|
||||
|
||||
if (userResult.rows.length > 0) {
|
||||
const currentUserMongoId = userResult.rows[0].mongo_id;
|
||||
|
||||
// Get display name from subscription table for this user
|
||||
// Use room mongo_id to match with subscription.rid
|
||||
const roomMongoResult = await global.pool.query(`
|
||||
SELECT mongo_id FROM room WHERE id = $1
|
||||
`, [roomId]);
|
||||
|
||||
if (roomMongoResult.rows.length > 0) {
|
||||
const roomMongoId = roomMongoResult.rows[0].mongo_id;
|
||||
|
||||
const subscriptionResult = await global.pool.query(`
|
||||
SELECT fname, name FROM subscription
|
||||
WHERE rid = $1 AND u->>'_id' = $2
|
||||
`, [roomMongoId, currentUserMongoId]);
|
||||
|
||||
if (subscriptionResult.rows.length > 0) {
|
||||
const sub = subscriptionResult.rows[0];
|
||||
room.display_name = sub.fname || sub.name || 'Unknown User';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get all participants for additional info
|
||||
const participantResult = await global.pool.query(`
|
||||
SELECT username, name
|
||||
FROM users
|
||||
WHERE mongo_id = ANY($1::text[])
|
||||
`, [room.uids]);
|
||||
|
||||
room.participants = participantResult.rows;
|
||||
} else {
|
||||
// For channels/groups, use room's fname or name
|
||||
room.display_name = room.fname || room.name || 'Unnamed Room';
|
||||
}
|
||||
|
||||
res.json({
|
||||
status: 'success',
|
||||
room: room
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error fetching room details:', error);
|
||||
res.status(500).json({
|
||||
status: 'error',
|
||||
error: 'Failed to fetch room details',
|
||||
details: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Get messages for a specific room (fast, without attachments)
|
||||
router.get('/rooms/:roomId/messages', async (req, res) => {
|
||||
const { roomId } = req.params;
|
||||
const { limit = 50, offset = 0, before } = req.query;
|
||||
|
||||
try {
|
||||
// Fast query - just get messages without expensive attachment joins
|
||||
let query = `
|
||||
SELECT m.id, m.msg, m.ts, m.u, m._updatedat, m.urls, m.mentions, m.md
|
||||
FROM message m
|
||||
JOIN room r ON m.rid = r.mongo_id
|
||||
WHERE r.id = $1
|
||||
`;
|
||||
|
||||
const params = [roomId];
|
||||
|
||||
if (before) {
|
||||
query += ` AND m.ts < $${params.length + 1}`;
|
||||
params.push(before);
|
||||
}
|
||||
|
||||
query += ` ORDER BY m.ts DESC LIMIT $${params.length + 1} OFFSET $${params.length + 2}`;
|
||||
params.push(limit, offset);
|
||||
|
||||
const result = await global.pool.query(query, params);
|
||||
|
||||
// Add empty attachments array for now - attachments will be loaded separately if needed
|
||||
const messages = result.rows.map(msg => ({
|
||||
...msg,
|
||||
attachments: []
|
||||
}));
|
||||
|
||||
res.json({
|
||||
status: 'success',
|
||||
messages: messages.reverse() // Reverse to show oldest first
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error fetching messages:', error);
|
||||
res.status(500).json({
|
||||
status: 'error',
|
||||
error: 'Failed to fetch messages',
|
||||
details: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Get attachments for specific messages (called separately for performance)
|
||||
router.post('/messages/attachments', async (req, res) => {
|
||||
const { messageIds } = req.body;
|
||||
|
||||
if (!messageIds || !Array.isArray(messageIds) || messageIds.length === 0) {
|
||||
return res.json({ status: 'success', attachments: {} });
|
||||
}
|
||||
|
||||
try {
|
||||
// Get room mongo_id from first message to limit search scope
|
||||
const roomQuery = await global.pool.query(`
|
||||
SELECT r.mongo_id as room_mongo_id
|
||||
FROM message m
|
||||
JOIN room r ON m.rid = r.mongo_id
|
||||
WHERE m.id = $1
|
||||
LIMIT 1
|
||||
`, [messageIds[0]]);
|
||||
|
||||
if (roomQuery.rows.length === 0) {
|
||||
return res.json({ status: 'success', attachments: {} });
|
||||
}
|
||||
|
||||
const roomMongoId = roomQuery.rows[0].room_mongo_id;
|
||||
|
||||
// Get messages and their upload timestamps
|
||||
const messagesQuery = await global.pool.query(`
|
||||
SELECT m.id, m.ts, m.u->>'_id' as user_id
|
||||
FROM message m
|
||||
WHERE m.id = ANY($1::int[])
|
||||
`, [messageIds]);
|
||||
|
||||
if (messagesQuery.rows.length === 0) {
|
||||
return res.json({ status: 'success', attachments: {} });
|
||||
}
|
||||
|
||||
// Build a map of user_id -> array of message timestamps for efficient lookup
|
||||
const userTimeMap = {};
|
||||
const messageMap = {};
|
||||
messagesQuery.rows.forEach(msg => {
|
||||
if (!userTimeMap[msg.user_id]) {
|
||||
userTimeMap[msg.user_id] = [];
|
||||
}
|
||||
userTimeMap[msg.user_id].push(msg.ts);
|
||||
messageMap[msg.id] = { ts: msg.ts, user_id: msg.user_id };
|
||||
});
|
||||
|
||||
// Get attachments for this room and these users
|
||||
const uploadsQuery = await global.pool.query(`
|
||||
SELECT mongo_id, name, size, type, url, path, typegroup, identify,
|
||||
userid, uploadedat
|
||||
FROM uploads
|
||||
WHERE rid = $1
|
||||
AND userid = ANY($2::text[])
|
||||
ORDER BY uploadedat
|
||||
`, [roomMongoId, Object.keys(userTimeMap)]);
|
||||
|
||||
// Match attachments to messages based on timestamp proximity (within 5 minutes)
|
||||
const attachmentsByMessage = {};
|
||||
|
||||
uploadsQuery.rows.forEach(upload => {
|
||||
const uploadTime = new Date(upload.uploadedat).getTime();
|
||||
|
||||
// Find the closest message from this user within 5 minutes
|
||||
let closestMessageId = null;
|
||||
let closestTimeDiff = Infinity;
|
||||
|
||||
Object.entries(messageMap).forEach(([msgId, msgData]) => {
|
||||
if (msgData.user_id === upload.userid) {
|
||||
const msgTime = new Date(msgData.ts).getTime();
|
||||
const timeDiff = Math.abs(uploadTime - msgTime);
|
||||
|
||||
if (timeDiff < 300000 && timeDiff < closestTimeDiff) { // 5 minutes = 300000ms
|
||||
closestMessageId = msgId;
|
||||
closestTimeDiff = timeDiff;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
if (closestMessageId) {
|
||||
if (!attachmentsByMessage[closestMessageId]) {
|
||||
attachmentsByMessage[closestMessageId] = [];
|
||||
}
|
||||
|
||||
attachmentsByMessage[closestMessageId].push({
|
||||
id: upload.id,
|
||||
mongo_id: upload.mongo_id,
|
||||
name: upload.name,
|
||||
size: upload.size,
|
||||
type: upload.type,
|
||||
url: upload.url,
|
||||
path: upload.path,
|
||||
typegroup: upload.typegroup,
|
||||
identify: upload.identify
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
res.json({
|
||||
status: 'success',
|
||||
attachments: attachmentsByMessage
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error fetching message attachments:', error);
|
||||
res.status(500).json({
|
||||
status: 'error',
|
||||
error: 'Failed to fetch attachments',
|
||||
details: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Search messages in accessible rooms for a user
|
||||
router.get('/users/:userId/search', async (req, res) => {
|
||||
const { userId } = req.params;
|
||||
const { q, limit = 20 } = req.query;
|
||||
|
||||
if (!q || q.length < 2) {
|
||||
return res.status(400).json({
|
||||
status: 'error',
|
||||
error: 'Search query must be at least 2 characters'
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const userResult = await global.pool.query(`
|
||||
SELECT mongo_id FROM users WHERE id = $1
|
||||
`, [userId]);
|
||||
|
||||
if (userResult.rows.length === 0) {
|
||||
return res.status(404).json({
|
||||
status: 'error',
|
||||
error: 'User not found'
|
||||
});
|
||||
}
|
||||
|
||||
const currentUserMongoId = userResult.rows[0].mongo_id;
|
||||
|
||||
const result = await global.pool.query(`
|
||||
SELECT m.id, m.msg, m.ts, m.u, r.id as room_id, r.name as room_name, r.fname as room_fname, r.t as room_type
|
||||
FROM message m
|
||||
JOIN room r ON m.rid = r.mongo_id
|
||||
JOIN subscription s ON s.rid = r.mongo_id AND s.u->>'_id' = $1
|
||||
WHERE m.msg ILIKE $2
|
||||
AND r.archived IS NOT TRUE
|
||||
ORDER BY m.ts DESC
|
||||
LIMIT $3
|
||||
`, [currentUserMongoId, `%${q}%`, limit]);
|
||||
|
||||
res.json({
|
||||
status: 'success',
|
||||
results: result.rows
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error searching messages:', error);
|
||||
res.status(500).json({
|
||||
status: 'error',
|
||||
error: 'Failed to search messages',
|
||||
details: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,83 @@
|
||||
require('dotenv').config({ path: '../.env' });
|
||||
const express = require('express');
|
||||
const cors = require('cors');
|
||||
const { Pool } = require('pg');
|
||||
const morgan = require('morgan');
|
||||
const chatRoutes = require('./routes');
|
||||
|
||||
// Log startup configuration
|
||||
console.log('Starting chat server with config:', {
|
||||
host: process.env.CHAT_DB_HOST,
|
||||
user: process.env.CHAT_DB_USER,
|
||||
database: process.env.CHAT_DB_NAME || 'rocketchat_converted',
|
||||
port: process.env.CHAT_DB_PORT,
|
||||
chat_port: process.env.CHAT_PORT || 3014
|
||||
});
|
||||
|
||||
const app = express();
|
||||
const port = process.env.CHAT_PORT || 3014;
|
||||
|
||||
// Database configuration for rocketchat_converted database
|
||||
const pool = new Pool({
|
||||
host: process.env.CHAT_DB_HOST,
|
||||
user: process.env.CHAT_DB_USER,
|
||||
password: process.env.CHAT_DB_PASSWORD,
|
||||
database: process.env.CHAT_DB_NAME || 'rocketchat_converted',
|
||||
port: process.env.CHAT_DB_PORT,
|
||||
});
|
||||
|
||||
// Make pool available globally
|
||||
global.pool = pool;
|
||||
|
||||
// Middleware
|
||||
app.use(express.json());
|
||||
app.use(morgan('combined'));
|
||||
app.use(cors({
|
||||
origin: ['http://localhost:5175', 'http://localhost:5174', 'https://inventory.kent.pw', 'https://acot.site', 'https://tools.acherryontop.com'],
|
||||
credentials: true
|
||||
}));
|
||||
|
||||
// Test database connection endpoint
|
||||
app.get('/test-db', async (req, res) => {
|
||||
try {
|
||||
const result = await pool.query('SELECT COUNT(*) as user_count FROM users WHERE active = true');
|
||||
const messageResult = await pool.query('SELECT COUNT(*) as message_count FROM message');
|
||||
const roomResult = await pool.query('SELECT COUNT(*) as room_count FROM room');
|
||||
|
||||
res.json({
|
||||
status: 'success',
|
||||
database: 'rocketchat_converted',
|
||||
stats: {
|
||||
active_users: parseInt(result.rows[0].user_count),
|
||||
total_messages: parseInt(messageResult.rows[0].message_count),
|
||||
total_rooms: parseInt(roomResult.rows[0].room_count)
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Database test error:', error);
|
||||
res.status(500).json({
|
||||
status: 'error',
|
||||
error: 'Database connection failed',
|
||||
details: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Mount all routes from routes.js
|
||||
app.use('/', chatRoutes);
|
||||
|
||||
// Health check endpoint
|
||||
app.get('/health', (req, res) => {
|
||||
res.json({ status: 'healthy' });
|
||||
});
|
||||
|
||||
// Error handling middleware
|
||||
app.use((err, req, res, next) => {
|
||||
console.error(err.stack);
|
||||
res.status(500).json({ error: 'Something broke!' });
|
||||
});
|
||||
|
||||
// Start server
|
||||
app.listen(port, () => {
|
||||
console.log(`Chat server running on port ${port}`);
|
||||
});
|
||||
@@ -0,0 +1,26 @@
|
||||
# Chat Server Database Configuration Template
|
||||
# Copy this to your .env file and update the values for your new server
|
||||
|
||||
# Database Configuration for New Server
|
||||
CHAT_DB_HOST=your-new-server-ip-or-hostname
|
||||
CHAT_DB_PORT=5432
|
||||
CHAT_DB_NAME=rocketchat_converted
|
||||
CHAT_DB_USER=rocketchat_user
|
||||
CHAT_DB_PASSWORD=your-secure-password
|
||||
|
||||
# Chat Server Port
|
||||
CHAT_PORT=3014
|
||||
|
||||
# Example configuration:
|
||||
# CHAT_DB_HOST=192.168.1.100
|
||||
# CHAT_DB_PORT=5432
|
||||
# CHAT_DB_NAME=rocketchat_converted
|
||||
# CHAT_DB_USER=rocketchat_user
|
||||
# CHAT_DB_PASSWORD=MySecureP@ssw0rd123
|
||||
|
||||
# Notes:
|
||||
# - Replace 'your-new-server-ip-or-hostname' with actual server address
|
||||
# - Use a strong password for CHAT_DB_PASSWORD
|
||||
# - Ensure the new server allows connections from your application server
|
||||
# - Update any firewall rules to allow PostgreSQL connections (port 5432)
|
||||
# - Test connectivity before updating production configuration
|
||||
@@ -0,0 +1,231 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
/**
|
||||
* Chat Database Migration Verification Script
|
||||
*
|
||||
* This script verifies that the chat database migration was successful
|
||||
* by comparing record counts and testing basic functionality.
|
||||
*/
|
||||
|
||||
require('dotenv').config({ path: '../.env' });
|
||||
const { Pool } = require('pg');
|
||||
|
||||
// Database configuration
|
||||
const pool = new Pool({
|
||||
host: process.env.CHAT_DB_HOST || 'localhost',
|
||||
user: process.env.CHAT_DB_USER || 'rocketchat_user',
|
||||
password: process.env.CHAT_DB_PASSWORD,
|
||||
database: process.env.CHAT_DB_NAME || 'rocketchat_converted',
|
||||
port: process.env.CHAT_DB_PORT || 5432,
|
||||
});
|
||||
|
||||
const originalStats = process.argv[2] ? JSON.parse(process.argv[2]) : null;
|
||||
|
||||
async function verifyMigration() {
|
||||
console.log('🔍 Starting migration verification...\n');
|
||||
|
||||
try {
|
||||
// Test basic connection
|
||||
console.log('🔗 Testing database connection...');
|
||||
const versionResult = await pool.query('SELECT version()');
|
||||
console.log('✅ Database connection successful');
|
||||
console.log(` PostgreSQL version: ${versionResult.rows[0].version.split(' ')[1]}\n`);
|
||||
|
||||
// Get table statistics
|
||||
console.log('📊 Checking table statistics...');
|
||||
const statsResult = await pool.query(`
|
||||
SELECT
|
||||
tablename,
|
||||
n_live_tup as row_count,
|
||||
n_dead_tup as dead_rows,
|
||||
schemaname
|
||||
FROM pg_stat_user_tables
|
||||
WHERE schemaname = 'public'
|
||||
ORDER BY n_live_tup DESC
|
||||
`);
|
||||
|
||||
if (statsResult.rows.length === 0) {
|
||||
console.log('❌ No tables found! Migration may have failed.');
|
||||
return false;
|
||||
}
|
||||
|
||||
console.log('📋 Table Statistics:');
|
||||
console.log(' Table Name | Row Count | Dead Rows');
|
||||
console.log(' -------------------|-----------|----------');
|
||||
|
||||
let totalRows = 0;
|
||||
const tableStats = {};
|
||||
|
||||
for (const row of statsResult.rows) {
|
||||
const rowCount = parseInt(row.row_count) || 0;
|
||||
const deadRows = parseInt(row.dead_rows) || 0;
|
||||
totalRows += rowCount;
|
||||
tableStats[row.tablename] = rowCount;
|
||||
|
||||
console.log(` ${row.tablename.padEnd(18)} | ${rowCount.toString().padStart(9)} | ${deadRows.toString().padStart(8)}`);
|
||||
}
|
||||
|
||||
console.log(`\n Total rows across all tables: ${totalRows}\n`);
|
||||
|
||||
// Verify critical tables exist and have data
|
||||
const criticalTables = ['users', 'message', 'room'];
|
||||
console.log('🔑 Checking critical tables...');
|
||||
|
||||
for (const table of criticalTables) {
|
||||
if (tableStats[table] > 0) {
|
||||
console.log(`✅ ${table}: ${tableStats[table]} rows`);
|
||||
} else if (tableStats[table] === 0) {
|
||||
console.log(`⚠️ ${table}: table exists but is empty`);
|
||||
} else {
|
||||
console.log(`❌ ${table}: table not found`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Test specific functionality
|
||||
console.log('\n🧪 Testing specific functionality...');
|
||||
|
||||
// Test users table
|
||||
const userTest = await pool.query(`
|
||||
SELECT COUNT(*) as total_users,
|
||||
COUNT(*) FILTER (WHERE active = true) as active_users,
|
||||
COUNT(*) FILTER (WHERE type = 'user') as regular_users
|
||||
FROM users
|
||||
`);
|
||||
|
||||
if (userTest.rows[0]) {
|
||||
const { total_users, active_users, regular_users } = userTest.rows[0];
|
||||
console.log(`✅ Users: ${total_users} total, ${active_users} active, ${regular_users} regular users`);
|
||||
}
|
||||
|
||||
// Test messages table
|
||||
const messageTest = await pool.query(`
|
||||
SELECT COUNT(*) as total_messages,
|
||||
COUNT(DISTINCT rid) as unique_rooms,
|
||||
MIN(ts) as oldest_message,
|
||||
MAX(ts) as newest_message
|
||||
FROM message
|
||||
`);
|
||||
|
||||
if (messageTest.rows[0]) {
|
||||
const { total_messages, unique_rooms, oldest_message, newest_message } = messageTest.rows[0];
|
||||
console.log(`✅ Messages: ${total_messages} total across ${unique_rooms} rooms`);
|
||||
if (oldest_message && newest_message) {
|
||||
console.log(` Date range: ${oldest_message.toISOString().split('T')[0]} to ${newest_message.toISOString().split('T')[0]}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Test rooms table
|
||||
const roomTest = await pool.query(`
|
||||
SELECT COUNT(*) as total_rooms,
|
||||
COUNT(*) FILTER (WHERE t = 'c') as channels,
|
||||
COUNT(*) FILTER (WHERE t = 'p') as private_groups,
|
||||
COUNT(*) FILTER (WHERE t = 'd') as direct_messages
|
||||
FROM room
|
||||
`);
|
||||
|
||||
if (roomTest.rows[0]) {
|
||||
const { total_rooms, channels, private_groups, direct_messages } = roomTest.rows[0];
|
||||
console.log(`✅ Rooms: ${total_rooms} total (${channels} channels, ${private_groups} private, ${direct_messages} DMs)`);
|
||||
}
|
||||
|
||||
// Test file uploads if table exists
|
||||
if (tableStats.uploads > 0) {
|
||||
const uploadTest = await pool.query(`
|
||||
SELECT COUNT(*) as total_uploads,
|
||||
COUNT(DISTINCT typegroup) as file_types,
|
||||
pg_size_pretty(SUM(size)) as total_size
|
||||
FROM uploads
|
||||
WHERE size IS NOT NULL
|
||||
`);
|
||||
|
||||
if (uploadTest.rows[0]) {
|
||||
const { total_uploads, file_types, total_size } = uploadTest.rows[0];
|
||||
console.log(`✅ Uploads: ${total_uploads} files, ${file_types} types, ${total_size || 'unknown size'}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Test server health endpoint simulation
|
||||
console.log('\n🏥 Testing application endpoints simulation...');
|
||||
|
||||
try {
|
||||
const healthTest = await pool.query(`
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM users WHERE active = true) as active_users,
|
||||
(SELECT COUNT(*) FROM message) as total_messages,
|
||||
(SELECT COUNT(*) FROM room) as total_rooms
|
||||
`);
|
||||
|
||||
if (healthTest.rows[0]) {
|
||||
const stats = healthTest.rows[0];
|
||||
console.log('✅ Health check simulation passed');
|
||||
console.log(` Active users: ${stats.active_users}`);
|
||||
console.log(` Total messages: ${stats.total_messages}`);
|
||||
console.log(` Total rooms: ${stats.total_rooms}`);
|
||||
}
|
||||
} catch (error) {
|
||||
console.log(`⚠️ Health check simulation failed: ${error.message}`);
|
||||
}
|
||||
|
||||
// Check indexes
|
||||
console.log('\n📇 Checking database indexes...');
|
||||
const indexResult = await pool.query(`
|
||||
SELECT
|
||||
schemaname,
|
||||
tablename,
|
||||
indexname,
|
||||
indexdef
|
||||
FROM pg_indexes
|
||||
WHERE schemaname = 'public'
|
||||
ORDER BY tablename, indexname
|
||||
`);
|
||||
|
||||
const indexesByTable = {};
|
||||
for (const idx of indexResult.rows) {
|
||||
if (!indexesByTable[idx.tablename]) {
|
||||
indexesByTable[idx.tablename] = [];
|
||||
}
|
||||
indexesByTable[idx.tablename].push(idx.indexname);
|
||||
}
|
||||
|
||||
for (const [table, indexes] of Object.entries(indexesByTable)) {
|
||||
console.log(` ${table}: ${indexes.length} indexes`);
|
||||
}
|
||||
|
||||
console.log('\n🎉 Migration verification completed successfully!');
|
||||
console.log('\n✅ Summary:');
|
||||
console.log(` - Database connection: Working`);
|
||||
console.log(` - Tables created: ${statsResult.rows.length}`);
|
||||
console.log(` - Total data rows: ${totalRows}`);
|
||||
console.log(` - Critical tables: All present`);
|
||||
console.log(` - Indexes: ${indexResult.rows.length} total`);
|
||||
|
||||
console.log('\n🚀 Next steps:');
|
||||
console.log(' 1. Update your application configuration');
|
||||
console.log(' 2. Start your chat server');
|
||||
console.log(' 3. Test chat functionality in the browser');
|
||||
console.log(' 4. Monitor logs for any issues');
|
||||
|
||||
return true;
|
||||
|
||||
} catch (error) {
|
||||
console.error('❌ Migration verification failed:', error.message);
|
||||
console.error('\n🔧 Troubleshooting steps:');
|
||||
console.error(' 1. Check database connection settings');
|
||||
console.error(' 2. Verify database and user exist');
|
||||
console.error(' 3. Check PostgreSQL logs');
|
||||
console.error(' 4. Ensure import completed without errors');
|
||||
return false;
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
// Run verification
|
||||
if (require.main === module) {
|
||||
verifyMigration().then(success => {
|
||||
process.exit(success ? 0 : 1);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { verifyMigration };
|
||||
@@ -0,0 +1,32 @@
|
||||
# dashboard-server .env template (Phase 4)
|
||||
#
|
||||
# The merged dashboard-server reads /var/www/inventory/.env FIRST (provides
|
||||
# JWT_SECRET, DB_*, REDIS_*) and then layers this .env on top for vendor keys.
|
||||
# Shared/security-critical vars stay in /var/www/inventory/.env so they aren't
|
||||
# duplicated; vendor keys live here.
|
||||
#
|
||||
# Copy to .env and populate. Do NOT commit the populated file.
|
||||
|
||||
# Port the merged service listens on
|
||||
DASHBOARD_PORT=3015
|
||||
|
||||
# Klaviyo (replaces klaviyo-server/.env)
|
||||
KLAVIYO_API_KEY=
|
||||
KLAVIYO_API_REVISION=2024-02-15
|
||||
KLAVIYO_API_URL=https://a.klaviyo.com/api
|
||||
|
||||
# Meta / Facebook Ads (replaces meta-server/.env)
|
||||
META_ACCESS_TOKEN=
|
||||
META_AD_ACCOUNT_ID=
|
||||
META_API_VERSION=v21.0
|
||||
|
||||
# Google Analytics (replaces google-server/.env)
|
||||
GA_PROPERTY_ID=
|
||||
GOOGLE_APPLICATION_CREDENTIALS_JSON=
|
||||
|
||||
# Typeform (replaces typeform-server/.env)
|
||||
TYPEFORM_ACCESS_TOKEN=
|
||||
|
||||
# Vendors share the inventory REDIS_URL or REDIS_HOST/PORT/USERNAME/PASSWORD
|
||||
# from the parent .env. Do NOT redeclare here unless you need a vendor-only
|
||||
# override (rare; would need to fork shared/db/redis.js too).
|
||||
@@ -0,0 +1,205 @@
|
||||
# ACOT Server
|
||||
|
||||
This server replaces the Klaviyo integration with direct database queries to the production MySQL database via SSH tunnel. It provides seamless API compatibility for all frontend components without requiring any frontend changes.
|
||||
|
||||
## Setup
|
||||
|
||||
1. **Environment Variables**: Copy `.env.example` to `.env` and configure:
|
||||
```
|
||||
DB_HOST=localhost
|
||||
DB_PORT=3306
|
||||
DB_USER=your_db_user
|
||||
DB_PASSWORD=your_db_password
|
||||
DB_NAME=your_db_name
|
||||
PORT=3007
|
||||
NODE_ENV=development
|
||||
```
|
||||
|
||||
2. **SSH Tunnel**: Ensure your SSH tunnel to the production database is running on localhost:3306.
|
||||
|
||||
3. **Install Dependencies**:
|
||||
```bash
|
||||
npm install
|
||||
```
|
||||
|
||||
4. **Start Server**:
|
||||
```bash
|
||||
npm start
|
||||
```
|
||||
|
||||
## API Endpoints
|
||||
|
||||
All endpoints provide exact API compatibility with the previous Klaviyo implementation:
|
||||
|
||||
### Main Statistics
|
||||
- `GET /api/acot/events/stats` - Complete statistics dashboard data
|
||||
- Query params: `timeRange` (today, yesterday, thisWeek, lastWeek, thisMonth, lastMonth, last7days, last30days, last90days) or `startDate`/`endDate` for custom ranges
|
||||
- Returns: Revenue, orders, AOV, shipping data, order types, brands/categories, refunds, cancellations, best day, peak hour, order ranges, period progress, projections
|
||||
|
||||
### Daily Details
|
||||
- `GET /api/acot/events/stats/details` - Daily breakdown with previous period comparisons
|
||||
- Query params: `timeRange`, `metric` (revenue, orders, average_order, etc.), `daily=true`
|
||||
- Returns: Array of daily data points with trend comparisons
|
||||
|
||||
### Products
|
||||
- `GET /api/acot/events/products` - Top products with sales data
|
||||
- Query params: `timeRange`
|
||||
- Returns: Product list with images, sales quantities, revenue, and order counts
|
||||
|
||||
### Projections
|
||||
- `GET /api/acot/events/projection` - Smart revenue projections for incomplete periods
|
||||
- Query params: `timeRange`
|
||||
- Returns: Projected revenue with confidence levels based on historical patterns
|
||||
|
||||
### Health Check
|
||||
- `GET /api/acot/test` - Server health and database connectivity test
|
||||
|
||||
## Database Schema
|
||||
|
||||
The server queries the following main tables:
|
||||
|
||||
### Orders (`_order`)
|
||||
- **Key fields**: `order_id`, `date_placed`, `summary_total`, `order_status`, `ship_method_selected`, `stats_waiting_preorder`
|
||||
- **Valid orders**: `order_status > 15`
|
||||
- **Cancelled orders**: `order_status = 15`
|
||||
- **Shipped orders**: `order_status IN (100, 92)`
|
||||
- **Pre-orders**: `stats_waiting_preorder > 0`
|
||||
- **Local pickup**: `ship_method_selected = 'localpickup'`
|
||||
- **On-hold orders**: `ship_method_selected = 'holdit'`
|
||||
|
||||
### Order Items (`order_items`)
|
||||
- **Fields**: `order_id`, `prod_pid`, `qty_ordered`, `prod_price`
|
||||
- **Purpose**: Links orders to products for detailed analysis
|
||||
|
||||
### Products (`products`)
|
||||
- **Fields**: `pid`, `description` (product name), `company`
|
||||
- **Purpose**: Product information and brand data
|
||||
|
||||
### Product Images (`product_images`)
|
||||
- **Fields**: `pid`, `iid`, `order` (priority)
|
||||
- **Primary image**: `order = 255` (highest priority)
|
||||
- **Image URL generation**: `https://sbing.com/i/products/0000/{prefix}/{pid}-{type}-{iid}.jpg`
|
||||
|
||||
### Payments (`order_payment`)
|
||||
- **Refunds**: `payment_amount < 0`
|
||||
- **Purpose**: Track refund amounts and counts
|
||||
|
||||
## Business Logic
|
||||
|
||||
### Time Handling
|
||||
- **Timezone**: All calculations in UTC-5 (Eastern Time)
|
||||
- **Business Day**: 1 AM - 12:59 AM Eastern (25-hour business day)
|
||||
- **Format**: MySQL DATETIME format (YYYY-MM-DD HH:MM:SS)
|
||||
- **Period Boundaries**: Calculated using `timeUtils.js` for consistent time range handling
|
||||
|
||||
### Order Processing
|
||||
- **Revenue Calculation**: Only includes orders with `order_status > 15`
|
||||
- **Order Types**:
|
||||
- Pre-orders: `stats_waiting_preorder > 0`
|
||||
- Local pickup: `ship_method_selected = 'localpickup'`
|
||||
- On-hold: `ship_method_selected = 'holdit'`
|
||||
- **Shipping Methods**: Mapped to friendly names (e.g., `usps_ground_advantage` → "USPS Ground Advantage")
|
||||
|
||||
### Projections
|
||||
- **Period Progress**: Calculated based on current time within the selected period
|
||||
- **Simple Projection**: Linear extrapolation based on current progress
|
||||
- **Smart Projection**: Uses historical data patterns for more accurate forecasting
|
||||
- **Confidence Levels**: Based on data consistency and historical accuracy
|
||||
|
||||
### Image URL Generation
|
||||
- **Pattern**: `https://sbing.com/i/products/0000/{prefix}/{pid}-{type}-{iid}.jpg`
|
||||
- **Prefix**: First 2 digits of product ID
|
||||
- **Type**: "main" for primary images
|
||||
- **Fallback**: Uses primary image (order=255) when available
|
||||
|
||||
## Frontend Integration
|
||||
|
||||
### Service Layer (`services/acotService.js`)
|
||||
- **Purpose**: Replaces direct Klaviyo API calls with acot-server calls
|
||||
- **Methods**: `getStats()`, `getStatsDetails()`, `getProducts()`, `getProjection()`
|
||||
- **Logging**: Axios interceptors for request/response logging
|
||||
- **Environment**: Automatic URL handling (proxy in dev, direct in production)
|
||||
|
||||
### Component Updates
|
||||
All 5 main components updated to use `acotService`:
|
||||
- **StatCards.jsx**: Main dashboard statistics
|
||||
- **MiniStatCards.jsx**: Compact statistics view
|
||||
- **SalesChart.jsx**: Revenue and order trends
|
||||
- **MiniSalesChart.jsx**: Compact chart view
|
||||
- **ProductGrid.jsx**: Top products table
|
||||
|
||||
### Proxy Configuration (`vite.config.js`)
|
||||
```javascript
|
||||
'/api/acot': {
|
||||
target: 'http://localhost:3007',
|
||||
changeOrigin: true,
|
||||
secure: false
|
||||
}
|
||||
```
|
||||
|
||||
## Key Features
|
||||
|
||||
### Complete Business Intelligence
|
||||
- **Revenue Analytics**: Total revenue, trends, projections
|
||||
- **Order Analysis**: Counts, types, status tracking
|
||||
- **Product Performance**: Top sellers, revenue contribution
|
||||
- **Shipping Intelligence**: Methods, locations, distribution
|
||||
- **Customer Insights**: Order value ranges, patterns
|
||||
- **Operational Metrics**: Refunds, cancellations, peak hours
|
||||
|
||||
### Performance Optimizations
|
||||
- **Connection Pooling**: Efficient database connection management
|
||||
- **Query Optimization**: Indexed queries with proper WHERE clauses
|
||||
- **Caching Strategy**: Frontend caching for detail views
|
||||
- **Batch Processing**: Efficient data aggregation
|
||||
|
||||
### Error Handling
|
||||
- **Database Connectivity**: Graceful handling of connection issues
|
||||
- **Query Failures**: Detailed error logging and user-friendly messages
|
||||
- **Data Validation**: Input sanitization and validation
|
||||
- **Fallback Mechanisms**: Default values for missing data
|
||||
|
||||
## Simplified Elements
|
||||
|
||||
Due to database complexity, some features are simplified:
|
||||
- **Brands**: Shows "Various Brands" (companies table structure complex)
|
||||
- **Categories**: Shows "General" (category relationships complex)
|
||||
|
||||
These can be enhanced in future iterations with proper category mapping.
|
||||
|
||||
## Testing
|
||||
|
||||
Test the server functionality:
|
||||
|
||||
```bash
|
||||
# Health check
|
||||
curl http://localhost:3007/api/acot/test
|
||||
|
||||
# Today's stats
|
||||
curl http://localhost:3007/api/acot/events/stats?timeRange=today
|
||||
|
||||
# Last 30 days with details
|
||||
curl http://localhost:3007/api/acot/events/stats/details?timeRange=last30days&daily=true
|
||||
|
||||
# Top products
|
||||
curl http://localhost:3007/api/acot/events/products?timeRange=thisWeek
|
||||
|
||||
# Revenue projection
|
||||
curl http://localhost:3007/api/acot/events/projection?timeRange=today
|
||||
```
|
||||
|
||||
## Development Notes
|
||||
|
||||
- **No Frontend Changes**: Complete drop-in replacement for Klaviyo
|
||||
- **API Compatibility**: Maintains exact response structure
|
||||
- **Business Logic**: Implements all complex e-commerce calculations
|
||||
- **Scalability**: Designed for production workloads
|
||||
- **Maintainability**: Well-documented code with clear separation of concerns
|
||||
|
||||
## Future Enhancements
|
||||
|
||||
- Enhanced category and brand mapping
|
||||
- Real-time notifications for significant events
|
||||
- Advanced analytics and forecasting
|
||||
- Customer segmentation analysis
|
||||
- Inventory integration
|
||||
@@ -0,0 +1,297 @@
|
||||
const { Client } = require('ssh2');
|
||||
const mysql = require('mysql2/promise');
|
||||
const fs = require('fs');
|
||||
|
||||
// Connection pool configuration
|
||||
const connectionPool = {
|
||||
connections: [],
|
||||
maxConnections: 20,
|
||||
currentConnections: 0,
|
||||
pendingRequests: [],
|
||||
// Cache for query results (key: query string, value: {data, timestamp})
|
||||
queryCache: new Map(),
|
||||
// Cache duration for different query types in milliseconds
|
||||
cacheDuration: {
|
||||
'stats': 60 * 1000, // 1 minute for stats
|
||||
'products': 5 * 60 * 1000, // 5 minutes for products
|
||||
'orders': 60 * 1000, // 1 minute for orders
|
||||
'default': 60 * 1000 // 1 minute default
|
||||
},
|
||||
// Circuit breaker state
|
||||
circuitBreaker: {
|
||||
failures: 0,
|
||||
lastFailure: 0,
|
||||
isOpen: false,
|
||||
threshold: 5,
|
||||
timeout: 30000 // 30 seconds
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Get a database connection from the pool
|
||||
* @returns {Promise<{connection: object, release: function}>} The database connection and release function
|
||||
*/
|
||||
async function getDbConnection() {
|
||||
return new Promise(async (resolve, reject) => {
|
||||
// Check circuit breaker
|
||||
const now = Date.now();
|
||||
if (connectionPool.circuitBreaker.isOpen) {
|
||||
if (now - connectionPool.circuitBreaker.lastFailure > connectionPool.circuitBreaker.timeout) {
|
||||
// Reset circuit breaker
|
||||
connectionPool.circuitBreaker.isOpen = false;
|
||||
connectionPool.circuitBreaker.failures = 0;
|
||||
console.log('Circuit breaker reset');
|
||||
} else {
|
||||
reject(new Error('Circuit breaker is open - too many connection failures'));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Check if there's an available connection in the pool
|
||||
if (connectionPool.connections.length > 0) {
|
||||
const conn = connectionPool.connections.pop();
|
||||
console.log(`Using pooled connection. Pool size: ${connectionPool.connections.length}`);
|
||||
resolve({
|
||||
connection: conn.connection,
|
||||
release: () => releaseConnection(conn)
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// If we haven't reached max connections, create a new one
|
||||
if (connectionPool.currentConnections < connectionPool.maxConnections) {
|
||||
try {
|
||||
console.log(`Creating new connection. Current: ${connectionPool.currentConnections}/${connectionPool.maxConnections}`);
|
||||
connectionPool.currentConnections++;
|
||||
|
||||
const tunnel = await setupSshTunnel();
|
||||
const { ssh, stream, dbConfig } = tunnel;
|
||||
|
||||
const connection = await mysql.createConnection({
|
||||
...dbConfig,
|
||||
stream
|
||||
});
|
||||
|
||||
const conn = { ssh, connection, inUse: true, created: Date.now() };
|
||||
|
||||
console.log('Database connection established');
|
||||
|
||||
// Reset circuit breaker on successful connection
|
||||
if (connectionPool.circuitBreaker.failures > 0) {
|
||||
connectionPool.circuitBreaker.failures = 0;
|
||||
connectionPool.circuitBreaker.isOpen = false;
|
||||
}
|
||||
|
||||
resolve({
|
||||
connection: conn.connection,
|
||||
release: () => releaseConnection(conn)
|
||||
});
|
||||
} catch (error) {
|
||||
connectionPool.currentConnections--;
|
||||
|
||||
// Track circuit breaker failures
|
||||
connectionPool.circuitBreaker.failures++;
|
||||
connectionPool.circuitBreaker.lastFailure = Date.now();
|
||||
|
||||
if (connectionPool.circuitBreaker.failures >= connectionPool.circuitBreaker.threshold) {
|
||||
connectionPool.circuitBreaker.isOpen = true;
|
||||
console.log(`Circuit breaker opened after ${connectionPool.circuitBreaker.failures} failures`);
|
||||
}
|
||||
|
||||
reject(error);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Pool is full, queue the request with timeout
|
||||
console.log('Connection pool full, queuing request...');
|
||||
const timeoutId = setTimeout(() => {
|
||||
// Remove from queue if still there
|
||||
const index = connectionPool.pendingRequests.findIndex(req => req.resolve === resolve);
|
||||
if (index !== -1) {
|
||||
connectionPool.pendingRequests.splice(index, 1);
|
||||
reject(new Error('Connection pool queue timeout after 15 seconds'));
|
||||
}
|
||||
}, 15000);
|
||||
|
||||
connectionPool.pendingRequests.push({
|
||||
resolve,
|
||||
reject,
|
||||
timeoutId,
|
||||
timestamp: Date.now()
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Release a connection back to the pool
|
||||
*/
|
||||
function releaseConnection(conn) {
|
||||
conn.inUse = false;
|
||||
|
||||
// Check if there are pending requests
|
||||
if (connectionPool.pendingRequests.length > 0) {
|
||||
const { resolve, timeoutId } = connectionPool.pendingRequests.shift();
|
||||
|
||||
// Clear the timeout since we're serving the request
|
||||
if (timeoutId) {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
|
||||
conn.inUse = true;
|
||||
console.log(`Serving queued request. Queue length: ${connectionPool.pendingRequests.length}`);
|
||||
resolve({
|
||||
connection: conn.connection,
|
||||
release: () => releaseConnection(conn)
|
||||
});
|
||||
} else {
|
||||
// Return to pool
|
||||
connectionPool.connections.push(conn);
|
||||
console.log(`Connection returned to pool. Pool size: ${connectionPool.connections.length}, Active: ${connectionPool.currentConnections}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cached query results or execute query if not cached
|
||||
* @param {string} cacheKey - Unique key to identify the query
|
||||
* @param {string} queryType - Type of query (stats, products, orders, etc.)
|
||||
* @param {Function} queryFn - Function to execute if cache miss
|
||||
* @returns {Promise<any>} The query result
|
||||
*/
|
||||
async function getCachedQuery(cacheKey, queryType, queryFn) {
|
||||
// Get cache duration based on query type
|
||||
const cacheDuration = connectionPool.cacheDuration[queryType] || connectionPool.cacheDuration.default;
|
||||
|
||||
// Check if we have a valid cached result
|
||||
const cachedResult = connectionPool.queryCache.get(cacheKey);
|
||||
const now = Date.now();
|
||||
|
||||
if (cachedResult && (now - cachedResult.timestamp < cacheDuration)) {
|
||||
console.log(`Cache hit for ${queryType} query: ${cacheKey}`);
|
||||
return cachedResult.data;
|
||||
}
|
||||
|
||||
// No valid cache found, execute the query
|
||||
console.log(`Cache miss for ${queryType} query: ${cacheKey}`);
|
||||
const result = await queryFn();
|
||||
|
||||
// Cache the result
|
||||
connectionPool.queryCache.set(cacheKey, {
|
||||
data: result,
|
||||
timestamp: now
|
||||
});
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup SSH tunnel to production database
|
||||
* @private - Should only be used by getDbConnection
|
||||
* @returns {Promise<{ssh: object, stream: object, dbConfig: object}>}
|
||||
*/
|
||||
async function setupSshTunnel() {
|
||||
const sshConfig = {
|
||||
host: process.env.PROD_SSH_HOST,
|
||||
port: process.env.PROD_SSH_PORT || 22,
|
||||
username: process.env.PROD_SSH_USER,
|
||||
privateKey: process.env.PROD_SSH_KEY_PATH
|
||||
? fs.readFileSync(process.env.PROD_SSH_KEY_PATH)
|
||||
: undefined,
|
||||
compress: true
|
||||
};
|
||||
|
||||
const dbConfig = {
|
||||
host: process.env.PROD_DB_HOST || 'localhost',
|
||||
user: process.env.PROD_DB_USER,
|
||||
password: process.env.PROD_DB_PASSWORD,
|
||||
database: process.env.PROD_DB_NAME,
|
||||
port: process.env.PROD_DB_PORT || 3306,
|
||||
timezone: 'Z'
|
||||
};
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const ssh = new Client();
|
||||
|
||||
ssh.on('error', (err) => {
|
||||
console.error('SSH connection error:', err);
|
||||
reject(err);
|
||||
});
|
||||
|
||||
ssh.on('ready', () => {
|
||||
ssh.forwardOut(
|
||||
'127.0.0.1',
|
||||
0,
|
||||
dbConfig.host,
|
||||
dbConfig.port,
|
||||
(err, stream) => {
|
||||
if (err) reject(err);
|
||||
resolve({ ssh, stream, dbConfig });
|
||||
}
|
||||
);
|
||||
}).connect(sshConfig);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear cached query results
|
||||
* @param {string} [cacheKey] - Specific cache key to clear (clears all if not provided)
|
||||
*/
|
||||
function clearQueryCache(cacheKey) {
|
||||
if (cacheKey) {
|
||||
connectionPool.queryCache.delete(cacheKey);
|
||||
console.log(`Cleared cache for key: ${cacheKey}`);
|
||||
} else {
|
||||
connectionPool.queryCache.clear();
|
||||
console.log('Cleared all query cache');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Force close all active connections
|
||||
* Useful for server shutdown or manual connection reset
|
||||
*/
|
||||
async function closeAllConnections() {
|
||||
// Close all pooled connections
|
||||
for (const conn of connectionPool.connections) {
|
||||
try {
|
||||
await conn.connection.end();
|
||||
conn.ssh.end();
|
||||
console.log('Closed pooled connection');
|
||||
} catch (error) {
|
||||
console.error('Error closing pooled connection:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Reset pool state
|
||||
connectionPool.connections = [];
|
||||
connectionPool.currentConnections = 0;
|
||||
connectionPool.pendingRequests = [];
|
||||
connectionPool.queryCache.clear();
|
||||
|
||||
console.log('All connections closed and pool reset');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get connection pool status for debugging
|
||||
*/
|
||||
function getPoolStatus() {
|
||||
return {
|
||||
poolSize: connectionPool.connections.length,
|
||||
activeConnections: connectionPool.currentConnections,
|
||||
maxConnections: connectionPool.maxConnections,
|
||||
pendingRequests: connectionPool.pendingRequests.length,
|
||||
cacheSize: connectionPool.queryCache.size,
|
||||
queuedRequests: connectionPool.pendingRequests.map(req => ({
|
||||
waitTime: Date.now() - req.timestamp,
|
||||
hasTimeout: !!req.timeoutId
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
getDbConnection,
|
||||
getCachedQuery,
|
||||
clearQueryCache,
|
||||
closeAllConnections,
|
||||
getPoolStatus
|
||||
};
|
||||
+1553
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"name": "acot-server",
|
||||
"version": "1.0.0",
|
||||
"description": "A Cherry On Top production database server",
|
||||
"main": "server.js",
|
||||
"scripts": {
|
||||
"start": "node server.js",
|
||||
"dev": "nodemon server.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"express": "^4.18.2",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^16.3.1",
|
||||
"morgan": "^1.10.0",
|
||||
"ssh2": "^1.14.0",
|
||||
"mysql2": "^3.6.5",
|
||||
"compression": "^1.7.4",
|
||||
"luxon": "^3.5.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nodemon": "^3.0.1"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,322 @@
|
||||
// Customer lookup for the phone app (acot-phone-server).
|
||||
//
|
||||
// All queries hit the MySQL `sg` database via the shared SSH-tunneled pool in
|
||||
// db/connection.js. The stats/orders logic mirrors the freescout
|
||||
// ACOTCustomerData module so both apps display the same numbers for a given
|
||||
// customer — the difference is that we key by phone, not email.
|
||||
//
|
||||
// NOTE: `users.phone` is not yet indexed in production. Admin will add
|
||||
// `idx_phone (phone)` — queries here assume that exists for acceptable latency.
|
||||
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const { getDbConnection, getCachedQuery } = require('../db/connection');
|
||||
const { requirePhoneApiKey } = require('../utils/phoneAuth');
|
||||
|
||||
// Order status labels mirror ACOTCustomerDataServiceProvider.php.
|
||||
const ORDER_STATUS_LABEL = {
|
||||
0: 'Created', 10: 'Incomplete', 15: 'Cancelled', 16: 'Combined',
|
||||
20: 'Placed', 22: 'Placed (Incomplete)', 40: 'Awaiting Payment',
|
||||
45: 'Payment Pending', 50: 'Awaiting Products', 55: 'Shipping Later',
|
||||
56: 'Shipping Together', 60: 'Ready', 61: 'Flagged', 62: 'Fix Before Pick',
|
||||
65: 'Manual Picking', 67: 'Remote Send', 70: 'In PT', 80: 'Picked',
|
||||
90: 'Awaiting Shipment', 91: 'Remote Wait', 92: 'Awaiting Pickup',
|
||||
93: 'Fix Before Ship', 95: 'Shipped (Confirmed)', 100: 'Shipped',
|
||||
};
|
||||
const ORDER_STATUS_SHORT = {
|
||||
0: 'Created', 10: 'Incomplete', 15: 'Cancelled', 16: 'Combined',
|
||||
20: 'Placed', 22: 'Plcd Incomp', 40: 'Await Payment', 45: 'Pymt Pending',
|
||||
50: 'Await Products', 55: 'Ship Later', 56: 'Ship Togethr', 60: 'Ready',
|
||||
61: 'Flagged', 62: 'Fix Bfr Pick', 65: 'Manual Pick', 67: 'Remote Send',
|
||||
70: 'In PT', 80: 'Picked', 90: 'Await Ship', 91: 'Remote Wait',
|
||||
92: 'Await Pickup', 93: 'Fix Bfr Ship', 95: 'Shpd Confirm', 100: 'Shipped',
|
||||
};
|
||||
|
||||
function statusLabel(s) { return ORDER_STATUS_LABEL[s] ?? `Unknown (${s})`; }
|
||||
function statusShort(s) { return ORDER_STATUS_SHORT[s] ?? `Unknown (${s})`; }
|
||||
|
||||
// SIP trunks and historical CRM imports all disagree on phone format. Rather
|
||||
// than normalize everything upstream, we search across the most common
|
||||
// variations for US/Canada numbers. Falls through to the raw input for
|
||||
// international numbers we can't safely reformat.
|
||||
function phoneVariations(input) {
|
||||
const raw = String(input || '').trim();
|
||||
if (!raw) return [];
|
||||
const digits = raw.replace(/\D/g, '');
|
||||
const out = new Set([raw, digits]);
|
||||
if (digits.length === 10) {
|
||||
out.add(`+1${digits}`);
|
||||
out.add(`1${digits}`);
|
||||
} else if (digits.length === 11 && digits.startsWith('1')) {
|
||||
out.add(`+${digits}`);
|
||||
out.add(digits.slice(1)); // 10-digit form
|
||||
out.add(`+1${digits.slice(1)}`);
|
||||
}
|
||||
return Array.from(out).filter(Boolean);
|
||||
}
|
||||
|
||||
function trackingLink(method, tracking) {
|
||||
if (!tracking) return '';
|
||||
if (typeof method === 'string') {
|
||||
if (method.startsWith('usps_') || method === 'fedex_smartpost') {
|
||||
return `https://tools.usps.com/go/TrackConfirmAction?qtc_tLabels1=${tracking}`;
|
||||
}
|
||||
if (method.startsWith('fedex_')) {
|
||||
return `https://www.fedex.com/fedextrack/?trknbr=${tracking}`;
|
||||
}
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
// Matches ACOTCustomerDataServiceProvider::imageUrl — sbing.com/i/products/<dir1>/<dir2>/<pid>-t-<iid>.jpg
|
||||
function imageUrl(pid, iid = 1) {
|
||||
const padded = String(pid).padStart(10, '0');
|
||||
const dir1 = padded.slice(0, 4);
|
||||
const dir2 = padded.slice(4, 7);
|
||||
return `https://sbing.com/i/products/${dir1}/${dir2}/${pid}-t-${iid}.jpg`;
|
||||
}
|
||||
|
||||
router.use(requirePhoneApiKey);
|
||||
|
||||
// ── GET /by-phone ──────────────────────────────────────────────────────────
|
||||
// Returns top-line customer info for the incoming-call overlay.
|
||||
router.get('/by-phone', async (req, res) => {
|
||||
const phone = String(req.query.phone || '').trim();
|
||||
if (!phone) return res.status(400).json({ success: false, error: 'phone required' });
|
||||
|
||||
const variations = phoneVariations(phone);
|
||||
if (variations.length === 0) return res.json({ success: true, customer: null });
|
||||
|
||||
try {
|
||||
const data = await getCachedQuery(
|
||||
`customer-by-phone:${variations.join('|')}`,
|
||||
'default',
|
||||
async () => {
|
||||
const { connection, release } = await getDbConnection();
|
||||
try {
|
||||
const placeholders = variations.map(() => '?').join(',');
|
||||
// Tie-break by highest LTV per user instructions: subquery computes LTV
|
||||
// for every matching user, then we pick the biggest.
|
||||
const [users] = await connection.execute(
|
||||
`SELECT u.cid, u.uid, u.firstname, u.lastname, u.email, u.phone, u.points,
|
||||
COALESCE((
|
||||
SELECT SUM(summary_total)
|
||||
FROM _order
|
||||
WHERE order_cid = u.cid AND order_status >= 50
|
||||
), 0) AS lifetime_value,
|
||||
COALESCE((
|
||||
SELECT COUNT(*)
|
||||
FROM _order
|
||||
WHERE order_cid = u.cid AND order_status >= 20
|
||||
), 0) AS num_orders,
|
||||
(
|
||||
SELECT AVG(summary_total)
|
||||
FROM _order
|
||||
WHERE order_cid = u.cid AND order_status >= 20
|
||||
) AS avg_order
|
||||
FROM users u
|
||||
WHERE u.phone IN (${placeholders})
|
||||
ORDER BY lifetime_value DESC
|
||||
LIMIT 1`,
|
||||
variations
|
||||
);
|
||||
return users[0] ?? null;
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
if (!data) return res.json({ success: true, customer: null });
|
||||
res.json({
|
||||
success: true,
|
||||
customer: {
|
||||
cid: Number(data.cid),
|
||||
uid: data.uid,
|
||||
firstName: data.firstname || null,
|
||||
lastName: data.lastname || null,
|
||||
email: data.email || null,
|
||||
phone: data.phone,
|
||||
points: Number(data.points) || 0,
|
||||
lifetimeValue: Number(data.lifetime_value) || 0,
|
||||
orderCount: Number(data.num_orders) || 0,
|
||||
avgOrderValue: data.avg_order != null ? Number(data.avg_order) : 0,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('customers/by-phone failed:', err);
|
||||
res.status(500).json({ success: false, error: 'query_failed' });
|
||||
}
|
||||
});
|
||||
|
||||
// ── GET /search ────────────────────────────────────────────────────────────
|
||||
// Name search for the dialer. Accepts a free-text query; splits on whitespace.
|
||||
// - 1 token: LIKE against firstname OR lastname (prefix).
|
||||
// - 2+ tokens: firstname LIKE A% AND lastname LIKE B% (order-sensitive on purpose).
|
||||
router.get('/search', async (req, res) => {
|
||||
const q = String(req.query.q || '').trim();
|
||||
const limit = Math.min(Math.max(parseInt(req.query.limit || '10', 10) || 10, 1), 25);
|
||||
if (q.length < 2) return res.json({ success: true, results: [] });
|
||||
|
||||
try {
|
||||
const data = await getCachedQuery(
|
||||
`customer-search:${q}:${limit}`,
|
||||
'default',
|
||||
async () => {
|
||||
const { connection, release } = await getDbConnection();
|
||||
try {
|
||||
const tokens = q.split(/\s+/).filter(Boolean);
|
||||
let sql;
|
||||
let params;
|
||||
if (tokens.length === 1) {
|
||||
const pattern = `${tokens[0]}%`;
|
||||
sql = `SELECT cid, firstname, lastname, email, phone
|
||||
FROM users
|
||||
WHERE (firstname LIKE ? OR lastname LIKE ?)
|
||||
AND phone <> ''
|
||||
ORDER BY lastname, firstname
|
||||
LIMIT ?`;
|
||||
params = [pattern, pattern, limit];
|
||||
} else {
|
||||
const firstPat = `${tokens[0]}%`;
|
||||
const lastPat = `${tokens.slice(1).join(' ')}%`;
|
||||
sql = `SELECT cid, firstname, lastname, email, phone
|
||||
FROM users
|
||||
WHERE firstname LIKE ? AND lastname LIKE ?
|
||||
AND phone <> ''
|
||||
ORDER BY lastname, firstname
|
||||
LIMIT ?`;
|
||||
params = [firstPat, lastPat, limit];
|
||||
}
|
||||
const [rows] = await connection.execute(sql, params);
|
||||
return rows;
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
results: data.map((r) => ({
|
||||
cid: Number(r.cid),
|
||||
firstName: r.firstname || null,
|
||||
lastName: r.lastname || null,
|
||||
email: r.email || null,
|
||||
phone: r.phone,
|
||||
})),
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('customers/search failed:', err);
|
||||
res.status(500).json({ success: false, error: 'query_failed' });
|
||||
}
|
||||
});
|
||||
|
||||
// ── GET /:cid/orders ───────────────────────────────────────────────────────
|
||||
// Recent orders for the active-call screen — mirrors the freescout sidebar.
|
||||
router.get('/:cid/orders', async (req, res) => {
|
||||
const cid = Number(req.params.cid);
|
||||
if (!Number.isFinite(cid) || cid <= 0) {
|
||||
return res.status(400).json({ success: false, error: 'bad_cid' });
|
||||
}
|
||||
|
||||
try {
|
||||
const data = await getCachedQuery(
|
||||
`customer-orders:${cid}`,
|
||||
'orders',
|
||||
async () => {
|
||||
const { connection, release } = await getDbConnection();
|
||||
try {
|
||||
// MySQL-safe equivalent of the Laravel query in the freescout module.
|
||||
// Active = placed OR shipped within the last 3 months.
|
||||
const [ordersRaw] = await connection.execute(
|
||||
`SELECT order_id, order_status, order_type, summary_total,
|
||||
date_placed, ship_method_type, ship_method_tracking,
|
||||
CASE
|
||||
WHEN (order_status BETWEEN 20 AND 92
|
||||
OR date_shipped > DATE_SUB(NOW(), INTERVAL 3 MONTH))
|
||||
THEN 1 ELSE 0
|
||||
END AS _is_active
|
||||
FROM _order
|
||||
WHERE order_cid = ?
|
||||
AND (order_status >= 20
|
||||
OR date_shipped > DATE_SUB(NOW(), INTERVAL 3 MONTH))
|
||||
ORDER BY _is_active DESC, date_placed DESC`,
|
||||
[cid]
|
||||
);
|
||||
|
||||
const active = ordersRaw.filter((o) => o._is_active === 1);
|
||||
const inactive = ordersRaw.filter((o) => o._is_active === 0);
|
||||
const orders = active.concat(inactive.slice(0, Math.max(0, 10 - active.length)));
|
||||
|
||||
if (orders.length === 0) return [];
|
||||
|
||||
const orderIds = orders.map((o) => o.order_id);
|
||||
const idPlaceholders = orderIds.map(() => '?').join(',');
|
||||
|
||||
const [items] = await connection.execute(
|
||||
`SELECT order_id, prod_pid, prod_itemnumber, prod_description, prod_price, qty_ordered
|
||||
FROM order_items
|
||||
WHERE order_id IN (${idPlaceholders})`,
|
||||
orderIds
|
||||
);
|
||||
|
||||
// Main-image lookup: per-pid highest \`order\` at type=3 (matches the
|
||||
// freescout module's raw SQL).
|
||||
const pids = [...new Set(items.map((i) => Number(i.prod_pid)).filter(Boolean))];
|
||||
const mainImagesByPid = new Map();
|
||||
if (pids.length > 0) {
|
||||
const pidList = pids.join(',');
|
||||
const [imgRows] = await connection.execute(
|
||||
`SELECT pi.pid, pi.iid
|
||||
FROM product_images pi
|
||||
INNER JOIN (
|
||||
SELECT pid, MAX(\`order\`) AS max_order
|
||||
FROM product_images
|
||||
WHERE pid IN (${pidList}) AND type = 3
|
||||
GROUP BY pid
|
||||
) pm ON pi.pid = pm.pid AND pi.\`order\` = pm.max_order AND pi.type = 3`
|
||||
);
|
||||
for (const r of imgRows) mainImagesByPid.set(Number(r.pid), Number(r.iid));
|
||||
}
|
||||
|
||||
const itemsByOrder = new Map();
|
||||
for (const it of items) {
|
||||
const oid = Number(it.order_id);
|
||||
if (!itemsByOrder.has(oid)) itemsByOrder.set(oid, []);
|
||||
const iid = mainImagesByPid.get(Number(it.prod_pid)) ?? 1;
|
||||
itemsByOrder.get(oid).push({
|
||||
pid: Number(it.prod_pid),
|
||||
sku: it.prod_itemnumber || null,
|
||||
name: it.prod_description || null,
|
||||
price: Number(it.prod_price) || 0,
|
||||
quantity: Number(it.qty_ordered) || 0,
|
||||
imageUrl: imageUrl(it.prod_pid, iid),
|
||||
});
|
||||
}
|
||||
|
||||
return orders.map((o) => ({
|
||||
orderId: Number(o.order_id),
|
||||
datePlaced: o.date_placed,
|
||||
total: Number(o.summary_total) || 0,
|
||||
status: Number(o.order_status),
|
||||
statusLabel: statusLabel(Number(o.order_status)),
|
||||
statusShort: statusShort(Number(o.order_status)),
|
||||
trackingNumber: o.ship_method_tracking || '',
|
||||
trackingUrl: trackingLink(o.ship_method_type, o.ship_method_tracking),
|
||||
items: itemsByOrder.get(Number(o.order_id)) || [],
|
||||
}));
|
||||
} finally {
|
||||
release();
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
res.json({ success: true, orders: data });
|
||||
} catch (err) {
|
||||
console.error('customers/:cid/orders failed:', err);
|
||||
res.status(500).json({ success: false, error: 'query_failed' });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,576 @@
|
||||
const express = require('express');
|
||||
const { DateTime } = require('luxon');
|
||||
const { getDbConnection } = require('../db/connection');
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
// Bucket boundaries by summary_subtotal (post-item-sale, pre-order-promo).
|
||||
// The final entry is open-ended: all orders >= the last bound land there.
|
||||
const RANGE_BOUNDS = [
|
||||
10, 20, 30, 40, 50, 60, 70, 80, 90,
|
||||
100, 110, 120, 130, 140, 150, 160, 170, 180, 190, 200,
|
||||
300, 400, 500, 1000, 1500
|
||||
];
|
||||
|
||||
const FINAL_BUCKET_KEY = '99999';
|
||||
|
||||
function buildRangeDefinitions() {
|
||||
const ranges = [];
|
||||
let previous = 0;
|
||||
for (const bound of RANGE_BOUNDS) {
|
||||
const key = bound.toString().padStart(5, '0');
|
||||
ranges.push({
|
||||
min: previous,
|
||||
max: bound,
|
||||
label: `$${previous.toLocaleString()} - $${bound.toLocaleString()}`,
|
||||
key,
|
||||
});
|
||||
previous = bound;
|
||||
}
|
||||
const lastBound = RANGE_BOUNDS[RANGE_BOUNDS.length - 1];
|
||||
ranges.push({
|
||||
min: lastBound,
|
||||
max: null,
|
||||
label: `$${lastBound.toLocaleString()}+`,
|
||||
key: FINAL_BUCKET_KEY,
|
||||
});
|
||||
return ranges;
|
||||
}
|
||||
|
||||
const RANGE_DEFINITIONS = buildRangeDefinitions();
|
||||
|
||||
function bucketKeyFor(subtotal) {
|
||||
for (const range of RANGE_DEFINITIONS) {
|
||||
if (range.max == null) return range.key;
|
||||
if (subtotal <= range.max) return range.key;
|
||||
}
|
||||
return FINAL_BUCKET_KEY;
|
||||
}
|
||||
|
||||
const DEFAULT_POINT_DOLLAR_VALUE = 0.005;
|
||||
|
||||
const DEFAULTS = {
|
||||
merchantFeePercent: 2.9,
|
||||
fixedCostPerOrder: 1.25,
|
||||
pointDollarValue: DEFAULT_POINT_DOLLAR_VALUE,
|
||||
};
|
||||
|
||||
function parseDate(value, fallback) {
|
||||
if (!value) {
|
||||
return fallback;
|
||||
}
|
||||
const parsed = DateTime.fromISO(value);
|
||||
if (!parsed.isValid) {
|
||||
return fallback;
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function formatDateForSql(dt) {
|
||||
return dt.toFormat('yyyy-LL-dd HH:mm:ss');
|
||||
}
|
||||
|
||||
router.get('/promos', async (req, res) => {
|
||||
let connection;
|
||||
try {
|
||||
const { connection: conn, release } = await getDbConnection();
|
||||
connection = conn;
|
||||
const releaseConnection = release;
|
||||
|
||||
const { startDate, endDate } = req.query || {};
|
||||
const now = DateTime.now().endOf('day');
|
||||
const defaultStart = now.minus({ years: 3 }).startOf('day');
|
||||
|
||||
const parsedStart = startDate ? parseDate(startDate, defaultStart).startOf('day') : defaultStart;
|
||||
const parsedEnd = endDate ? parseDate(endDate, now).endOf('day') : now;
|
||||
|
||||
const rangeStart = parsedStart <= parsedEnd ? parsedStart : parsedEnd;
|
||||
const rangeEnd = parsedEnd >= parsedStart ? parsedEnd : parsedStart;
|
||||
|
||||
const rangeStartSql = formatDateForSql(rangeStart);
|
||||
const rangeEndSql = formatDateForSql(rangeEnd);
|
||||
|
||||
const sql = `
|
||||
SELECT
|
||||
p.promo_id AS id,
|
||||
p.promo_code AS code,
|
||||
p.promo_description_online AS description_online,
|
||||
p.promo_description_private AS description_private,
|
||||
p.date_start,
|
||||
p.date_end,
|
||||
COALESCE(u.usage_count, 0) AS usage_count
|
||||
FROM promos p
|
||||
LEFT JOIN (
|
||||
SELECT
|
||||
discount_code,
|
||||
COUNT(DISTINCT order_id) AS usage_count
|
||||
FROM order_discounts
|
||||
WHERE discount_type = 10 AND discount_active = 1
|
||||
GROUP BY discount_code
|
||||
) u ON u.discount_code = p.promo_id
|
||||
WHERE p.date_start IS NOT NULL
|
||||
AND p.date_end IS NOT NULL
|
||||
AND NOT (p.date_end < ? OR p.date_start > ?)
|
||||
AND p.store = 1
|
||||
AND p.date_start >= '2010-01-01'
|
||||
ORDER BY p.promo_id DESC
|
||||
LIMIT 200
|
||||
`;
|
||||
|
||||
const [rows] = await connection.execute(sql, [rangeStartSql, rangeEndSql]);
|
||||
releaseConnection();
|
||||
|
||||
const promos = rows.map(row => ({
|
||||
id: Number(row.id),
|
||||
code: row.code,
|
||||
description: row.description_online || row.description_private || '',
|
||||
privateDescription: row.description_private || '',
|
||||
promo_description_online: row.description_online || '',
|
||||
promo_description_private: row.description_private || '',
|
||||
dateStart: row.date_start,
|
||||
dateEnd: row.date_end,
|
||||
usageCount: Number(row.usage_count || 0)
|
||||
}));
|
||||
|
||||
res.json({ promos });
|
||||
} catch (error) {
|
||||
if (connection) {
|
||||
try {
|
||||
connection.destroy();
|
||||
} catch (destroyError) {
|
||||
console.error('Failed to destroy connection after error:', destroyError);
|
||||
}
|
||||
}
|
||||
console.error('Error fetching promos:', error);
|
||||
res.status(500).json({ error: 'Failed to fetch promos' });
|
||||
}
|
||||
});
|
||||
|
||||
function emptyBucketAccumulator(range) {
|
||||
return {
|
||||
key: range.key,
|
||||
label: range.label,
|
||||
min: range.min,
|
||||
max: range.max,
|
||||
orderCount: 0,
|
||||
sumOrderValue: 0,
|
||||
sumProductDiscountAmount: 0,
|
||||
sumPromoProductDiscount: 0,
|
||||
sumCustomerItemCost: 0,
|
||||
sumShippingChargeBase: 0,
|
||||
sumShippingAfterAuto: 0,
|
||||
sumShipPromoDiscount: 0,
|
||||
sumShippingSurcharge: 0,
|
||||
sumOrderSurcharge: 0,
|
||||
sumCustomerShipCost: 0,
|
||||
sumActualShippingCost: 0,
|
||||
sumTotalRevenue: 0,
|
||||
sumProductCogs: 0,
|
||||
sumMerchantFees: 0,
|
||||
sumPointsCost: 0,
|
||||
sumFixedCosts: 0,
|
||||
sumTotalCosts: 0,
|
||||
sumProfit: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function simulateOrder(order, config, derived) {
|
||||
const orderValue = Number(order.summary_subtotal) || 0;
|
||||
const retail = Number(order.summary_subtotal_retail) || orderValue;
|
||||
const productDiscountAmount = Number(order.summary_discount_subtotal) || 0;
|
||||
const pointsRedeemedDollars = Number(order.points_redeemed) || 0;
|
||||
// summary_discount_subtotal is a kitchen-sink rollup that includes points
|
||||
// redemptions (type 20). pointsCost already accrues for points awarded, so
|
||||
// the points portion of historical discount must be excluded here to avoid
|
||||
// double-counting it on orders that redeemed points.
|
||||
const historicalProductDiscountExPoints = Math.max(0, productDiscountAmount - pointsRedeemedDollars);
|
||||
const shippingChargeBase =
|
||||
(Number(order.summary_shipping) || 0) + (Number(order.summary_shipping_rush) || 0);
|
||||
const actualShippingCost = Number(order.ship_method_cost) || 0;
|
||||
const cogs = Number(order.total_cogs) || 0;
|
||||
|
||||
let promoProductDiscount = 0;
|
||||
if (config.productPromo.type === 'percentage_subtotal' && orderValue >= config.productPromo.minSubtotal) {
|
||||
promoProductDiscount = orderValue * (config.productPromo.value / 100);
|
||||
} else if (config.productPromo.type === 'percentage_regular' && orderValue >= config.productPromo.minSubtotal) {
|
||||
const targetRate = config.productPromo.value / 100;
|
||||
const targetCustomerPrice = retail * (1 - targetRate);
|
||||
promoProductDiscount = Math.max(0, orderValue - targetCustomerPrice);
|
||||
} else if (config.productPromo.type === 'fixed_amount' && orderValue >= config.productPromo.minSubtotal) {
|
||||
promoProductDiscount = config.productPromo.value;
|
||||
} else if (config.productPromo.type === 'none' && config.applyHistoricalProductPromo) {
|
||||
promoProductDiscount = historicalProductDiscountExPoints;
|
||||
}
|
||||
promoProductDiscount = Math.max(0, Math.min(promoProductDiscount, orderValue));
|
||||
|
||||
let shippingAfterAuto = shippingChargeBase;
|
||||
for (const tier of config.shippingTiers) {
|
||||
if (orderValue >= tier.threshold) {
|
||||
if (tier.mode === 'percentage') {
|
||||
shippingAfterAuto = shippingChargeBase * Math.max(0, 1 - tier.value / 100);
|
||||
} else if (tier.mode === 'flat') {
|
||||
shippingAfterAuto = tier.value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let shipPromoDiscount = 0;
|
||||
if (config.shippingPromo.type !== 'none' && orderValue >= config.shippingPromo.minSubtotal) {
|
||||
if (config.shippingPromo.type === 'percentage') {
|
||||
shipPromoDiscount = shippingAfterAuto * (config.shippingPromo.value / 100);
|
||||
} else if (config.shippingPromo.type === 'fixed') {
|
||||
shipPromoDiscount = config.shippingPromo.value;
|
||||
}
|
||||
if (config.shippingPromo.maxDiscount > 0) {
|
||||
shipPromoDiscount = Math.min(shipPromoDiscount, config.shippingPromo.maxDiscount);
|
||||
}
|
||||
shipPromoDiscount = Math.min(shipPromoDiscount, shippingAfterAuto);
|
||||
}
|
||||
|
||||
let shippingSurcharge = 0;
|
||||
let orderSurcharge = 0;
|
||||
for (const surcharge of config.surcharges) {
|
||||
const meetsMin = orderValue >= surcharge.threshold;
|
||||
const meetsMax = surcharge.maxThreshold == null || orderValue < surcharge.maxThreshold;
|
||||
if (meetsMin && meetsMax) {
|
||||
if (surcharge.target === 'shipping') shippingSurcharge += surcharge.amount;
|
||||
else if (surcharge.target === 'order') orderSurcharge += surcharge.amount;
|
||||
}
|
||||
}
|
||||
|
||||
const customerShipCost = Math.max(0, shippingAfterAuto - shipPromoDiscount + shippingSurcharge);
|
||||
const customerItemCost = Math.max(0, orderValue - promoProductDiscount + orderSurcharge);
|
||||
const totalRevenue = customerItemCost + customerShipCost;
|
||||
|
||||
const productCogs = config.cogsCalculationMode === 'average'
|
||||
? orderValue * derived.overallCogsPercentage
|
||||
: cogs;
|
||||
|
||||
const merchantFees = totalRevenue * (config.merchantFeePercent / 100);
|
||||
const pointsCost = orderValue * derived.pointsPerDollar * derived.redemptionRate * derived.pointDollarValue;
|
||||
const fixedCosts = config.fixedCostPerOrder;
|
||||
const totalCosts = productCogs + actualShippingCost + merchantFees + pointsCost + fixedCosts;
|
||||
const profit = totalRevenue - totalCosts;
|
||||
|
||||
return {
|
||||
orderValue,
|
||||
productDiscountAmount,
|
||||
promoProductDiscount,
|
||||
customerItemCost,
|
||||
shippingChargeBase,
|
||||
shippingAfterAuto,
|
||||
shipPromoDiscount,
|
||||
shippingSurcharge,
|
||||
orderSurcharge,
|
||||
customerShipCost,
|
||||
actualShippingCost,
|
||||
totalRevenue,
|
||||
productCogs,
|
||||
merchantFees,
|
||||
pointsCost,
|
||||
fixedCosts,
|
||||
totalCosts,
|
||||
profit,
|
||||
};
|
||||
}
|
||||
|
||||
function accumulate(bucket, sim) {
|
||||
bucket.orderCount += 1;
|
||||
bucket.sumOrderValue += sim.orderValue;
|
||||
bucket.sumProductDiscountAmount += sim.productDiscountAmount;
|
||||
bucket.sumPromoProductDiscount += sim.promoProductDiscount;
|
||||
bucket.sumCustomerItemCost += sim.customerItemCost;
|
||||
bucket.sumShippingChargeBase += sim.shippingChargeBase;
|
||||
bucket.sumShippingAfterAuto += sim.shippingAfterAuto;
|
||||
bucket.sumShipPromoDiscount += sim.shipPromoDiscount;
|
||||
bucket.sumShippingSurcharge += sim.shippingSurcharge;
|
||||
bucket.sumOrderSurcharge += sim.orderSurcharge;
|
||||
bucket.sumCustomerShipCost += sim.customerShipCost;
|
||||
bucket.sumActualShippingCost += sim.actualShippingCost;
|
||||
bucket.sumTotalRevenue += sim.totalRevenue;
|
||||
bucket.sumProductCogs += sim.productCogs;
|
||||
bucket.sumMerchantFees += sim.merchantFees;
|
||||
bucket.sumPointsCost += sim.pointsCost;
|
||||
bucket.sumFixedCosts += sim.fixedCosts;
|
||||
bucket.sumTotalCosts += sim.totalCosts;
|
||||
bucket.sumProfit += sim.profit;
|
||||
}
|
||||
|
||||
function finalizeBucket(b, totalOrders) {
|
||||
const n = b.orderCount;
|
||||
const avg = (sum) => (n > 0 ? sum / n : 0);
|
||||
return {
|
||||
key: b.key,
|
||||
label: b.label,
|
||||
min: b.min,
|
||||
max: b.max,
|
||||
orderCount: n,
|
||||
weight: totalOrders > 0 ? n / totalOrders : 0,
|
||||
orderValue: avg(b.sumOrderValue),
|
||||
productDiscountAmount: avg(b.sumProductDiscountAmount),
|
||||
promoProductDiscount: avg(b.sumPromoProductDiscount),
|
||||
customerItemCost: avg(b.sumCustomerItemCost),
|
||||
shippingChargeBase: avg(b.sumShippingChargeBase),
|
||||
shippingAfterAuto: avg(b.sumShippingAfterAuto),
|
||||
shipPromoDiscount: avg(b.sumShipPromoDiscount),
|
||||
shippingSurcharge: avg(b.sumShippingSurcharge),
|
||||
orderSurcharge: avg(b.sumOrderSurcharge),
|
||||
customerShipCost: avg(b.sumCustomerShipCost),
|
||||
actualShippingCost: avg(b.sumActualShippingCost),
|
||||
totalRevenue: avg(b.sumTotalRevenue),
|
||||
productCogs: avg(b.sumProductCogs),
|
||||
merchantFees: avg(b.sumMerchantFees),
|
||||
pointsCost: avg(b.sumPointsCost),
|
||||
fixedCosts: avg(b.sumFixedCosts),
|
||||
totalCosts: avg(b.sumTotalCosts),
|
||||
profit: avg(b.sumProfit),
|
||||
profitPercent: b.sumTotalRevenue > 0 ? b.sumProfit / b.sumTotalRevenue : 0,
|
||||
};
|
||||
}
|
||||
|
||||
router.post('/simulate', async (req, res) => {
|
||||
const {
|
||||
dateRange = {},
|
||||
filters = {},
|
||||
productPromo = {},
|
||||
shippingPromo = {},
|
||||
shippingTiers = [],
|
||||
surcharges = [],
|
||||
merchantFeePercent,
|
||||
fixedCostPerOrder,
|
||||
cogsCalculationMode = 'actual',
|
||||
applyHistoricalProductPromo = false,
|
||||
pointsConfig = {}
|
||||
} = req.body || {};
|
||||
|
||||
const endDefault = DateTime.now();
|
||||
const startDefault = endDefault.minus({ months: 6 });
|
||||
const startDt = parseDate(dateRange.start, startDefault).startOf('day');
|
||||
const endDt = parseDate(dateRange.end, endDefault).endOf('day');
|
||||
|
||||
const shipCountry = filters.shipCountry || 'US';
|
||||
const promoIds = Array.from(
|
||||
new Set(
|
||||
[
|
||||
...(Array.isArray(filters.promoIds) ? filters.promoIds : []),
|
||||
...(Array.isArray(filters.promoCodes) ? filters.promoCodes : []),
|
||||
]
|
||||
.map((value) => {
|
||||
if (typeof value === 'string') return value.trim();
|
||||
if (typeof value === 'number') return String(value);
|
||||
return '';
|
||||
})
|
||||
.filter((value) => value.length > 0)
|
||||
)
|
||||
);
|
||||
|
||||
const config = {
|
||||
merchantFeePercent: typeof merchantFeePercent === 'number' ? merchantFeePercent : DEFAULTS.merchantFeePercent,
|
||||
fixedCostPerOrder: typeof fixedCostPerOrder === 'number' ? fixedCostPerOrder : DEFAULTS.fixedCostPerOrder,
|
||||
cogsCalculationMode,
|
||||
applyHistoricalProductPromo: applyHistoricalProductPromo === true,
|
||||
productPromo: {
|
||||
type: productPromo.type || 'none',
|
||||
value: Number(productPromo.value || 0),
|
||||
minSubtotal: Number(productPromo.minSubtotal || 0)
|
||||
},
|
||||
shippingPromo: {
|
||||
type: shippingPromo.type || 'none',
|
||||
value: Number(shippingPromo.value || 0),
|
||||
minSubtotal: Number(shippingPromo.minSubtotal || 0),
|
||||
maxDiscount: Number(shippingPromo.maxDiscount || 0)
|
||||
},
|
||||
shippingTiers: Array.isArray(shippingTiers)
|
||||
? shippingTiers
|
||||
.map(tier => ({
|
||||
threshold: Number(tier.threshold || 0),
|
||||
mode: tier.mode === 'percentage' || tier.mode === 'flat' ? tier.mode : 'percentage',
|
||||
value: Number(tier.value || 0)
|
||||
}))
|
||||
.filter(tier => tier.threshold >= 0 && tier.value >= 0)
|
||||
.sort((a, b) => a.threshold - b.threshold)
|
||||
: [],
|
||||
surcharges: Array.isArray(surcharges)
|
||||
? surcharges
|
||||
.map(s => ({
|
||||
threshold: Number(s.threshold || 0),
|
||||
maxThreshold: typeof s.maxThreshold === 'number' && s.maxThreshold > 0 ? s.maxThreshold : null,
|
||||
target: s.target === 'shipping' || s.target === 'order' ? s.target : 'shipping',
|
||||
amount: Number(s.amount || 0)
|
||||
}))
|
||||
.filter(s => s.threshold >= 0 && s.amount >= 0)
|
||||
.sort((a, b) => a.threshold - b.threshold)
|
||||
: [],
|
||||
points: {
|
||||
pointsPerDollar: typeof pointsConfig.pointsPerDollar === 'number' ? pointsConfig.pointsPerDollar : null,
|
||||
redemptionRate: typeof pointsConfig.redemptionRate === 'number' ? pointsConfig.redemptionRate : null,
|
||||
pointDollarValue: typeof pointsConfig.pointDollarValue === 'number'
|
||||
? pointsConfig.pointDollarValue
|
||||
: DEFAULT_POINT_DOLLAR_VALUE
|
||||
}
|
||||
};
|
||||
|
||||
let connection;
|
||||
let release;
|
||||
|
||||
try {
|
||||
const dbConn = await getDbConnection();
|
||||
connection = dbConn.connection;
|
||||
release = dbConn.release;
|
||||
|
||||
const params = [shipCountry, formatDateForSql(startDt), formatDateForSql(endDt)];
|
||||
let promoExistsClause = '';
|
||||
if (promoIds.length > 0) {
|
||||
const placeholders = promoIds.map(() => '?').join(',');
|
||||
promoExistsClause = `
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM order_discounts od
|
||||
WHERE od.order_id = o.order_id
|
||||
AND od.discount_active = 1
|
||||
AND od.discount_type = 10
|
||||
AND od.discount_code IN (${placeholders})
|
||||
)
|
||||
`;
|
||||
params.push(...promoIds);
|
||||
}
|
||||
|
||||
const ordersQuery = `
|
||||
SELECT
|
||||
o.order_id,
|
||||
o.summary_subtotal,
|
||||
COALESCE(o.summary_subtotal_retail, o.summary_subtotal) AS summary_subtotal_retail,
|
||||
COALESCE(o.summary_discount_subtotal, 0) AS summary_discount_subtotal,
|
||||
COALESCE(o.summary_shipping, 0) AS summary_shipping,
|
||||
COALESCE(o.summary_shipping_rush, 0) AS summary_shipping_rush,
|
||||
COALESCE(o.ship_method_cost, 0) AS ship_method_cost,
|
||||
COALESCE(o.summary_points, 0) AS summary_points,
|
||||
COALESCE(c.total_cogs, 0) AS total_cogs,
|
||||
COALESCE(p.points_redeemed, 0) AS points_redeemed
|
||||
FROM _order o
|
||||
LEFT JOIN (
|
||||
SELECT order_id, SUM(cogs_amount) AS total_cogs
|
||||
FROM report_sales_data
|
||||
WHERE action IN (1,2,3)
|
||||
GROUP BY order_id
|
||||
) c ON c.order_id = o.order_id
|
||||
LEFT JOIN (
|
||||
SELECT order_id, SUM(discount_amount_subtotal) AS points_redeemed
|
||||
FROM order_discounts
|
||||
WHERE discount_type = 20 AND discount_active = 1
|
||||
GROUP BY order_id
|
||||
) p ON p.order_id = o.order_id
|
||||
WHERE o.summary_total > 0
|
||||
AND o.order_status >= 20
|
||||
AND o.ship_method_selected <> 'holdit'
|
||||
AND o.ship_country = ?
|
||||
AND o.date_placed BETWEEN ? AND ?
|
||||
${promoExistsClause}
|
||||
`;
|
||||
|
||||
const [orders] = await connection.execute(ordersQuery, params);
|
||||
|
||||
if (release) {
|
||||
release();
|
||||
release = null;
|
||||
}
|
||||
|
||||
let totalSubtotal = 0;
|
||||
let totalProductDiscount = 0;
|
||||
let totalCogs = 0;
|
||||
let totalPointsAwarded = 0;
|
||||
let totalPointsRedeemedDollars = 0;
|
||||
for (const o of orders) {
|
||||
totalSubtotal += Number(o.summary_subtotal) || 0;
|
||||
totalProductDiscount += Number(o.summary_discount_subtotal) || 0;
|
||||
totalCogs += Number(o.total_cogs) || 0;
|
||||
totalPointsAwarded += Number(o.summary_points) || 0;
|
||||
totalPointsRedeemedDollars += Number(o.points_redeemed) || 0;
|
||||
}
|
||||
|
||||
const productDiscountRate = totalSubtotal > 0 ? totalProductDiscount / totalSubtotal : 0;
|
||||
const overallCogsPercentage = totalSubtotal > 0 ? totalCogs / totalSubtotal : 0;
|
||||
const pointsPerDollar = config.points.pointsPerDollar != null
|
||||
? config.points.pointsPerDollar
|
||||
: (totalSubtotal > 0 ? totalPointsAwarded / totalSubtotal : 0);
|
||||
const pointDollarValue = config.points.pointDollarValue || DEFAULT_POINT_DOLLAR_VALUE;
|
||||
let redemptionRate;
|
||||
if (config.points.redemptionRate != null) {
|
||||
redemptionRate = config.points.redemptionRate;
|
||||
} else if (totalPointsAwarded > 0 && pointDollarValue > 0) {
|
||||
const totalRedeemedPoints = totalPointsRedeemedDollars / pointDollarValue;
|
||||
redemptionRate = Math.min(1, totalRedeemedPoints / totalPointsAwarded);
|
||||
} else {
|
||||
redemptionRate = 0;
|
||||
}
|
||||
|
||||
const derived = {
|
||||
overallCogsPercentage,
|
||||
pointsPerDollar,
|
||||
redemptionRate,
|
||||
pointDollarValue,
|
||||
};
|
||||
|
||||
const buckets = new Map();
|
||||
for (const range of RANGE_DEFINITIONS) {
|
||||
buckets.set(range.key, emptyBucketAccumulator(range));
|
||||
}
|
||||
|
||||
let grandTotalProfit = 0;
|
||||
let grandTotalRevenue = 0;
|
||||
|
||||
for (const order of orders) {
|
||||
const sim = simulateOrder(order, config, derived);
|
||||
const bucketKey = bucketKeyFor(sim.orderValue);
|
||||
const bucket = buckets.get(bucketKey);
|
||||
accumulate(bucket, sim);
|
||||
grandTotalProfit += sim.profit;
|
||||
grandTotalRevenue += sim.totalRevenue;
|
||||
}
|
||||
|
||||
const totalOrders = orders.length;
|
||||
const bucketResults = RANGE_DEFINITIONS.map((range) =>
|
||||
finalizeBucket(buckets.get(range.key), totalOrders)
|
||||
);
|
||||
|
||||
const weightedProfitAmount = totalOrders > 0 ? grandTotalProfit / totalOrders : 0;
|
||||
const weightedProfitPercent = grandTotalRevenue > 0 ? grandTotalProfit / grandTotalRevenue : 0;
|
||||
|
||||
res.json({
|
||||
dateRange: {
|
||||
start: startDt.toISO(),
|
||||
end: endDt.toISO()
|
||||
},
|
||||
totals: {
|
||||
orders: totalOrders,
|
||||
subtotal: totalSubtotal,
|
||||
productDiscountRate,
|
||||
pointsPerDollar,
|
||||
redemptionRate,
|
||||
pointDollarValue,
|
||||
weightedProfitAmount,
|
||||
weightedProfitPercent,
|
||||
overallCogsPercentage: cogsCalculationMode === 'average' ? overallCogsPercentage : undefined
|
||||
},
|
||||
buckets: bucketResults
|
||||
});
|
||||
} catch (error) {
|
||||
if (release) {
|
||||
try {
|
||||
release();
|
||||
} catch (releaseError) {
|
||||
console.error('Failed to release connection after error:', releaseError);
|
||||
}
|
||||
} else if (connection) {
|
||||
try {
|
||||
connection.destroy();
|
||||
} catch (destroyError) {
|
||||
console.error('Failed to destroy connection after error:', destroyError);
|
||||
}
|
||||
}
|
||||
|
||||
console.error('Error running discount simulation:', error);
|
||||
res.status(500).json({ error: 'Failed to run discount simulation' });
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,683 @@
|
||||
const express = require('express');
|
||||
const { DateTime } = require('luxon');
|
||||
|
||||
const router = express.Router();
|
||||
const { getDbConnection, getPoolStatus } = require('../db/connection');
|
||||
const {
|
||||
getTimeRangeConditions,
|
||||
_internal: timeHelpers
|
||||
} = require('../utils/timeUtils');
|
||||
|
||||
const TIMEZONE = 'America/New_York';
|
||||
|
||||
// Punch types from the database
|
||||
const PUNCH_TYPES = {
|
||||
OUT: 0,
|
||||
IN: 1,
|
||||
BREAK_START: 2,
|
||||
BREAK_END: 3,
|
||||
};
|
||||
|
||||
// Standard hours for FTE calculation (40 hours per week)
|
||||
const STANDARD_WEEKLY_HOURS = 40;
|
||||
|
||||
/**
|
||||
* Calculate working hours from timeclock entries
|
||||
* Groups punches by employee and date, pairs in/out punches
|
||||
* Returns both total hours (with breaks, for FTE) and productive hours (without breaks, for productivity)
|
||||
*/
|
||||
function calculateHoursFromPunches(punches) {
|
||||
// Group by employee
|
||||
const byEmployee = new Map();
|
||||
|
||||
punches.forEach(punch => {
|
||||
if (!byEmployee.has(punch.EmployeeID)) {
|
||||
byEmployee.set(punch.EmployeeID, []);
|
||||
}
|
||||
byEmployee.get(punch.EmployeeID).push(punch);
|
||||
});
|
||||
|
||||
const employeeHours = [];
|
||||
let totalHours = 0;
|
||||
let totalBreakHours = 0;
|
||||
|
||||
byEmployee.forEach((employeePunches, employeeId) => {
|
||||
// Sort by timestamp
|
||||
employeePunches.sort((a, b) => new Date(a.TimeStamp) - new Date(b.TimeStamp));
|
||||
|
||||
let hours = 0;
|
||||
let breakHours = 0;
|
||||
let currentIn = null;
|
||||
let breakStart = null;
|
||||
|
||||
employeePunches.forEach(punch => {
|
||||
const punchTime = new Date(punch.TimeStamp);
|
||||
|
||||
switch (punch.PunchType) {
|
||||
case PUNCH_TYPES.IN:
|
||||
currentIn = punchTime;
|
||||
break;
|
||||
case PUNCH_TYPES.OUT:
|
||||
if (currentIn) {
|
||||
hours += (punchTime - currentIn) / (1000 * 60 * 60); // Convert ms to hours
|
||||
currentIn = null;
|
||||
}
|
||||
break;
|
||||
case PUNCH_TYPES.BREAK_START:
|
||||
breakStart = punchTime;
|
||||
break;
|
||||
case PUNCH_TYPES.BREAK_END:
|
||||
if (breakStart) {
|
||||
breakHours += (punchTime - breakStart) / (1000 * 60 * 60);
|
||||
breakStart = null;
|
||||
}
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
totalHours += hours;
|
||||
totalBreakHours += breakHours;
|
||||
|
||||
employeeHours.push({
|
||||
employeeId,
|
||||
hours,
|
||||
breakHours,
|
||||
productiveHours: hours - breakHours,
|
||||
});
|
||||
});
|
||||
|
||||
return {
|
||||
employeeHours,
|
||||
totalHours,
|
||||
totalBreakHours,
|
||||
totalProductiveHours: totalHours - totalBreakHours
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate FTE (Full Time Equivalents) for a period
|
||||
* @param {number} totalHours - Total hours worked
|
||||
* @param {Date} startDate - Period start
|
||||
* @param {Date} endDate - Period end
|
||||
*/
|
||||
function calculateFTE(totalHours, startDate, endDate) {
|
||||
const start = new Date(startDate);
|
||||
const end = new Date(endDate);
|
||||
const days = Math.max(1, (end - start) / (1000 * 60 * 60 * 24));
|
||||
const weeks = days / 7;
|
||||
const expectedHours = weeks * STANDARD_WEEKLY_HOURS;
|
||||
|
||||
return expectedHours > 0 ? totalHours / expectedHours : 0;
|
||||
}
|
||||
|
||||
// Main employee metrics endpoint
|
||||
router.get('/', async (req, res) => {
|
||||
const startTime = Date.now();
|
||||
console.log(`[EMPLOYEE-METRICS] Starting request for timeRange: ${req.query.timeRange}`);
|
||||
|
||||
const timeoutPromise = new Promise((_, reject) => {
|
||||
setTimeout(() => reject(new Error('Request timeout after 30 seconds')), 30000);
|
||||
});
|
||||
|
||||
try {
|
||||
const mainOperation = async () => {
|
||||
const { timeRange, startDate, endDate } = req.query;
|
||||
console.log(`[EMPLOYEE-METRICS] Getting DB connection...`);
|
||||
const { connection, release } = await getDbConnection();
|
||||
console.log(`[EMPLOYEE-METRICS] DB connection obtained in ${Date.now() - startTime}ms`);
|
||||
|
||||
const { whereClause, params, dateRange } = getTimeRangeConditions(timeRange, startDate, endDate);
|
||||
|
||||
// Adapt where clause for timeclock table (uses TimeStamp instead of date_placed)
|
||||
const timeclockWhere = whereClause.replace(/date_placed/g, 'tc.TimeStamp');
|
||||
|
||||
// Query for timeclock data with employee names
|
||||
const timeclockQuery = `
|
||||
SELECT
|
||||
tc.EmployeeID,
|
||||
tc.TimeStamp,
|
||||
tc.PunchType,
|
||||
e.firstname,
|
||||
e.lastname
|
||||
FROM timeclock tc
|
||||
LEFT JOIN employees e ON tc.EmployeeID = e.employeeid
|
||||
WHERE ${timeclockWhere}
|
||||
AND e.hidden = 0
|
||||
AND e.disabled = 0
|
||||
ORDER BY tc.EmployeeID, tc.TimeStamp
|
||||
`;
|
||||
|
||||
const [timeclockRows] = await connection.execute(timeclockQuery, params);
|
||||
|
||||
// Calculate hours (includes both total hours for FTE and productive hours for productivity)
|
||||
const { employeeHours, totalHours, totalBreakHours, totalProductiveHours } = calculateHoursFromPunches(timeclockRows);
|
||||
|
||||
// Get employee names for the results
|
||||
const employeeNames = new Map();
|
||||
timeclockRows.forEach(row => {
|
||||
if (!employeeNames.has(row.EmployeeID)) {
|
||||
employeeNames.set(row.EmployeeID, {
|
||||
firstname: row.firstname || '',
|
||||
lastname: row.lastname || '',
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Enrich employee hours with names
|
||||
const enrichedEmployeeHours = employeeHours.map(eh => ({
|
||||
...eh,
|
||||
name: employeeNames.has(eh.employeeId)
|
||||
? `${employeeNames.get(eh.employeeId).firstname} ${employeeNames.get(eh.employeeId).lastname}`.trim()
|
||||
: `Employee ${eh.employeeId}`,
|
||||
})).sort((a, b) => b.hours - a.hours);
|
||||
|
||||
// Query for picking tickets - using subquery to avoid duplication from bucket join
|
||||
// Ship-together orders: only count main orders (is_sub = 0 or NULL), not sub-orders
|
||||
const pickingWhere = whereClause.replace(/date_placed/g, 'pt.createddate');
|
||||
|
||||
// First get picking ticket stats without the bucket join (to avoid duplication)
|
||||
const pickingStatsQuery = `
|
||||
SELECT
|
||||
pt.createdby as employeeId,
|
||||
e.firstname,
|
||||
e.lastname,
|
||||
COUNT(DISTINCT pt.pickingid) as ticketCount,
|
||||
SUM(pt.totalpieces_picked) as piecesPicked,
|
||||
SUM(TIMESTAMPDIFF(SECOND, pt.createddate, pt.closeddate)) as pickingTimeSeconds,
|
||||
AVG(NULLIF(pt.picking_speed, 0)) as avgPickingSpeed
|
||||
FROM picking_ticket pt
|
||||
LEFT JOIN employees e ON pt.createdby = e.employeeid
|
||||
WHERE ${pickingWhere}
|
||||
AND pt.closeddate IS NOT NULL
|
||||
GROUP BY pt.createdby, e.firstname, e.lastname
|
||||
`;
|
||||
|
||||
// Separate query for order counts (needs bucket join for ship-together handling)
|
||||
const orderCountQuery = `
|
||||
SELECT
|
||||
pt.createdby as employeeId,
|
||||
COUNT(DISTINCT CASE WHEN ptb.is_sub = 0 OR ptb.is_sub IS NULL THEN ptb.orderid END) as ordersPicked
|
||||
FROM picking_ticket pt
|
||||
LEFT JOIN picking_ticket_buckets ptb ON pt.pickingid = ptb.pickingid
|
||||
WHERE ${pickingWhere}
|
||||
AND pt.closeddate IS NOT NULL
|
||||
GROUP BY pt.createdby
|
||||
`;
|
||||
|
||||
const [[pickingStatsRows], [orderCountRows]] = await Promise.all([
|
||||
connection.execute(pickingStatsQuery, params),
|
||||
connection.execute(orderCountQuery, params)
|
||||
]);
|
||||
|
||||
// Merge the results
|
||||
const orderCountMap = new Map();
|
||||
orderCountRows.forEach(row => {
|
||||
orderCountMap.set(row.employeeId, parseInt(row.ordersPicked || 0));
|
||||
});
|
||||
|
||||
// Aggregate picking totals
|
||||
let totalOrdersPicked = 0;
|
||||
let totalPiecesPicked = 0;
|
||||
let totalTickets = 0;
|
||||
let totalPickingTimeSeconds = 0;
|
||||
let pickingSpeedSum = 0;
|
||||
let pickingSpeedCount = 0;
|
||||
|
||||
const pickingByEmployee = pickingStatsRows.map(row => {
|
||||
const ordersPicked = orderCountMap.get(row.employeeId) || 0;
|
||||
totalOrdersPicked += ordersPicked;
|
||||
totalPiecesPicked += parseInt(row.piecesPicked || 0);
|
||||
totalTickets += parseInt(row.ticketCount || 0);
|
||||
totalPickingTimeSeconds += parseInt(row.pickingTimeSeconds || 0);
|
||||
if (row.avgPickingSpeed && row.avgPickingSpeed > 0) {
|
||||
pickingSpeedSum += parseFloat(row.avgPickingSpeed);
|
||||
pickingSpeedCount++;
|
||||
}
|
||||
|
||||
const empPickingHours = parseInt(row.pickingTimeSeconds || 0) / 3600;
|
||||
|
||||
return {
|
||||
employeeId: row.employeeId,
|
||||
name: `${row.firstname || ''} ${row.lastname || ''}`.trim() || `Employee ${row.employeeId}`,
|
||||
ticketCount: parseInt(row.ticketCount || 0),
|
||||
ordersPicked,
|
||||
piecesPicked: parseInt(row.piecesPicked || 0),
|
||||
pickingHours: empPickingHours,
|
||||
avgPickingSpeed: row.avgPickingSpeed ? parseFloat(row.avgPickingSpeed) : null,
|
||||
};
|
||||
});
|
||||
|
||||
const totalPickingHours = totalPickingTimeSeconds / 3600;
|
||||
const avgPickingSpeed = pickingSpeedCount > 0 ? pickingSpeedSum / pickingSpeedCount : 0;
|
||||
|
||||
// Query for shipped orders - totals
|
||||
// Ship-together orders: only count main orders (order_type != 8 for sub-orders, or use parent tracking)
|
||||
const shippingWhere = whereClause.replace(/date_placed/g, 'o.date_shipped');
|
||||
|
||||
const shippingQuery = `
|
||||
SELECT
|
||||
COUNT(DISTINCT CASE WHEN o.order_type != 8 OR o.order_type IS NULL THEN o.order_id END) as ordersShipped,
|
||||
COALESCE(SUM(o.stats_prod_pieces), 0) as piecesShipped
|
||||
FROM _order o
|
||||
WHERE ${shippingWhere}
|
||||
AND o.order_status IN (100, 92)
|
||||
`;
|
||||
|
||||
const [shippingRows] = await connection.execute(shippingQuery, params);
|
||||
const shipping = shippingRows[0] || { ordersShipped: 0, piecesShipped: 0 };
|
||||
|
||||
// Query for shipped orders by employee
|
||||
const shippingByEmployeeQuery = `
|
||||
SELECT
|
||||
e.employeeid,
|
||||
e.firstname,
|
||||
e.lastname,
|
||||
COUNT(DISTINCT CASE WHEN o.order_type != 8 OR o.order_type IS NULL THEN o.order_id END) as ordersShipped,
|
||||
COALESCE(SUM(o.stats_prod_pieces), 0) as piecesShipped
|
||||
FROM _order o
|
||||
JOIN employees e ON o.stats_cid_shipped = e.cid
|
||||
WHERE ${shippingWhere}
|
||||
AND o.order_status IN (100, 92)
|
||||
AND e.hidden = 0
|
||||
AND e.disabled = 0
|
||||
GROUP BY e.employeeid, e.firstname, e.lastname
|
||||
ORDER BY ordersShipped DESC
|
||||
`;
|
||||
|
||||
const [shippingByEmployeeRows] = await connection.execute(shippingByEmployeeQuery, params);
|
||||
const shippingByEmployee = shippingByEmployeeRows.map(row => ({
|
||||
employeeId: row.employeeid,
|
||||
name: `${row.firstname || ''} ${row.lastname || ''}`.trim() || `Employee ${row.employeeid}`,
|
||||
ordersShipped: parseInt(row.ordersShipped || 0),
|
||||
piecesShipped: parseInt(row.piecesShipped || 0),
|
||||
}));
|
||||
|
||||
// Calculate period dates for FTE calculation
|
||||
let periodStart, periodEnd;
|
||||
if (dateRange?.start) {
|
||||
periodStart = new Date(dateRange.start);
|
||||
} else if (params[0]) {
|
||||
periodStart = new Date(params[0]);
|
||||
} else {
|
||||
periodStart = new Date();
|
||||
periodStart.setDate(periodStart.getDate() - 30);
|
||||
}
|
||||
|
||||
if (dateRange?.end) {
|
||||
periodEnd = new Date(dateRange.end);
|
||||
} else if (params[1]) {
|
||||
periodEnd = new Date(params[1]);
|
||||
} else {
|
||||
periodEnd = new Date();
|
||||
}
|
||||
|
||||
const fte = calculateFTE(totalHours, periodStart, periodEnd);
|
||||
const activeEmployees = enrichedEmployeeHours.filter(e => e.hours > 0).length;
|
||||
|
||||
// Calculate weeks in period for weekly averages
|
||||
const periodDays = Math.max(1, (periodEnd - periodStart) / (1000 * 60 * 60 * 24));
|
||||
const weeksInPeriod = periodDays / 7;
|
||||
|
||||
// Get daily trend data for hours
|
||||
// Use DATE_FORMAT to get date string in Eastern timezone, avoiding JS timezone conversion issues
|
||||
// Business day starts at 1 AM, so subtract 1 hour before taking the date
|
||||
const trendWhere = whereClause.replace(/date_placed/g, 'tc.TimeStamp');
|
||||
const trendQuery = `
|
||||
SELECT
|
||||
DATE_FORMAT(DATE_SUB(tc.TimeStamp, INTERVAL 1 HOUR), '%Y-%m-%d') as date,
|
||||
tc.EmployeeID,
|
||||
tc.TimeStamp,
|
||||
tc.PunchType
|
||||
FROM timeclock tc
|
||||
LEFT JOIN employees e ON tc.EmployeeID = e.employeeid
|
||||
WHERE ${trendWhere}
|
||||
AND e.hidden = 0
|
||||
AND e.disabled = 0
|
||||
ORDER BY date, tc.EmployeeID, tc.TimeStamp
|
||||
`;
|
||||
|
||||
const [trendRows] = await connection.execute(trendQuery, params);
|
||||
|
||||
// Get daily picking data for trend
|
||||
// Ship-together orders: only count main orders (is_sub = 0 or NULL)
|
||||
// Use DATE_FORMAT for consistent date string format
|
||||
const pickingTrendWhere = whereClause.replace(/date_placed/g, 'pt.createddate');
|
||||
const pickingTrendQuery = `
|
||||
SELECT
|
||||
DATE_FORMAT(DATE_SUB(pt.createddate, INTERVAL 1 HOUR), '%Y-%m-%d') as date,
|
||||
COUNT(DISTINCT CASE WHEN ptb.is_sub = 0 OR ptb.is_sub IS NULL THEN ptb.orderid END) as ordersPicked,
|
||||
COALESCE(SUM(pt.totalpieces_picked), 0) as piecesPicked
|
||||
FROM picking_ticket pt
|
||||
LEFT JOIN picking_ticket_buckets ptb ON pt.pickingid = ptb.pickingid
|
||||
WHERE ${pickingTrendWhere}
|
||||
AND pt.closeddate IS NOT NULL
|
||||
GROUP BY DATE_FORMAT(DATE_SUB(pt.createddate, INTERVAL 1 HOUR), '%Y-%m-%d')
|
||||
ORDER BY date
|
||||
`;
|
||||
|
||||
const [pickingTrendRows] = await connection.execute(pickingTrendQuery, params);
|
||||
|
||||
// Create a map of picking data by date
|
||||
const pickingByDate = new Map();
|
||||
pickingTrendRows.forEach(row => {
|
||||
// Date is already a string in YYYY-MM-DD format from DATE_FORMAT
|
||||
const date = String(row.date);
|
||||
pickingByDate.set(date, {
|
||||
ordersPicked: parseInt(row.ordersPicked || 0),
|
||||
piecesPicked: parseInt(row.piecesPicked || 0),
|
||||
});
|
||||
});
|
||||
|
||||
// Group timeclock by date for trend
|
||||
const byDate = new Map();
|
||||
trendRows.forEach(row => {
|
||||
// Date is already a string in YYYY-MM-DD format from DATE_FORMAT
|
||||
const date = String(row.date);
|
||||
if (!byDate.has(date)) {
|
||||
byDate.set(date, []);
|
||||
}
|
||||
byDate.get(date).push(row);
|
||||
});
|
||||
|
||||
// Generate all dates in the period range for complete trend data
|
||||
const allDatesInRange = [];
|
||||
const startDt = DateTime.fromJSDate(periodStart).setZone(TIMEZONE).startOf('day');
|
||||
const endDt = DateTime.fromJSDate(periodEnd).setZone(TIMEZONE).startOf('day');
|
||||
|
||||
let currentDt = startDt;
|
||||
while (currentDt <= endDt) {
|
||||
allDatesInRange.push(currentDt.toFormat('yyyy-MM-dd'));
|
||||
currentDt = currentDt.plus({ days: 1 });
|
||||
}
|
||||
|
||||
// Build trend data for all dates in range, filling zeros for missing days
|
||||
const trend = allDatesInRange.map(date => {
|
||||
const punches = byDate.get(date) || [];
|
||||
const { totalHours: dayHours, employeeHours: dayEmployeeHours } = calculateHoursFromPunches(punches);
|
||||
const picking = pickingByDate.get(date) || { ordersPicked: 0, piecesPicked: 0 };
|
||||
|
||||
// Parse date string in Eastern timezone to get proper ISO timestamp
|
||||
const dateDt = DateTime.fromFormat(date, 'yyyy-MM-dd', { zone: TIMEZONE });
|
||||
|
||||
return {
|
||||
date,
|
||||
timestamp: dateDt.toISO(),
|
||||
hours: dayHours,
|
||||
activeEmployees: dayEmployeeHours.filter(e => e.hours > 0).length,
|
||||
ordersPicked: picking.ordersPicked,
|
||||
piecesPicked: picking.piecesPicked,
|
||||
};
|
||||
});
|
||||
|
||||
// Get previous period data for comparison
|
||||
const previousRange = getPreviousPeriodRange(timeRange, startDate, endDate);
|
||||
let comparison = null;
|
||||
let previousTotals = null;
|
||||
|
||||
if (previousRange) {
|
||||
const prevTimeclockWhere = previousRange.whereClause.replace(/date_placed/g, 'tc.TimeStamp');
|
||||
|
||||
const [prevTimeclockRows] = await connection.execute(
|
||||
`SELECT tc.EmployeeID, tc.TimeStamp, tc.PunchType
|
||||
FROM timeclock tc
|
||||
LEFT JOIN employees e ON tc.EmployeeID = e.employeeid
|
||||
WHERE ${prevTimeclockWhere}
|
||||
AND e.hidden = 0
|
||||
AND e.disabled = 0
|
||||
ORDER BY tc.EmployeeID, tc.TimeStamp`,
|
||||
previousRange.params
|
||||
);
|
||||
|
||||
const {
|
||||
totalHours: prevTotalHours,
|
||||
totalProductiveHours: prevProductiveHours,
|
||||
employeeHours: prevEmployeeHours
|
||||
} = calculateHoursFromPunches(prevTimeclockRows);
|
||||
const prevActiveEmployees = prevEmployeeHours.filter(e => e.hours > 0).length;
|
||||
|
||||
// Previous picking data (ship-together fix applied)
|
||||
// Use separate queries to avoid duplication from bucket join
|
||||
const prevPickingWhere = previousRange.whereClause.replace(/date_placed/g, 'pt.createddate');
|
||||
|
||||
const [[prevPickingStatsRows], [prevOrderCountRows]] = await Promise.all([
|
||||
connection.execute(
|
||||
`SELECT
|
||||
SUM(pt.totalpieces_picked) as piecesPicked,
|
||||
SUM(TIMESTAMPDIFF(SECOND, pt.createddate, pt.closeddate)) as pickingTimeSeconds
|
||||
FROM picking_ticket pt
|
||||
WHERE ${prevPickingWhere}
|
||||
AND pt.closeddate IS NOT NULL`,
|
||||
previousRange.params
|
||||
),
|
||||
connection.execute(
|
||||
`SELECT
|
||||
COUNT(DISTINCT CASE WHEN ptb.is_sub = 0 OR ptb.is_sub IS NULL THEN ptb.orderid END) as ordersPicked
|
||||
FROM picking_ticket pt
|
||||
LEFT JOIN picking_ticket_buckets ptb ON pt.pickingid = ptb.pickingid
|
||||
WHERE ${prevPickingWhere}
|
||||
AND pt.closeddate IS NOT NULL`,
|
||||
previousRange.params
|
||||
)
|
||||
]);
|
||||
|
||||
const prevPickingStats = prevPickingStatsRows[0] || { piecesPicked: 0, pickingTimeSeconds: 0 };
|
||||
const prevOrderCount = prevOrderCountRows[0] || { ordersPicked: 0 };
|
||||
const prevPicking = {
|
||||
ordersPicked: parseInt(prevOrderCount.ordersPicked || 0),
|
||||
piecesPicked: parseInt(prevPickingStats.piecesPicked || 0),
|
||||
pickingTimeSeconds: parseInt(prevPickingStats.pickingTimeSeconds || 0)
|
||||
};
|
||||
const prevPickingHours = prevPicking.pickingTimeSeconds / 3600;
|
||||
|
||||
// Previous shipping data
|
||||
const prevShippingWhere = previousRange.whereClause.replace(/date_placed/g, 'o.date_shipped');
|
||||
const [prevShippingRows] = await connection.execute(
|
||||
`SELECT
|
||||
COUNT(DISTINCT CASE WHEN o.order_type != 8 OR o.order_type IS NULL THEN o.order_id END) as ordersShipped,
|
||||
COALESCE(SUM(o.stats_prod_pieces), 0) as piecesShipped
|
||||
FROM _order o
|
||||
WHERE ${prevShippingWhere}
|
||||
AND o.order_status IN (100, 92)`,
|
||||
previousRange.params
|
||||
);
|
||||
const prevShipping = prevShippingRows[0] || { ordersShipped: 0, piecesShipped: 0 };
|
||||
|
||||
// Calculate previous period FTE and productivity
|
||||
const prevFte = calculateFTE(prevTotalHours, previousRange.start || periodStart, previousRange.end || periodEnd);
|
||||
const prevOrdersPerHour = prevProductiveHours > 0 ? parseInt(prevPicking.ordersPicked || 0) / prevProductiveHours : 0;
|
||||
const prevPiecesPerHour = prevProductiveHours > 0 ? parseInt(prevPicking.piecesPicked || 0) / prevProductiveHours : 0;
|
||||
|
||||
previousTotals = {
|
||||
hours: prevTotalHours,
|
||||
productiveHours: prevProductiveHours,
|
||||
activeEmployees: prevActiveEmployees,
|
||||
fte: prevFte,
|
||||
ordersPicked: parseInt(prevPicking.ordersPicked || 0),
|
||||
piecesPicked: parseInt(prevPicking.piecesPicked || 0),
|
||||
pickingHours: prevPickingHours,
|
||||
ordersShipped: parseInt(prevShipping.ordersShipped || 0),
|
||||
piecesShipped: parseInt(prevShipping.piecesShipped || 0),
|
||||
ordersPerHour: prevOrdersPerHour,
|
||||
piecesPerHour: prevPiecesPerHour,
|
||||
};
|
||||
|
||||
// Calculate productivity metrics for comparison
|
||||
const currentOrdersPerHour = totalProductiveHours > 0 ? totalOrdersPicked / totalProductiveHours : 0;
|
||||
const currentPiecesPerHour = totalProductiveHours > 0 ? totalPiecesPicked / totalProductiveHours : 0;
|
||||
|
||||
comparison = {
|
||||
hours: calculateComparison(totalHours, prevTotalHours),
|
||||
productiveHours: calculateComparison(totalProductiveHours, prevProductiveHours),
|
||||
activeEmployees: calculateComparison(activeEmployees, prevActiveEmployees),
|
||||
fte: calculateComparison(fte, prevFte),
|
||||
ordersPicked: calculateComparison(totalOrdersPicked, parseInt(prevPicking.ordersPicked || 0)),
|
||||
piecesPicked: calculateComparison(totalPiecesPicked, parseInt(prevPicking.piecesPicked || 0)),
|
||||
ordersShipped: calculateComparison(parseInt(shipping.ordersShipped || 0), parseInt(prevShipping.ordersShipped || 0)),
|
||||
piecesShipped: calculateComparison(parseInt(shipping.piecesShipped || 0), parseInt(prevShipping.piecesShipped || 0)),
|
||||
ordersPerHour: calculateComparison(currentOrdersPerHour, prevOrdersPerHour),
|
||||
piecesPerHour: calculateComparison(currentPiecesPerHour, prevPiecesPerHour),
|
||||
};
|
||||
}
|
||||
|
||||
// Calculate efficiency (picking time vs productive hours)
|
||||
const pickingEfficiency = totalProductiveHours > 0 ? (totalPickingHours / totalProductiveHours) * 100 : 0;
|
||||
|
||||
const response = {
|
||||
dateRange,
|
||||
totals: {
|
||||
// Time metrics
|
||||
hours: totalHours,
|
||||
breakHours: totalBreakHours,
|
||||
productiveHours: totalProductiveHours,
|
||||
pickingHours: totalPickingHours,
|
||||
|
||||
// Employee metrics
|
||||
activeEmployees,
|
||||
fte,
|
||||
weeksInPeriod,
|
||||
|
||||
// Picking metrics
|
||||
ordersPicked: totalOrdersPicked,
|
||||
piecesPicked: totalPiecesPicked,
|
||||
ticketCount: totalTickets,
|
||||
|
||||
// Shipping metrics
|
||||
ordersShipped: parseInt(shipping.ordersShipped || 0),
|
||||
piecesShipped: parseInt(shipping.piecesShipped || 0),
|
||||
|
||||
// Calculated metrics - standardized to weekly
|
||||
hoursPerWeek: weeksInPeriod > 0 ? totalHours / weeksInPeriod : 0,
|
||||
hoursPerEmployeePerWeek: activeEmployees > 0 && weeksInPeriod > 0
|
||||
? (totalHours / activeEmployees) / weeksInPeriod
|
||||
: 0,
|
||||
|
||||
// Productivity metrics (uses productive hours - excludes breaks)
|
||||
ordersPerHour: totalProductiveHours > 0 ? totalOrdersPicked / totalProductiveHours : 0,
|
||||
piecesPerHour: totalProductiveHours > 0 ? totalPiecesPicked / totalProductiveHours : 0,
|
||||
|
||||
// Picking speed from database (more accurate, only counts picking time)
|
||||
avgPickingSpeed,
|
||||
|
||||
// Efficiency metrics
|
||||
pickingEfficiency,
|
||||
},
|
||||
previousTotals,
|
||||
comparison,
|
||||
byEmployee: {
|
||||
hours: enrichedEmployeeHours,
|
||||
picking: pickingByEmployee,
|
||||
shipping: shippingByEmployee,
|
||||
},
|
||||
trend,
|
||||
};
|
||||
|
||||
return { response, release };
|
||||
};
|
||||
|
||||
let result;
|
||||
try {
|
||||
result = await Promise.race([mainOperation(), timeoutPromise]);
|
||||
} catch (error) {
|
||||
if (error.message.includes('timeout')) {
|
||||
console.log(`[EMPLOYEE-METRICS] Request timed out in ${Date.now() - startTime}ms`);
|
||||
throw error;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
const { response, release } = result;
|
||||
|
||||
if (release) release();
|
||||
|
||||
console.log(`[EMPLOYEE-METRICS] Request completed in ${Date.now() - startTime}ms`);
|
||||
res.json(response);
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error in /employee-metrics:', error);
|
||||
console.log(`[EMPLOYEE-METRICS] Request failed in ${Date.now() - startTime}ms`);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Health check
|
||||
router.get('/health', async (req, res) => {
|
||||
try {
|
||||
const { connection, release } = await getDbConnection();
|
||||
await connection.execute('SELECT 1 as test');
|
||||
release();
|
||||
|
||||
res.json({
|
||||
status: 'healthy',
|
||||
timestamp: new Date().toISOString(),
|
||||
pool: getPoolStatus(),
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(500).json({
|
||||
status: 'unhealthy',
|
||||
timestamp: new Date().toISOString(),
|
||||
error: error.message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Helper functions
|
||||
function calculateComparison(currentValue, previousValue) {
|
||||
if (typeof previousValue !== 'number') {
|
||||
return { absolute: null, percentage: null };
|
||||
}
|
||||
|
||||
const absolute = typeof currentValue === 'number' ? currentValue - previousValue : null;
|
||||
const percentage =
|
||||
absolute !== null && previousValue !== 0
|
||||
? (absolute / Math.abs(previousValue)) * 100
|
||||
: null;
|
||||
|
||||
return { absolute, percentage };
|
||||
}
|
||||
|
||||
function getPreviousPeriodRange(timeRange, startDate, endDate) {
|
||||
if (timeRange && timeRange !== 'custom') {
|
||||
const prevTimeRange = getPreviousTimeRange(timeRange);
|
||||
if (!prevTimeRange || prevTimeRange === timeRange) {
|
||||
return null;
|
||||
}
|
||||
return getTimeRangeConditions(prevTimeRange);
|
||||
}
|
||||
|
||||
const hasCustomDates = (timeRange === 'custom' || !timeRange) && startDate && endDate;
|
||||
if (!hasCustomDates) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const start = new Date(startDate);
|
||||
const end = new Date(endDate);
|
||||
|
||||
if (Number.isNaN(start.getTime()) || Number.isNaN(end.getTime())) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const duration = end.getTime() - start.getTime();
|
||||
if (!Number.isFinite(duration) || duration <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const prevEnd = new Date(start.getTime() - 1);
|
||||
const prevStart = new Date(prevEnd.getTime() - duration);
|
||||
|
||||
return getTimeRangeConditions('custom', prevStart.toISOString(), prevEnd.toISOString());
|
||||
}
|
||||
|
||||
function getPreviousTimeRange(timeRange) {
|
||||
const map = {
|
||||
today: 'yesterday',
|
||||
thisWeek: 'lastWeek',
|
||||
thisMonth: 'lastMonth',
|
||||
last7days: 'previous7days',
|
||||
last30days: 'previous30days',
|
||||
last90days: 'previous90days',
|
||||
yesterday: 'twoDaysAgo'
|
||||
};
|
||||
return map[timeRange] || timeRange;
|
||||
}
|
||||
|
||||
module.exports = router;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,484 @@
|
||||
const express = require('express');
|
||||
const { DateTime } = require('luxon');
|
||||
|
||||
const router = express.Router();
|
||||
const { getDbConnection, getPoolStatus } = require('../db/connection');
|
||||
const {
|
||||
getTimeRangeConditions,
|
||||
} = require('../utils/timeUtils');
|
||||
|
||||
const TIMEZONE = 'America/New_York';
|
||||
|
||||
// Main operations metrics endpoint - focused on picking and shipping
|
||||
router.get('/', async (req, res) => {
|
||||
const startTime = Date.now();
|
||||
console.log(`[OPERATIONS-METRICS] Starting request for timeRange: ${req.query.timeRange}`);
|
||||
|
||||
const timeoutPromise = new Promise((_, reject) => {
|
||||
setTimeout(() => reject(new Error('Request timeout after 30 seconds')), 30000);
|
||||
});
|
||||
|
||||
try {
|
||||
const mainOperation = async () => {
|
||||
const { timeRange, startDate, endDate } = req.query;
|
||||
console.log(`[OPERATIONS-METRICS] Getting DB connection...`);
|
||||
const { connection, release } = await getDbConnection();
|
||||
console.log(`[OPERATIONS-METRICS] DB connection obtained in ${Date.now() - startTime}ms`);
|
||||
|
||||
const { whereClause, params, dateRange } = getTimeRangeConditions(timeRange, startDate, endDate);
|
||||
|
||||
// Query for picking tickets - using subquery to avoid duplication from bucket join
|
||||
// Ship-together orders: only count main orders (is_sub = 0 or NULL), not sub-orders
|
||||
const pickingWhere = whereClause.replace(/date_placed/g, 'pt.createddate');
|
||||
|
||||
// First get picking ticket stats without the bucket join (to avoid duplication)
|
||||
const pickingStatsQuery = `
|
||||
SELECT
|
||||
pt.createdby as employeeId,
|
||||
e.firstname,
|
||||
e.lastname,
|
||||
COUNT(DISTINCT pt.pickingid) as ticketCount,
|
||||
SUM(pt.totalpieces_picked) as piecesPicked,
|
||||
SUM(TIMESTAMPDIFF(SECOND, pt.createddate, pt.closeddate)) as pickingTimeSeconds,
|
||||
AVG(NULLIF(pt.picking_speed, 0)) as avgPickingSpeed
|
||||
FROM picking_ticket pt
|
||||
LEFT JOIN employees e ON pt.createdby = e.employeeid
|
||||
WHERE ${pickingWhere}
|
||||
AND pt.closeddate IS NOT NULL
|
||||
GROUP BY pt.createdby, e.firstname, e.lastname
|
||||
`;
|
||||
|
||||
// Separate query for order counts (needs bucket join for ship-together handling)
|
||||
const orderCountQuery = `
|
||||
SELECT
|
||||
pt.createdby as employeeId,
|
||||
COUNT(DISTINCT CASE WHEN ptb.is_sub = 0 OR ptb.is_sub IS NULL THEN ptb.orderid END) as ordersPicked
|
||||
FROM picking_ticket pt
|
||||
LEFT JOIN picking_ticket_buckets ptb ON pt.pickingid = ptb.pickingid
|
||||
WHERE ${pickingWhere}
|
||||
AND pt.closeddate IS NOT NULL
|
||||
GROUP BY pt.createdby
|
||||
`;
|
||||
|
||||
const [[pickingStatsRows], [orderCountRows]] = await Promise.all([
|
||||
connection.execute(pickingStatsQuery, params),
|
||||
connection.execute(orderCountQuery, params)
|
||||
]);
|
||||
|
||||
// Merge the results
|
||||
const orderCountMap = new Map();
|
||||
orderCountRows.forEach(row => {
|
||||
orderCountMap.set(row.employeeId, parseInt(row.ordersPicked || 0));
|
||||
});
|
||||
|
||||
// Aggregate picking totals
|
||||
let totalOrdersPicked = 0;
|
||||
let totalPiecesPicked = 0;
|
||||
let totalTickets = 0;
|
||||
let totalPickingTimeSeconds = 0;
|
||||
let pickingSpeedSum = 0;
|
||||
let pickingSpeedCount = 0;
|
||||
|
||||
const pickingByEmployee = pickingStatsRows.map(row => {
|
||||
const ordersPicked = orderCountMap.get(row.employeeId) || 0;
|
||||
totalOrdersPicked += ordersPicked;
|
||||
totalPiecesPicked += parseInt(row.piecesPicked || 0);
|
||||
totalTickets += parseInt(row.ticketCount || 0);
|
||||
totalPickingTimeSeconds += parseInt(row.pickingTimeSeconds || 0);
|
||||
if (row.avgPickingSpeed && row.avgPickingSpeed > 0) {
|
||||
pickingSpeedSum += parseFloat(row.avgPickingSpeed);
|
||||
pickingSpeedCount++;
|
||||
}
|
||||
|
||||
const empPickingHours = parseInt(row.pickingTimeSeconds || 0) / 3600;
|
||||
|
||||
return {
|
||||
employeeId: row.employeeId,
|
||||
name: `${row.firstname || ''} ${row.lastname || ''}`.trim() || `Employee ${row.employeeId}`,
|
||||
ticketCount: parseInt(row.ticketCount || 0),
|
||||
ordersPicked,
|
||||
piecesPicked: parseInt(row.piecesPicked || 0),
|
||||
pickingHours: empPickingHours,
|
||||
avgPickingSpeed: row.avgPickingSpeed ? parseFloat(row.avgPickingSpeed) : null,
|
||||
};
|
||||
});
|
||||
|
||||
const totalPickingHours = totalPickingTimeSeconds / 3600;
|
||||
const avgPickingSpeed = pickingSpeedCount > 0 ? pickingSpeedSum / pickingSpeedCount : 0;
|
||||
|
||||
// Query for shipped orders - totals
|
||||
// Ship-together orders: only count main orders (order_type != 8 for sub-orders)
|
||||
const shippingWhere = whereClause.replace(/date_placed/g, 'o.date_shipped');
|
||||
|
||||
const shippingQuery = `
|
||||
SELECT
|
||||
COUNT(DISTINCT CASE WHEN o.order_type != 8 OR o.order_type IS NULL THEN o.order_id END) as ordersShipped,
|
||||
COALESCE(SUM(o.stats_prod_pieces), 0) as piecesShipped
|
||||
FROM _order o
|
||||
WHERE ${shippingWhere}
|
||||
AND o.order_status IN (100, 92)
|
||||
`;
|
||||
|
||||
const [shippingRows] = await connection.execute(shippingQuery, params);
|
||||
const shipping = shippingRows[0] || { ordersShipped: 0, piecesShipped: 0 };
|
||||
|
||||
// Query for shipped orders by employee
|
||||
const shippingByEmployeeQuery = `
|
||||
SELECT
|
||||
e.employeeid,
|
||||
e.firstname,
|
||||
e.lastname,
|
||||
COUNT(DISTINCT CASE WHEN o.order_type != 8 OR o.order_type IS NULL THEN o.order_id END) as ordersShipped,
|
||||
COALESCE(SUM(o.stats_prod_pieces), 0) as piecesShipped
|
||||
FROM _order o
|
||||
JOIN employees e ON o.stats_cid_shipped = e.cid
|
||||
WHERE ${shippingWhere}
|
||||
AND o.order_status IN (100, 92)
|
||||
AND e.hidden = 0
|
||||
AND e.disabled = 0
|
||||
GROUP BY e.employeeid, e.firstname, e.lastname
|
||||
ORDER BY ordersShipped DESC
|
||||
`;
|
||||
|
||||
const [shippingByEmployeeRows] = await connection.execute(shippingByEmployeeQuery, params);
|
||||
const shippingByEmployee = shippingByEmployeeRows.map(row => ({
|
||||
employeeId: row.employeeid,
|
||||
name: `${row.firstname || ''} ${row.lastname || ''}`.trim() || `Employee ${row.employeeid}`,
|
||||
ordersShipped: parseInt(row.ordersShipped || 0),
|
||||
piecesShipped: parseInt(row.piecesShipped || 0),
|
||||
}));
|
||||
|
||||
// Calculate period dates
|
||||
let periodStart, periodEnd;
|
||||
if (dateRange?.start) {
|
||||
periodStart = new Date(dateRange.start);
|
||||
} else if (params[0]) {
|
||||
periodStart = new Date(params[0]);
|
||||
} else {
|
||||
periodStart = new Date();
|
||||
periodStart.setDate(periodStart.getDate() - 30);
|
||||
}
|
||||
|
||||
if (dateRange?.end) {
|
||||
periodEnd = new Date(dateRange.end);
|
||||
} else if (params[1]) {
|
||||
periodEnd = new Date(params[1]);
|
||||
} else {
|
||||
periodEnd = new Date();
|
||||
}
|
||||
|
||||
// Calculate productivity (orders/pieces per picking hour)
|
||||
const ordersPerHour = totalPickingHours > 0 ? totalOrdersPicked / totalPickingHours : 0;
|
||||
const piecesPerHour = totalPickingHours > 0 ? totalPiecesPicked / totalPickingHours : 0;
|
||||
|
||||
// Get daily trend data for picking
|
||||
// Use DATE_FORMAT to get date string in Eastern timezone
|
||||
// Business day starts at 1 AM, so subtract 1 hour before taking the date
|
||||
const pickingTrendWhere = whereClause.replace(/date_placed/g, 'pt.createddate');
|
||||
const pickingTrendQuery = `
|
||||
SELECT
|
||||
pt_agg.date,
|
||||
COALESCE(order_counts.ordersPicked, 0) as ordersPicked,
|
||||
pt_agg.piecesPicked
|
||||
FROM (
|
||||
SELECT
|
||||
DATE_FORMAT(DATE_SUB(pt.createddate, INTERVAL 1 HOUR), '%Y-%m-%d') as date,
|
||||
COALESCE(SUM(pt.totalpieces_picked), 0) as piecesPicked
|
||||
FROM picking_ticket pt
|
||||
WHERE ${pickingTrendWhere}
|
||||
AND pt.closeddate IS NOT NULL
|
||||
GROUP BY DATE_FORMAT(DATE_SUB(pt.createddate, INTERVAL 1 HOUR), '%Y-%m-%d')
|
||||
) pt_agg
|
||||
LEFT JOIN (
|
||||
SELECT
|
||||
DATE_FORMAT(DATE_SUB(pt.createddate, INTERVAL 1 HOUR), '%Y-%m-%d') as date,
|
||||
COUNT(DISTINCT CASE WHEN ptb.is_sub = 0 OR ptb.is_sub IS NULL THEN ptb.orderid END) as ordersPicked
|
||||
FROM picking_ticket pt
|
||||
LEFT JOIN picking_ticket_buckets ptb ON pt.pickingid = ptb.pickingid
|
||||
WHERE ${pickingTrendWhere}
|
||||
AND pt.closeddate IS NOT NULL
|
||||
GROUP BY DATE_FORMAT(DATE_SUB(pt.createddate, INTERVAL 1 HOUR), '%Y-%m-%d')
|
||||
) order_counts ON pt_agg.date = order_counts.date
|
||||
ORDER BY pt_agg.date
|
||||
`;
|
||||
|
||||
// Get shipping trend data
|
||||
const shippingTrendWhere = whereClause.replace(/date_placed/g, 'o.date_shipped');
|
||||
const shippingTrendQuery = `
|
||||
SELECT
|
||||
DATE_FORMAT(DATE_SUB(o.date_shipped, INTERVAL 1 HOUR), '%Y-%m-%d') as date,
|
||||
COUNT(DISTINCT CASE WHEN o.order_type != 8 OR o.order_type IS NULL THEN o.order_id END) as ordersShipped,
|
||||
COALESCE(SUM(o.stats_prod_pieces), 0) as piecesShipped
|
||||
FROM _order o
|
||||
WHERE ${shippingTrendWhere}
|
||||
AND o.order_status IN (100, 92)
|
||||
GROUP BY DATE_FORMAT(DATE_SUB(o.date_shipped, INTERVAL 1 HOUR), '%Y-%m-%d')
|
||||
ORDER BY date
|
||||
`;
|
||||
|
||||
const [[pickingTrendRows], [shippingTrendRows]] = await Promise.all([
|
||||
connection.execute(pickingTrendQuery, [...params, ...params]),
|
||||
connection.execute(shippingTrendQuery, params),
|
||||
]);
|
||||
|
||||
// Create maps for trend data
|
||||
const pickingByDate = new Map();
|
||||
pickingTrendRows.forEach(row => {
|
||||
const date = String(row.date);
|
||||
pickingByDate.set(date, {
|
||||
ordersPicked: parseInt(row.ordersPicked || 0),
|
||||
piecesPicked: parseInt(row.piecesPicked || 0),
|
||||
});
|
||||
});
|
||||
|
||||
const shippingByDate = new Map();
|
||||
shippingTrendRows.forEach(row => {
|
||||
const date = String(row.date);
|
||||
shippingByDate.set(date, {
|
||||
ordersShipped: parseInt(row.ordersShipped || 0),
|
||||
piecesShipped: parseInt(row.piecesShipped || 0),
|
||||
});
|
||||
});
|
||||
|
||||
// Generate all dates in the period range for complete trend data
|
||||
const allDatesInRange = [];
|
||||
const startDt = DateTime.fromJSDate(periodStart).setZone(TIMEZONE).startOf('day');
|
||||
const endDt = DateTime.fromJSDate(periodEnd).setZone(TIMEZONE).startOf('day');
|
||||
|
||||
let currentDt = startDt;
|
||||
while (currentDt <= endDt) {
|
||||
allDatesInRange.push(currentDt.toFormat('yyyy-MM-dd'));
|
||||
currentDt = currentDt.plus({ days: 1 });
|
||||
}
|
||||
|
||||
// Build trend data for all dates in range
|
||||
const trend = allDatesInRange.map(date => {
|
||||
const picking = pickingByDate.get(date) || { ordersPicked: 0, piecesPicked: 0 };
|
||||
const shippingData = shippingByDate.get(date) || { ordersShipped: 0, piecesShipped: 0 };
|
||||
|
||||
// Parse date string in Eastern timezone to get proper ISO timestamp
|
||||
const dateDt = DateTime.fromFormat(date, 'yyyy-MM-dd', { zone: TIMEZONE });
|
||||
|
||||
return {
|
||||
date,
|
||||
timestamp: dateDt.toISO(),
|
||||
ordersPicked: picking.ordersPicked,
|
||||
piecesPicked: picking.piecesPicked,
|
||||
ordersShipped: shippingData.ordersShipped,
|
||||
piecesShipped: shippingData.piecesShipped,
|
||||
};
|
||||
});
|
||||
|
||||
// Get previous period data for comparison
|
||||
const previousRange = getPreviousPeriodRange(timeRange, startDate, endDate);
|
||||
let comparison = null;
|
||||
let previousTotals = null;
|
||||
|
||||
if (previousRange) {
|
||||
// Previous picking data
|
||||
const prevPickingWhere = previousRange.whereClause.replace(/date_placed/g, 'pt.createddate');
|
||||
|
||||
const [[prevPickingStatsRows], [prevOrderCountRows]] = await Promise.all([
|
||||
connection.execute(
|
||||
`SELECT
|
||||
SUM(pt.totalpieces_picked) as piecesPicked,
|
||||
SUM(TIMESTAMPDIFF(SECOND, pt.createddate, pt.closeddate)) as pickingTimeSeconds
|
||||
FROM picking_ticket pt
|
||||
WHERE ${prevPickingWhere}
|
||||
AND pt.closeddate IS NOT NULL`,
|
||||
previousRange.params
|
||||
),
|
||||
connection.execute(
|
||||
`SELECT
|
||||
COUNT(DISTINCT CASE WHEN ptb.is_sub = 0 OR ptb.is_sub IS NULL THEN ptb.orderid END) as ordersPicked
|
||||
FROM picking_ticket pt
|
||||
LEFT JOIN picking_ticket_buckets ptb ON pt.pickingid = ptb.pickingid
|
||||
WHERE ${prevPickingWhere}
|
||||
AND pt.closeddate IS NOT NULL`,
|
||||
previousRange.params
|
||||
)
|
||||
]);
|
||||
|
||||
const prevPickingStats = prevPickingStatsRows[0] || { piecesPicked: 0, pickingTimeSeconds: 0 };
|
||||
const prevOrderCount = prevOrderCountRows[0] || { ordersPicked: 0 };
|
||||
const prevPicking = {
|
||||
ordersPicked: parseInt(prevOrderCount.ordersPicked || 0),
|
||||
piecesPicked: parseInt(prevPickingStats.piecesPicked || 0),
|
||||
pickingTimeSeconds: parseInt(prevPickingStats.pickingTimeSeconds || 0)
|
||||
};
|
||||
const prevPickingHours = prevPicking.pickingTimeSeconds / 3600;
|
||||
|
||||
// Previous shipping data
|
||||
const prevShippingWhere = previousRange.whereClause.replace(/date_placed/g, 'o.date_shipped');
|
||||
const [prevShippingRows] = await connection.execute(
|
||||
`SELECT
|
||||
COUNT(DISTINCT CASE WHEN o.order_type != 8 OR o.order_type IS NULL THEN o.order_id END) as ordersShipped,
|
||||
COALESCE(SUM(o.stats_prod_pieces), 0) as piecesShipped
|
||||
FROM _order o
|
||||
WHERE ${prevShippingWhere}
|
||||
AND o.order_status IN (100, 92)`,
|
||||
previousRange.params
|
||||
);
|
||||
const prevShipping = prevShippingRows[0] || { ordersShipped: 0, piecesShipped: 0 };
|
||||
|
||||
// Calculate previous productivity
|
||||
const prevOrdersPerHour = prevPickingHours > 0 ? parseInt(prevPicking.ordersPicked || 0) / prevPickingHours : 0;
|
||||
const prevPiecesPerHour = prevPickingHours > 0 ? parseInt(prevPicking.piecesPicked || 0) / prevPickingHours : 0;
|
||||
|
||||
previousTotals = {
|
||||
ordersPicked: parseInt(prevPicking.ordersPicked || 0),
|
||||
piecesPicked: parseInt(prevPicking.piecesPicked || 0),
|
||||
pickingHours: prevPickingHours,
|
||||
ordersShipped: parseInt(prevShipping.ordersShipped || 0),
|
||||
piecesShipped: parseInt(prevShipping.piecesShipped || 0),
|
||||
ordersPerHour: prevOrdersPerHour,
|
||||
piecesPerHour: prevPiecesPerHour,
|
||||
};
|
||||
|
||||
comparison = {
|
||||
ordersPicked: calculateComparison(totalOrdersPicked, parseInt(prevPicking.ordersPicked || 0)),
|
||||
piecesPicked: calculateComparison(totalPiecesPicked, parseInt(prevPicking.piecesPicked || 0)),
|
||||
ordersShipped: calculateComparison(parseInt(shipping.ordersShipped || 0), parseInt(prevShipping.ordersShipped || 0)),
|
||||
piecesShipped: calculateComparison(parseInt(shipping.piecesShipped || 0), parseInt(prevShipping.piecesShipped || 0)),
|
||||
ordersPerHour: calculateComparison(ordersPerHour, prevOrdersPerHour),
|
||||
piecesPerHour: calculateComparison(piecesPerHour, prevPiecesPerHour),
|
||||
};
|
||||
}
|
||||
|
||||
const response = {
|
||||
dateRange,
|
||||
totals: {
|
||||
// Picking metrics
|
||||
ordersPicked: totalOrdersPicked,
|
||||
piecesPicked: totalPiecesPicked,
|
||||
ticketCount: totalTickets,
|
||||
pickingHours: totalPickingHours,
|
||||
|
||||
// Shipping metrics
|
||||
ordersShipped: parseInt(shipping.ordersShipped || 0),
|
||||
piecesShipped: parseInt(shipping.piecesShipped || 0),
|
||||
|
||||
// Productivity metrics
|
||||
ordersPerHour,
|
||||
piecesPerHour,
|
||||
avgPickingSpeed,
|
||||
},
|
||||
previousTotals,
|
||||
comparison,
|
||||
byEmployee: {
|
||||
picking: pickingByEmployee,
|
||||
shipping: shippingByEmployee,
|
||||
},
|
||||
trend,
|
||||
};
|
||||
|
||||
return { response, release };
|
||||
};
|
||||
|
||||
let result;
|
||||
try {
|
||||
result = await Promise.race([mainOperation(), timeoutPromise]);
|
||||
} catch (error) {
|
||||
if (error.message.includes('timeout')) {
|
||||
console.log(`[OPERATIONS-METRICS] Request timed out in ${Date.now() - startTime}ms`);
|
||||
throw error;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
const { response, release } = result;
|
||||
|
||||
if (release) release();
|
||||
|
||||
console.log(`[OPERATIONS-METRICS] Request completed in ${Date.now() - startTime}ms`);
|
||||
res.json(response);
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error in /operations-metrics:', error);
|
||||
console.log(`[OPERATIONS-METRICS] Request failed in ${Date.now() - startTime}ms`);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Health check
|
||||
router.get('/health', async (req, res) => {
|
||||
try {
|
||||
const { connection, release } = await getDbConnection();
|
||||
await connection.execute('SELECT 1 as test');
|
||||
release();
|
||||
|
||||
res.json({
|
||||
status: 'healthy',
|
||||
timestamp: new Date().toISOString(),
|
||||
pool: getPoolStatus(),
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(500).json({
|
||||
status: 'unhealthy',
|
||||
timestamp: new Date().toISOString(),
|
||||
error: error.message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Helper functions
|
||||
function calculateComparison(currentValue, previousValue) {
|
||||
if (typeof previousValue !== 'number') {
|
||||
return { absolute: null, percentage: null };
|
||||
}
|
||||
|
||||
const absolute = typeof currentValue === 'number' ? currentValue - previousValue : null;
|
||||
const percentage =
|
||||
absolute !== null && previousValue !== 0
|
||||
? (absolute / Math.abs(previousValue)) * 100
|
||||
: null;
|
||||
|
||||
return { absolute, percentage };
|
||||
}
|
||||
|
||||
function getPreviousPeriodRange(timeRange, startDate, endDate) {
|
||||
if (timeRange && timeRange !== 'custom') {
|
||||
const prevTimeRange = getPreviousTimeRange(timeRange);
|
||||
if (!prevTimeRange || prevTimeRange === timeRange) {
|
||||
return null;
|
||||
}
|
||||
return getTimeRangeConditions(prevTimeRange);
|
||||
}
|
||||
|
||||
const hasCustomDates = (timeRange === 'custom' || !timeRange) && startDate && endDate;
|
||||
if (!hasCustomDates) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const start = new Date(startDate);
|
||||
const end = new Date(endDate);
|
||||
|
||||
if (Number.isNaN(start.getTime()) || Number.isNaN(end.getTime())) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const duration = end.getTime() - start.getTime();
|
||||
if (!Number.isFinite(duration) || duration <= 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const prevEnd = new Date(start.getTime() - 1);
|
||||
const prevStart = new Date(prevEnd.getTime() - duration);
|
||||
|
||||
return getTimeRangeConditions('custom', prevStart.toISOString(), prevEnd.toISOString());
|
||||
}
|
||||
|
||||
function getPreviousTimeRange(timeRange) {
|
||||
const map = {
|
||||
today: 'yesterday',
|
||||
thisWeek: 'lastWeek',
|
||||
thisMonth: 'lastMonth',
|
||||
last7days: 'previous7days',
|
||||
last30days: 'previous30days',
|
||||
last90days: 'previous90days',
|
||||
yesterday: 'twoDaysAgo'
|
||||
};
|
||||
return map[timeRange] || timeRange;
|
||||
}
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,505 @@
|
||||
const express = require('express');
|
||||
const { DateTime } = require('luxon');
|
||||
|
||||
const router = express.Router();
|
||||
const { getDbConnection, getPoolStatus } = require('../db/connection');
|
||||
|
||||
const TIMEZONE = 'America/New_York';
|
||||
|
||||
// Punch types from the database
|
||||
const PUNCH_TYPES = {
|
||||
OUT: 0,
|
||||
IN: 1,
|
||||
BREAK_START: 2,
|
||||
BREAK_END: 3,
|
||||
};
|
||||
|
||||
// Standard hours for overtime calculation (40 hours per week)
|
||||
const STANDARD_WEEKLY_HOURS = 40;
|
||||
|
||||
// Reference pay period start date (January 25, 2026 is a Sunday, first day of a pay period)
|
||||
const PAY_PERIOD_REFERENCE = DateTime.fromObject(
|
||||
{ year: 2026, month: 1, day: 25 },
|
||||
{ zone: TIMEZONE }
|
||||
);
|
||||
|
||||
/**
|
||||
* Calculate the pay period that contains a given date
|
||||
* Pay periods are 14 days starting on Sunday
|
||||
* @param {DateTime} date - The date to find the pay period for
|
||||
* @returns {{ start: DateTime, end: DateTime, week1: { start: DateTime, end: DateTime }, week2: { start: DateTime, end: DateTime } }}
|
||||
*/
|
||||
function getPayPeriodForDate(date) {
|
||||
const dt = DateTime.isDateTime(date) ? date : DateTime.fromJSDate(date, { zone: TIMEZONE });
|
||||
|
||||
// Calculate days since reference
|
||||
const daysSinceReference = Math.floor(dt.diff(PAY_PERIOD_REFERENCE, 'days').days);
|
||||
|
||||
// Find which pay period this falls into (can be negative for dates before reference)
|
||||
const payPeriodIndex = Math.floor(daysSinceReference / 14);
|
||||
|
||||
// Calculate the start of this pay period
|
||||
const start = PAY_PERIOD_REFERENCE.plus({ days: payPeriodIndex * 14 }).startOf('day');
|
||||
const end = start.plus({ days: 13 }).endOf('day');
|
||||
|
||||
// Week 1: Sunday through Saturday
|
||||
const week1Start = start;
|
||||
const week1End = start.plus({ days: 6 }).endOf('day');
|
||||
|
||||
// Week 2: Sunday through Saturday
|
||||
const week2Start = start.plus({ days: 7 }).startOf('day');
|
||||
const week2End = end;
|
||||
|
||||
return {
|
||||
start,
|
||||
end,
|
||||
week1: { start: week1Start, end: week1End },
|
||||
week2: { start: week2Start, end: week2End },
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current pay period
|
||||
*/
|
||||
function getCurrentPayPeriod() {
|
||||
return getPayPeriodForDate(DateTime.now().setZone(TIMEZONE));
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to previous or next pay period
|
||||
* @param {DateTime} currentStart - Current pay period start
|
||||
* @param {number} offset - Number of pay periods to move (negative for previous)
|
||||
*/
|
||||
function navigatePayPeriod(currentStart, offset) {
|
||||
const newStart = currentStart.plus({ days: offset * 14 });
|
||||
return getPayPeriodForDate(newStart);
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate working hours from timeclock entries, broken down by week
|
||||
* @param {Array} punches - Timeclock punch entries
|
||||
* @param {Object} payPeriod - Pay period with week boundaries
|
||||
*/
|
||||
function calculateHoursByWeek(punches, payPeriod) {
|
||||
// Group by employee
|
||||
const byEmployee = new Map();
|
||||
|
||||
punches.forEach(punch => {
|
||||
if (!byEmployee.has(punch.EmployeeID)) {
|
||||
byEmployee.set(punch.EmployeeID, {
|
||||
employeeId: punch.EmployeeID,
|
||||
firstname: punch.firstname || '',
|
||||
lastname: punch.lastname || '',
|
||||
punches: [],
|
||||
});
|
||||
}
|
||||
byEmployee.get(punch.EmployeeID).punches.push(punch);
|
||||
});
|
||||
|
||||
const employeeResults = [];
|
||||
let totalHours = 0;
|
||||
let totalBreakHours = 0;
|
||||
let totalOvertimeHours = 0;
|
||||
let totalRegularHours = 0;
|
||||
let week1TotalHours = 0;
|
||||
let week1TotalOvertime = 0;
|
||||
let week2TotalHours = 0;
|
||||
let week2TotalOvertime = 0;
|
||||
|
||||
byEmployee.forEach((employeeData) => {
|
||||
// Sort punches by timestamp
|
||||
employeeData.punches.sort((a, b) => new Date(a.TimeStamp) - new Date(b.TimeStamp));
|
||||
|
||||
// Calculate hours for each week
|
||||
const week1Punches = employeeData.punches.filter(p => {
|
||||
const dt = DateTime.fromJSDate(new Date(p.TimeStamp), { zone: TIMEZONE });
|
||||
return dt >= payPeriod.week1.start && dt <= payPeriod.week1.end;
|
||||
});
|
||||
|
||||
const week2Punches = employeeData.punches.filter(p => {
|
||||
const dt = DateTime.fromJSDate(new Date(p.TimeStamp), { zone: TIMEZONE });
|
||||
return dt >= payPeriod.week2.start && dt <= payPeriod.week2.end;
|
||||
});
|
||||
|
||||
const week1Hours = calculateHoursFromPunches(week1Punches);
|
||||
const week2Hours = calculateHoursFromPunches(week2Punches);
|
||||
|
||||
// Calculate overtime per week (anything over 40 hours)
|
||||
const week1Overtime = Math.max(0, week1Hours.hours - STANDARD_WEEKLY_HOURS);
|
||||
const week2Overtime = Math.max(0, week2Hours.hours - STANDARD_WEEKLY_HOURS);
|
||||
const week1Regular = week1Hours.hours - week1Overtime;
|
||||
const week2Regular = week2Hours.hours - week2Overtime;
|
||||
|
||||
const employeeTotal = week1Hours.hours + week2Hours.hours;
|
||||
const employeeBreaks = week1Hours.breakHours + week2Hours.breakHours;
|
||||
const employeeOvertime = week1Overtime + week2Overtime;
|
||||
const employeeRegular = employeeTotal - employeeOvertime;
|
||||
|
||||
totalHours += employeeTotal;
|
||||
totalBreakHours += employeeBreaks;
|
||||
totalOvertimeHours += employeeOvertime;
|
||||
totalRegularHours += employeeRegular;
|
||||
week1TotalHours += week1Hours.hours;
|
||||
week1TotalOvertime += week1Overtime;
|
||||
week2TotalHours += week2Hours.hours;
|
||||
week2TotalOvertime += week2Overtime;
|
||||
|
||||
employeeResults.push({
|
||||
employeeId: employeeData.employeeId,
|
||||
name: `${employeeData.firstname} ${employeeData.lastname}`.trim() || `Employee ${employeeData.employeeId}`,
|
||||
week1Hours: week1Hours.hours,
|
||||
week1BreakHours: week1Hours.breakHours,
|
||||
week1Overtime,
|
||||
week1Regular,
|
||||
week2Hours: week2Hours.hours,
|
||||
week2BreakHours: week2Hours.breakHours,
|
||||
week2Overtime,
|
||||
week2Regular,
|
||||
totalHours: employeeTotal,
|
||||
totalBreakHours: employeeBreaks,
|
||||
overtimeHours: employeeOvertime,
|
||||
regularHours: employeeRegular,
|
||||
});
|
||||
});
|
||||
|
||||
// Sort by total hours descending
|
||||
employeeResults.sort((a, b) => b.totalHours - a.totalHours);
|
||||
|
||||
return {
|
||||
byEmployee: employeeResults,
|
||||
totals: {
|
||||
hours: totalHours,
|
||||
breakHours: totalBreakHours,
|
||||
overtimeHours: totalOvertimeHours,
|
||||
regularHours: totalRegularHours,
|
||||
activeEmployees: employeeResults.filter(e => e.totalHours > 0).length,
|
||||
},
|
||||
byWeek: [
|
||||
{
|
||||
week: 1,
|
||||
start: payPeriod.week1.start.toISODate(),
|
||||
end: payPeriod.week1.end.toISODate(),
|
||||
hours: week1TotalHours,
|
||||
overtime: week1TotalOvertime,
|
||||
regular: week1TotalHours - week1TotalOvertime,
|
||||
},
|
||||
{
|
||||
week: 2,
|
||||
start: payPeriod.week2.start.toISODate(),
|
||||
end: payPeriod.week2.end.toISODate(),
|
||||
hours: week2TotalHours,
|
||||
overtime: week2TotalOvertime,
|
||||
regular: week2TotalHours - week2TotalOvertime,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate hours from a set of punches
|
||||
*/
|
||||
function calculateHoursFromPunches(punches) {
|
||||
let hours = 0;
|
||||
let breakHours = 0;
|
||||
let currentIn = null;
|
||||
let breakStart = null;
|
||||
|
||||
punches.forEach(punch => {
|
||||
const punchTime = new Date(punch.TimeStamp);
|
||||
|
||||
switch (punch.PunchType) {
|
||||
case PUNCH_TYPES.IN:
|
||||
currentIn = punchTime;
|
||||
break;
|
||||
case PUNCH_TYPES.OUT:
|
||||
if (currentIn) {
|
||||
hours += (punchTime - currentIn) / (1000 * 60 * 60);
|
||||
currentIn = null;
|
||||
}
|
||||
break;
|
||||
case PUNCH_TYPES.BREAK_START:
|
||||
breakStart = punchTime;
|
||||
break;
|
||||
case PUNCH_TYPES.BREAK_END:
|
||||
if (breakStart) {
|
||||
breakHours += (punchTime - breakStart) / (1000 * 60 * 60);
|
||||
breakStart = null;
|
||||
}
|
||||
break;
|
||||
}
|
||||
});
|
||||
|
||||
return { hours, breakHours };
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate FTE for a pay period (based on 80 hours = 1 FTE for 2-week period)
|
||||
* @param {number} totalHours - Total hours worked
|
||||
* @param {number} elapsedFraction - Fraction of the period elapsed (0-1). Defaults to 1 for complete periods.
|
||||
*/
|
||||
function calculateFTE(totalHours, elapsedFraction = 1) {
|
||||
const fullTimePeriodHours = STANDARD_WEEKLY_HOURS * 2; // 80 hours for 2 weeks
|
||||
const proratedHours = fullTimePeriodHours * elapsedFraction;
|
||||
return proratedHours > 0 ? totalHours / proratedHours : 0;
|
||||
}
|
||||
|
||||
// Main payroll metrics endpoint
|
||||
router.get('/', async (req, res) => {
|
||||
const startTime = Date.now();
|
||||
console.log(`[PAYROLL-METRICS] Starting request`);
|
||||
|
||||
const timeoutPromise = new Promise((_, reject) => {
|
||||
setTimeout(() => reject(new Error('Request timeout after 30 seconds')), 30000);
|
||||
});
|
||||
|
||||
try {
|
||||
const mainOperation = async () => {
|
||||
const { payPeriodStart, navigate } = req.query;
|
||||
|
||||
let payPeriod;
|
||||
|
||||
if (payPeriodStart) {
|
||||
// Parse the provided start date
|
||||
const startDate = DateTime.fromISO(payPeriodStart, { zone: TIMEZONE });
|
||||
if (!startDate.isValid) {
|
||||
return res.status(400).json({ error: 'Invalid payPeriodStart date format' });
|
||||
}
|
||||
payPeriod = getPayPeriodForDate(startDate);
|
||||
} else {
|
||||
// Default to current pay period
|
||||
payPeriod = getCurrentPayPeriod();
|
||||
}
|
||||
|
||||
// Handle navigation if requested
|
||||
if (navigate) {
|
||||
const offset = parseInt(navigate, 10);
|
||||
if (!isNaN(offset)) {
|
||||
payPeriod = navigatePayPeriod(payPeriod.start, offset);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[PAYROLL-METRICS] Getting DB connection...`);
|
||||
const { connection, release } = await getDbConnection();
|
||||
console.log(`[PAYROLL-METRICS] DB connection obtained in ${Date.now() - startTime}ms`);
|
||||
|
||||
// Build query for the pay period
|
||||
const periodStart = payPeriod.start.toJSDate();
|
||||
const periodEnd = payPeriod.end.toJSDate();
|
||||
|
||||
const timeclockQuery = `
|
||||
SELECT
|
||||
tc.EmployeeID,
|
||||
tc.TimeStamp,
|
||||
tc.PunchType,
|
||||
e.firstname,
|
||||
e.lastname
|
||||
FROM timeclock tc
|
||||
LEFT JOIN employees e ON tc.EmployeeID = e.employeeid
|
||||
WHERE tc.TimeStamp >= ? AND tc.TimeStamp <= ?
|
||||
AND e.hidden = 0
|
||||
AND e.disabled = 0
|
||||
ORDER BY tc.EmployeeID, tc.TimeStamp
|
||||
`;
|
||||
|
||||
const [timeclockRows] = await connection.execute(timeclockQuery, [periodStart, periodEnd]);
|
||||
|
||||
// Calculate hours with week breakdown
|
||||
const hoursData = calculateHoursByWeek(timeclockRows, payPeriod);
|
||||
|
||||
// Calculate FTE — prorate for in-progress periods so the value reflects
|
||||
// the pace employees are on rather than raw hours / 80
|
||||
let elapsedFraction = 1;
|
||||
if (isCurrentPayPeriod(payPeriod)) {
|
||||
const now = DateTime.now().setZone(TIMEZONE);
|
||||
const elapsedDays = Math.max(1, Math.ceil(now.diff(payPeriod.start, 'days').days));
|
||||
elapsedFraction = Math.min(1, elapsedDays / 14);
|
||||
}
|
||||
const fte = calculateFTE(hoursData.totals.hours, elapsedFraction);
|
||||
const activeEmployees = hoursData.totals.activeEmployees;
|
||||
const avgHoursPerEmployee = activeEmployees > 0 ? hoursData.totals.hours / activeEmployees : 0;
|
||||
|
||||
// Get previous pay period data for comparison
|
||||
const prevPayPeriod = navigatePayPeriod(payPeriod.start, -1);
|
||||
const [prevTimeclockRows] = await connection.execute(timeclockQuery, [
|
||||
prevPayPeriod.start.toJSDate(),
|
||||
prevPayPeriod.end.toJSDate(),
|
||||
]);
|
||||
|
||||
const prevHoursData = calculateHoursByWeek(prevTimeclockRows, prevPayPeriod);
|
||||
const prevFte = calculateFTE(prevHoursData.totals.hours);
|
||||
|
||||
// Calculate comparisons
|
||||
const comparison = {
|
||||
hours: calculateComparison(hoursData.totals.hours, prevHoursData.totals.hours),
|
||||
overtimeHours: calculateComparison(hoursData.totals.overtimeHours, prevHoursData.totals.overtimeHours),
|
||||
fte: calculateComparison(fte, prevFte),
|
||||
activeEmployees: calculateComparison(hoursData.totals.activeEmployees, prevHoursData.totals.activeEmployees),
|
||||
};
|
||||
|
||||
const response = {
|
||||
payPeriod: {
|
||||
start: payPeriod.start.toISODate(),
|
||||
end: payPeriod.end.toISODate(),
|
||||
label: formatPayPeriodLabel(payPeriod),
|
||||
week1: {
|
||||
start: payPeriod.week1.start.toISODate(),
|
||||
end: payPeriod.week1.end.toISODate(),
|
||||
label: formatWeekLabel(payPeriod.week1),
|
||||
},
|
||||
week2: {
|
||||
start: payPeriod.week2.start.toISODate(),
|
||||
end: payPeriod.week2.end.toISODate(),
|
||||
label: formatWeekLabel(payPeriod.week2),
|
||||
},
|
||||
isCurrent: isCurrentPayPeriod(payPeriod),
|
||||
},
|
||||
totals: {
|
||||
hours: hoursData.totals.hours,
|
||||
breakHours: hoursData.totals.breakHours,
|
||||
overtimeHours: hoursData.totals.overtimeHours,
|
||||
regularHours: hoursData.totals.regularHours,
|
||||
activeEmployees,
|
||||
fte,
|
||||
avgHoursPerEmployee,
|
||||
},
|
||||
previousTotals: {
|
||||
hours: prevHoursData.totals.hours,
|
||||
overtimeHours: prevHoursData.totals.overtimeHours,
|
||||
activeEmployees: prevHoursData.totals.activeEmployees,
|
||||
fte: prevFte,
|
||||
},
|
||||
comparison,
|
||||
byEmployee: hoursData.byEmployee,
|
||||
byWeek: hoursData.byWeek,
|
||||
};
|
||||
|
||||
return { response, release };
|
||||
};
|
||||
|
||||
let result;
|
||||
try {
|
||||
result = await Promise.race([mainOperation(), timeoutPromise]);
|
||||
} catch (error) {
|
||||
if (error.message.includes('timeout')) {
|
||||
console.log(`[PAYROLL-METRICS] Request timed out in ${Date.now() - startTime}ms`);
|
||||
throw error;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
const { response, release } = result;
|
||||
|
||||
if (release) release();
|
||||
|
||||
console.log(`[PAYROLL-METRICS] Request completed in ${Date.now() - startTime}ms`);
|
||||
res.json(response);
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error in /payroll-metrics:', error);
|
||||
console.log(`[PAYROLL-METRICS] Request failed in ${Date.now() - startTime}ms`);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Get pay period info endpoint (for navigation without full data)
|
||||
router.get('/period-info', async (req, res) => {
|
||||
try {
|
||||
const { payPeriodStart, navigate } = req.query;
|
||||
|
||||
let payPeriod;
|
||||
|
||||
if (payPeriodStart) {
|
||||
const startDate = DateTime.fromISO(payPeriodStart, { zone: TIMEZONE });
|
||||
if (!startDate.isValid) {
|
||||
return res.status(400).json({ error: 'Invalid payPeriodStart date format' });
|
||||
}
|
||||
payPeriod = getPayPeriodForDate(startDate);
|
||||
} else {
|
||||
payPeriod = getCurrentPayPeriod();
|
||||
}
|
||||
|
||||
if (navigate) {
|
||||
const offset = parseInt(navigate, 10);
|
||||
if (!isNaN(offset)) {
|
||||
payPeriod = navigatePayPeriod(payPeriod.start, offset);
|
||||
}
|
||||
}
|
||||
|
||||
res.json({
|
||||
payPeriod: {
|
||||
start: payPeriod.start.toISODate(),
|
||||
end: payPeriod.end.toISODate(),
|
||||
label: formatPayPeriodLabel(payPeriod),
|
||||
week1: {
|
||||
start: payPeriod.week1.start.toISODate(),
|
||||
end: payPeriod.week1.end.toISODate(),
|
||||
label: formatWeekLabel(payPeriod.week1),
|
||||
},
|
||||
week2: {
|
||||
start: payPeriod.week2.start.toISODate(),
|
||||
end: payPeriod.week2.end.toISODate(),
|
||||
label: formatWeekLabel(payPeriod.week2),
|
||||
},
|
||||
isCurrent: isCurrentPayPeriod(payPeriod),
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error in /payroll-metrics/period-info:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Health check
|
||||
router.get('/health', async (req, res) => {
|
||||
try {
|
||||
const { connection, release } = await getDbConnection();
|
||||
await connection.execute('SELECT 1 as test');
|
||||
release();
|
||||
|
||||
res.json({
|
||||
status: 'healthy',
|
||||
timestamp: new Date().toISOString(),
|
||||
pool: getPoolStatus(),
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(500).json({
|
||||
status: 'unhealthy',
|
||||
timestamp: new Date().toISOString(),
|
||||
error: error.message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Helper functions
|
||||
function calculateComparison(currentValue, previousValue) {
|
||||
if (typeof previousValue !== 'number') {
|
||||
return { absolute: null, percentage: null };
|
||||
}
|
||||
|
||||
const absolute = typeof currentValue === 'number' ? currentValue - previousValue : null;
|
||||
const percentage =
|
||||
absolute !== null && previousValue !== 0
|
||||
? (absolute / Math.abs(previousValue)) * 100
|
||||
: null;
|
||||
|
||||
return { absolute, percentage };
|
||||
}
|
||||
|
||||
function formatPayPeriodLabel(payPeriod) {
|
||||
const startStr = payPeriod.start.toFormat('MMM d');
|
||||
const endStr = payPeriod.end.toFormat('MMM d, yyyy');
|
||||
return `${startStr} – ${endStr}`;
|
||||
}
|
||||
|
||||
function formatWeekLabel(week) {
|
||||
const startStr = week.start.toFormat('MMM d');
|
||||
const endStr = week.end.toFormat('MMM d');
|
||||
return `${startStr} – ${endStr}`;
|
||||
}
|
||||
|
||||
function isCurrentPayPeriod(payPeriod) {
|
||||
const now = DateTime.now().setZone(TIMEZONE);
|
||||
return now >= payPeriod.start && now <= payPeriod.end;
|
||||
}
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,57 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const { getDbConnection, getCachedQuery } = require('../db/connection');
|
||||
|
||||
// Test endpoint to count orders
|
||||
router.get('/order-count', async (req, res) => {
|
||||
try {
|
||||
const { connection } = await getDbConnection();
|
||||
|
||||
// Simple query to count orders from _order table
|
||||
const queryFn = async () => {
|
||||
const [rows] = await connection.execute('SELECT COUNT(*) as count FROM _order');
|
||||
return rows[0].count;
|
||||
};
|
||||
|
||||
const cacheKey = 'order-count';
|
||||
const count = await getCachedQuery(cacheKey, 'default', queryFn);
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
data: {
|
||||
orderCount: count,
|
||||
timestamp: new Date().toISOString()
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error fetching order count:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Test connection endpoint
|
||||
router.get('/test-connection', async (req, res) => {
|
||||
try {
|
||||
const { connection } = await getDbConnection();
|
||||
|
||||
// Test the connection with a simple query
|
||||
const [rows] = await connection.execute('SELECT 1 as test');
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
message: 'Database connection successful',
|
||||
data: rows[0]
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error testing connection:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
@@ -0,0 +1,103 @@
|
||||
require('dotenv').config();
|
||||
const express = require('express');
|
||||
const cors = require('cors');
|
||||
const morgan = require('morgan');
|
||||
const compression = require('compression');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { closeAllConnections } = require('./db/connection');
|
||||
|
||||
const app = express();
|
||||
const PORT = process.env.ACOT_PORT || 3012;
|
||||
|
||||
// Create logs directory if it doesn't exist
|
||||
const logDir = path.join(__dirname, 'logs/app');
|
||||
if (!fs.existsSync(logDir)) {
|
||||
fs.mkdirSync(logDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Create a write stream for access logs
|
||||
const accessLogStream = fs.createWriteStream(
|
||||
path.join(logDir, 'access.log'),
|
||||
{ flags: 'a' }
|
||||
);
|
||||
|
||||
// Middleware
|
||||
app.use(compression());
|
||||
app.use(cors());
|
||||
app.use(express.json());
|
||||
app.use(express.urlencoded({ extended: true }));
|
||||
|
||||
// Logging middleware
|
||||
if (process.env.NODE_ENV === 'production') {
|
||||
app.use(morgan('combined', { stream: accessLogStream }));
|
||||
} else {
|
||||
app.use(morgan('dev'));
|
||||
}
|
||||
|
||||
// Health check endpoint
|
||||
app.get('/health', (req, res) => {
|
||||
res.json({
|
||||
status: 'healthy',
|
||||
service: 'acot-server',
|
||||
timestamp: new Date().toISOString(),
|
||||
uptime: process.uptime()
|
||||
});
|
||||
});
|
||||
|
||||
// Routes
|
||||
app.use('/api/acot/test', require('./routes/test'));
|
||||
app.use('/api/acot/events', require('./routes/events'));
|
||||
app.use('/api/acot/discounts', require('./routes/discounts'));
|
||||
app.use('/api/acot/employee-metrics', require('./routes/employee-metrics'));
|
||||
app.use('/api/acot/payroll-metrics', require('./routes/payroll-metrics'));
|
||||
app.use('/api/acot/operations-metrics', require('./routes/operations-metrics'));
|
||||
app.use('/api/acot/customers', require('./routes/customers'));
|
||||
|
||||
// Error handling middleware
|
||||
app.use((err, req, res, next) => {
|
||||
console.error('Unhandled error:', err);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: process.env.NODE_ENV === 'production'
|
||||
? 'Internal server error'
|
||||
: err.message
|
||||
});
|
||||
});
|
||||
|
||||
// 404 handler
|
||||
app.use((req, res) => {
|
||||
res.status(404).json({
|
||||
success: false,
|
||||
error: 'Route not found'
|
||||
});
|
||||
});
|
||||
|
||||
// Start server
|
||||
const server = app.listen(PORT, () => {
|
||||
console.log(`ACOT Server running on port ${PORT}`);
|
||||
console.log(`Environment: ${process.env.NODE_ENV}`);
|
||||
});
|
||||
|
||||
// Graceful shutdown
|
||||
const gracefulShutdown = async () => {
|
||||
console.log('SIGTERM signal received: closing HTTP server');
|
||||
server.close(async () => {
|
||||
console.log('HTTP server closed');
|
||||
|
||||
// Close database connections
|
||||
try {
|
||||
await closeAllConnections();
|
||||
console.log('Database connections closed');
|
||||
} catch (error) {
|
||||
console.error('Error closing database connections:', error);
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
});
|
||||
};
|
||||
|
||||
process.on('SIGTERM', gracefulShutdown);
|
||||
process.on('SIGINT', gracefulShutdown);
|
||||
|
||||
module.exports = app;
|
||||
@@ -0,0 +1,28 @@
|
||||
// Shared-secret auth for customer-lookup endpoints that expose PII.
|
||||
// The acot-phone-server sends `x-acot-api-key` on every request; we compare
|
||||
// against ACOT_PHONE_API_KEY from the environment using timing-safe comparison.
|
||||
|
||||
const crypto = require('crypto');
|
||||
|
||||
function requirePhoneApiKey(req, res, next) {
|
||||
const expected = process.env.ACOT_PHONE_API_KEY;
|
||||
if (!expected) {
|
||||
console.error('ACOT_PHONE_API_KEY not configured; rejecting all requests');
|
||||
return res.status(503).json({ success: false, error: 'auth_not_configured' });
|
||||
}
|
||||
|
||||
const provided = req.get('x-acot-api-key') || '';
|
||||
const expectedBuf = Buffer.from(expected);
|
||||
const providedBuf = Buffer.from(provided);
|
||||
|
||||
if (
|
||||
providedBuf.length !== expectedBuf.length ||
|
||||
!crypto.timingSafeEqual(providedBuf, expectedBuf)
|
||||
) {
|
||||
return res.status(401).json({ success: false, error: 'unauthorized' });
|
||||
}
|
||||
|
||||
next();
|
||||
}
|
||||
|
||||
module.exports = { requirePhoneApiKey };
|
||||
@@ -0,0 +1,312 @@
|
||||
const { DateTime } = require('luxon');
|
||||
|
||||
const TIMEZONE = 'America/New_York';
|
||||
const DB_TIMEZONE = 'UTC-05:00';
|
||||
const BUSINESS_DAY_START_HOUR = 1; // 1 AM Eastern
|
||||
const WEEK_START_DAY = 7; // Sunday (Luxon uses 1 = Monday, 7 = Sunday)
|
||||
const DB_DATETIME_FORMAT = 'yyyy-LL-dd HH:mm:ss';
|
||||
|
||||
const isDateTime = (value) => DateTime.isDateTime(value);
|
||||
|
||||
const ensureDateTime = (value, { zone = TIMEZONE } = {}) => {
|
||||
if (!value) return null;
|
||||
|
||||
if (isDateTime(value)) {
|
||||
return value.setZone(zone);
|
||||
}
|
||||
|
||||
if (value instanceof Date) {
|
||||
return DateTime.fromJSDate(value, { zone });
|
||||
}
|
||||
|
||||
if (typeof value === 'number') {
|
||||
return DateTime.fromMillis(value, { zone });
|
||||
}
|
||||
|
||||
if (typeof value === 'string') {
|
||||
let dt = DateTime.fromISO(value, { zone, setZone: true });
|
||||
if (!dt.isValid) {
|
||||
dt = DateTime.fromSQL(value, { zone });
|
||||
}
|
||||
return dt.isValid ? dt : null;
|
||||
}
|
||||
|
||||
return null;
|
||||
};
|
||||
|
||||
const getNow = () => DateTime.now().setZone(TIMEZONE);
|
||||
|
||||
const getDayStart = (input = getNow()) => {
|
||||
const dt = ensureDateTime(input);
|
||||
if (!dt || !dt.isValid) {
|
||||
const fallback = getNow();
|
||||
return fallback.set({
|
||||
hour: BUSINESS_DAY_START_HOUR,
|
||||
minute: 0,
|
||||
second: 0,
|
||||
millisecond: 0
|
||||
});
|
||||
}
|
||||
|
||||
const sameDayStart = dt.set({
|
||||
hour: BUSINESS_DAY_START_HOUR,
|
||||
minute: 0,
|
||||
second: 0,
|
||||
millisecond: 0
|
||||
});
|
||||
|
||||
return dt.hour < BUSINESS_DAY_START_HOUR
|
||||
? sameDayStart.minus({ days: 1 })
|
||||
: sameDayStart;
|
||||
};
|
||||
|
||||
const getDayEnd = (input = getNow()) => {
|
||||
return getDayStart(input).plus({ days: 1 }).minus({ milliseconds: 1 });
|
||||
};
|
||||
|
||||
const getWeekStart = (input = getNow()) => {
|
||||
const dt = ensureDateTime(input);
|
||||
if (!dt || !dt.isValid) {
|
||||
return getDayStart();
|
||||
}
|
||||
|
||||
const startOfWeek = dt.set({ weekday: WEEK_START_DAY }).startOf('day');
|
||||
const normalized = startOfWeek > dt ? startOfWeek.minus({ weeks: 1 }) : startOfWeek;
|
||||
return normalized.set({
|
||||
hour: BUSINESS_DAY_START_HOUR,
|
||||
minute: 0,
|
||||
second: 0,
|
||||
millisecond: 0
|
||||
});
|
||||
};
|
||||
|
||||
const getRangeForTimeRange = (timeRange = 'today', now = getNow()) => {
|
||||
const current = ensureDateTime(now);
|
||||
if (!current || !current.isValid) {
|
||||
throw new Error('Invalid reference time for range calculation');
|
||||
}
|
||||
|
||||
switch (timeRange) {
|
||||
case 'today': {
|
||||
return {
|
||||
start: getDayStart(current),
|
||||
end: getDayEnd(current)
|
||||
};
|
||||
}
|
||||
case 'yesterday': {
|
||||
const target = current.minus({ days: 1 });
|
||||
return {
|
||||
start: getDayStart(target),
|
||||
end: getDayEnd(target)
|
||||
};
|
||||
}
|
||||
case 'twoDaysAgo': {
|
||||
const target = current.minus({ days: 2 });
|
||||
return {
|
||||
start: getDayStart(target),
|
||||
end: getDayEnd(target)
|
||||
};
|
||||
}
|
||||
case 'thisWeek': {
|
||||
return {
|
||||
start: getWeekStart(current),
|
||||
end: getDayEnd(current)
|
||||
};
|
||||
}
|
||||
case 'lastWeek': {
|
||||
const lastWeek = current.minus({ weeks: 1 });
|
||||
const weekStart = getWeekStart(lastWeek);
|
||||
const weekEnd = weekStart.plus({ days: 6 });
|
||||
return {
|
||||
start: weekStart,
|
||||
end: getDayEnd(weekEnd)
|
||||
};
|
||||
}
|
||||
case 'thisMonth': {
|
||||
const dayStart = getDayStart(current);
|
||||
const monthStart = dayStart.startOf('month').set({ hour: BUSINESS_DAY_START_HOUR });
|
||||
return {
|
||||
start: monthStart,
|
||||
end: getDayEnd(current)
|
||||
};
|
||||
}
|
||||
case 'lastMonth': {
|
||||
const lastMonth = current.minus({ months: 1 });
|
||||
const monthStart = lastMonth
|
||||
.startOf('month')
|
||||
.set({ hour: BUSINESS_DAY_START_HOUR, minute: 0, second: 0, millisecond: 0 });
|
||||
const monthEnd = monthStart.plus({ months: 1 }).minus({ days: 1 });
|
||||
return {
|
||||
start: monthStart,
|
||||
end: getDayEnd(monthEnd)
|
||||
};
|
||||
}
|
||||
case 'last7days': {
|
||||
const dayStart = getDayStart(current);
|
||||
return {
|
||||
start: dayStart.minus({ days: 6 }),
|
||||
end: getDayEnd(current)
|
||||
};
|
||||
}
|
||||
case 'last30days': {
|
||||
const dayStart = getDayStart(current);
|
||||
return {
|
||||
start: dayStart.minus({ days: 29 }),
|
||||
end: getDayEnd(current)
|
||||
};
|
||||
}
|
||||
case 'last90days': {
|
||||
const dayStart = getDayStart(current);
|
||||
return {
|
||||
start: dayStart.minus({ days: 89 }),
|
||||
end: getDayEnd(current)
|
||||
};
|
||||
}
|
||||
case 'previous7days': {
|
||||
const currentPeriodStart = getDayStart(current).minus({ days: 6 });
|
||||
const previousEndDay = currentPeriodStart.minus({ days: 1 });
|
||||
const previousStartDay = previousEndDay.minus({ days: 6 });
|
||||
return {
|
||||
start: getDayStart(previousStartDay),
|
||||
end: getDayEnd(previousEndDay)
|
||||
};
|
||||
}
|
||||
case 'previous30days': {
|
||||
const currentPeriodStart = getDayStart(current).minus({ days: 29 });
|
||||
const previousEndDay = currentPeriodStart.minus({ days: 1 });
|
||||
const previousStartDay = previousEndDay.minus({ days: 29 });
|
||||
return {
|
||||
start: getDayStart(previousStartDay),
|
||||
end: getDayEnd(previousEndDay)
|
||||
};
|
||||
}
|
||||
case 'previous90days': {
|
||||
const currentPeriodStart = getDayStart(current).minus({ days: 89 });
|
||||
const previousEndDay = currentPeriodStart.minus({ days: 1 });
|
||||
const previousStartDay = previousEndDay.minus({ days: 89 });
|
||||
return {
|
||||
start: getDayStart(previousStartDay),
|
||||
end: getDayEnd(previousEndDay)
|
||||
};
|
||||
}
|
||||
default:
|
||||
throw new Error(`Unknown time range: ${timeRange}`);
|
||||
}
|
||||
};
|
||||
|
||||
const toDatabaseSqlString = (dt) => {
|
||||
const normalized = ensureDateTime(dt);
|
||||
if (!normalized || !normalized.isValid) {
|
||||
throw new Error('Invalid datetime provided for SQL conversion');
|
||||
}
|
||||
const dbTime = normalized.setZone(DB_TIMEZONE, { keepLocalTime: true });
|
||||
return dbTime.toFormat(DB_DATETIME_FORMAT);
|
||||
};
|
||||
|
||||
const formatBusinessDate = (input) => {
|
||||
const dt = ensureDateTime(input);
|
||||
if (!dt || !dt.isValid) return '';
|
||||
return dt.setZone(TIMEZONE).toFormat('LLL d, yyyy');
|
||||
};
|
||||
|
||||
const getTimeRangeLabel = (timeRange) => {
|
||||
const labels = {
|
||||
today: 'Today',
|
||||
yesterday: 'Yesterday',
|
||||
twoDaysAgo: 'Two Days Ago',
|
||||
thisWeek: 'This Week',
|
||||
lastWeek: 'Last Week',
|
||||
thisMonth: 'This Month',
|
||||
lastMonth: 'Last Month',
|
||||
last7days: 'Last 7 Days',
|
||||
last30days: 'Last 30 Days',
|
||||
last90days: 'Last 90 Days',
|
||||
previous7days: 'Previous 7 Days',
|
||||
previous30days: 'Previous 30 Days',
|
||||
previous90days: 'Previous 90 Days'
|
||||
};
|
||||
|
||||
return labels[timeRange] || timeRange;
|
||||
};
|
||||
|
||||
const getTimeRangeConditions = (timeRange, startDate, endDate) => {
|
||||
if (timeRange === 'custom' && startDate && endDate) {
|
||||
const start = ensureDateTime(startDate);
|
||||
const end = ensureDateTime(endDate);
|
||||
|
||||
if (!start || !start.isValid || !end || !end.isValid) {
|
||||
throw new Error('Invalid custom date range provided');
|
||||
}
|
||||
|
||||
return {
|
||||
whereClause: 'date_placed >= ? AND date_placed <= ?',
|
||||
params: [toDatabaseSqlString(start), toDatabaseSqlString(end)],
|
||||
dateRange: {
|
||||
start: start.toUTC().toISO(),
|
||||
end: end.toUTC().toISO(),
|
||||
label: `${formatBusinessDate(start)} - ${formatBusinessDate(end)}`
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
const normalizedRange = timeRange || 'today';
|
||||
const range = getRangeForTimeRange(normalizedRange);
|
||||
|
||||
return {
|
||||
whereClause: 'date_placed >= ? AND date_placed <= ?',
|
||||
params: [toDatabaseSqlString(range.start), toDatabaseSqlString(range.end)],
|
||||
dateRange: {
|
||||
start: range.start.toUTC().toISO(),
|
||||
end: range.end.toUTC().toISO(),
|
||||
label: getTimeRangeLabel(normalizedRange)
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
const getBusinessDayBounds = (timeRange) => {
|
||||
const range = getRangeForTimeRange(timeRange);
|
||||
return {
|
||||
start: range.start.toJSDate(),
|
||||
end: range.end.toJSDate()
|
||||
};
|
||||
};
|
||||
|
||||
const parseBusinessDate = (mysqlDatetime) => {
|
||||
if (!mysqlDatetime || mysqlDatetime === '0000-00-00 00:00:00') {
|
||||
return null;
|
||||
}
|
||||
|
||||
const dt = DateTime.fromSQL(mysqlDatetime, { zone: DB_TIMEZONE });
|
||||
if (!dt.isValid) {
|
||||
console.error('[timeUtils] Failed to parse MySQL datetime:', mysqlDatetime, dt.invalidExplanation);
|
||||
return null;
|
||||
}
|
||||
|
||||
return dt.toUTC().toJSDate();
|
||||
};
|
||||
|
||||
const formatMySQLDate = (input) => {
|
||||
if (!input) return null;
|
||||
|
||||
const dt = ensureDateTime(input, { zone: 'utc' });
|
||||
if (!dt || !dt.isValid) return null;
|
||||
|
||||
return dt.setZone(DB_TIMEZONE).toFormat(DB_DATETIME_FORMAT);
|
||||
};
|
||||
|
||||
module.exports = {
|
||||
getBusinessDayBounds,
|
||||
getTimeRangeConditions,
|
||||
formatBusinessDate,
|
||||
getTimeRangeLabel,
|
||||
parseBusinessDate,
|
||||
formatMySQLDate,
|
||||
// Expose helpers for tests or advanced consumers
|
||||
_internal: {
|
||||
getDayStart,
|
||||
getDayEnd,
|
||||
getWeekStart,
|
||||
getRangeForTimeRange,
|
||||
BUSINESS_DAY_START_HOUR
|
||||
}
|
||||
};
|
||||
+2939
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"name": "dashboard-server",
|
||||
"version": "1.0.0",
|
||||
"description": "Merged ESM dashboard server (klaviyo + meta + google-analytics + typeform). Phase 4 of CONSOLIDATION_PLAN.md.",
|
||||
"main": "server.js",
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"start": "node server.js",
|
||||
"dev": "nodemon server.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"@google-analytics/data": "^4.0.0",
|
||||
"axios": "^1.7.9",
|
||||
"cors": "^2.8.5",
|
||||
"dotenv": "^16.4.7",
|
||||
"express": "^4.21.2",
|
||||
"ioredis": "^5.4.1",
|
||||
"jsonwebtoken": "^9.0.2",
|
||||
"lodash": "^4.17.21",
|
||||
"luxon": "^3.5.0",
|
||||
"node-fetch": "^3.3.2",
|
||||
"pg": "^8.18.0",
|
||||
"pino": "^9.5.0",
|
||||
"pino-http": "^10.3.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"nodemon": "^3.0.2"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
// Google Analytics router — ESM conversion of google-server/routes/analytics.routes.js.
|
||||
// All routes are read-only — authenticated-only is sufficient; no extra permission.
|
||||
// google_write is reserved for future write endpoints (per migration 005).
|
||||
|
||||
import express from 'express';
|
||||
import { AnalyticsService } from '../../services/google/analytics.service.js';
|
||||
|
||||
export function createGoogleRouter({ redis }) {
|
||||
const router = express.Router();
|
||||
const service = new AnalyticsService(redis);
|
||||
|
||||
router.get('/metrics', async (req, res) => {
|
||||
try {
|
||||
const { startDate = '7daysAgo' } = req.query;
|
||||
const data = await service.getBasicMetrics(startDate);
|
||||
res.json({ success: true, data });
|
||||
} catch (error) {
|
||||
console.error('Metrics error:', { startDate: req.query.startDate, error: error.message });
|
||||
res.status(500).json({ success: false, error: 'Failed to fetch metrics', details: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/realtime/basic', async (req, res) => {
|
||||
try {
|
||||
const data = await service.getRealTimeBasicData();
|
||||
res.json({ success: true, data });
|
||||
} catch (error) {
|
||||
console.error('Realtime basic error:', { error: error.message });
|
||||
res.status(500).json({ success: false, error: 'Failed to fetch realtime basic data', details: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/realtime/detailed', async (req, res) => {
|
||||
try {
|
||||
const data = await service.getRealTimeDetailedData();
|
||||
res.json({ success: true, data });
|
||||
} catch (error) {
|
||||
console.error('Realtime detailed error:', { error: error.message });
|
||||
res.status(500).json({ success: false, error: 'Failed to fetch realtime detailed data', details: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/user-behavior', async (req, res) => {
|
||||
try {
|
||||
const { timeRange = '30' } = req.query;
|
||||
const data = await service.getUserBehavior(timeRange);
|
||||
res.json({ success: true, data });
|
||||
} catch (error) {
|
||||
console.error('User behavior error:', { timeRange: req.query.timeRange, error: error.message });
|
||||
res.status(500).json({ success: false, error: 'Failed to fetch user behavior data', details: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
import express from 'express';
|
||||
import { CampaignsService } from '../../services/klaviyo/campaigns.service.js';
|
||||
import { TimeManager } from '../../utils/time.utils.js';
|
||||
|
||||
export function createCampaignsRouter(apiKey, apiRevision, redis) {
|
||||
const router = express.Router();
|
||||
const timeManager = new TimeManager();
|
||||
const campaignsService = new CampaignsService(apiKey, apiRevision, redis);
|
||||
|
||||
// Get campaigns with optional filtering
|
||||
router.get('/', async (req, res) => {
|
||||
try {
|
||||
const params = {
|
||||
pageSize: parseInt(req.query.pageSize) || 50,
|
||||
sort: req.query.sort || '-send_time',
|
||||
status: req.query.status,
|
||||
startDate: req.query.startDate,
|
||||
endDate: req.query.endDate,
|
||||
pageCursor: req.query.pageCursor
|
||||
};
|
||||
|
||||
console.log('[Campaigns Route] Fetching campaigns with params:', params);
|
||||
const data = await campaignsService.getCampaigns(params);
|
||||
console.log('[Campaigns Route] Success:', {
|
||||
count: data.data?.length || 0
|
||||
});
|
||||
res.json(data);
|
||||
} catch (error) {
|
||||
console.error('[Campaigns Route] Error:', error);
|
||||
res.status(500).json({
|
||||
status: 'error',
|
||||
message: error.message,
|
||||
details: error.response?.data || null
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Get campaigns by time range
|
||||
router.get('/:timeRange', async (req, res) => {
|
||||
try {
|
||||
const { timeRange } = req.params;
|
||||
const { status } = req.query;
|
||||
|
||||
let result;
|
||||
if (timeRange === 'custom') {
|
||||
const { startDate, endDate } = req.query;
|
||||
if (!startDate || !endDate) {
|
||||
return res.status(400).json({ error: 'Custom range requires startDate and endDate' });
|
||||
}
|
||||
|
||||
result = await campaignsService.getCampaigns({
|
||||
startDate,
|
||||
endDate,
|
||||
status
|
||||
});
|
||||
} else {
|
||||
result = await campaignsService.getCampaignsByTimeRange(
|
||||
timeRange,
|
||||
{ status }
|
||||
);
|
||||
}
|
||||
|
||||
res.json(result);
|
||||
} catch (error) {
|
||||
console.error("[Campaigns Route] Error:", error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
@@ -0,0 +1,485 @@
|
||||
import express from 'express';
|
||||
import { EventsService } from '../../services/klaviyo/events.service.js';
|
||||
import { TimeManager } from '../../utils/time.utils.js';
|
||||
import { RedisService } from '../../services/klaviyo/redis.service.js';
|
||||
import { requirePermission } from '../../../shared/auth/middleware.js';
|
||||
|
||||
// Import METRIC_IDS from events service
|
||||
const METRIC_IDS = {
|
||||
PLACED_ORDER: 'Y8cqcF',
|
||||
SHIPPED_ORDER: 'VExpdL',
|
||||
ACCOUNT_CREATED: 'TeeypV',
|
||||
CANCELED_ORDER: 'YjVMNg',
|
||||
NEW_BLOG_POST: 'YcxeDr',
|
||||
PAYMENT_REFUNDED: 'R7XUYh'
|
||||
};
|
||||
|
||||
export function createEventsRouter(apiKey, apiRevision, redis) {
|
||||
const router = express.Router();
|
||||
const timeManager = new TimeManager();
|
||||
const eventsService = new EventsService(apiKey, apiRevision, redis);
|
||||
const redisService = new RedisService(redis);
|
||||
|
||||
// Phase 6.2: clearCache is operational maintenance — requires klaviyo_admin.
|
||||
// Mounted as path-level middleware so the existing POST handler below stays untouched.
|
||||
router.use('/clearCache', requirePermission('klaviyo_admin'));
|
||||
|
||||
// Get events with optional filtering
|
||||
router.get('/', async (req, res) => {
|
||||
try {
|
||||
const params = {
|
||||
pageSize: parseInt(req.query.pageSize) || 50,
|
||||
sort: req.query.sort || '-datetime',
|
||||
metricId: req.query.metricId,
|
||||
startDate: req.query.startDate,
|
||||
endDate: req.query.endDate,
|
||||
pageCursor: req.query.pageCursor,
|
||||
fields: {}
|
||||
};
|
||||
|
||||
// Parse fields parameter if provided
|
||||
if (req.query.fields) {
|
||||
try {
|
||||
params.fields = JSON.parse(req.query.fields);
|
||||
} catch (e) {
|
||||
console.warn('[Events Route] Invalid fields parameter:', e);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('[Events Route] Fetching events with params:', params);
|
||||
const data = await eventsService.getEvents(params);
|
||||
console.log('[Events Route] Success:', {
|
||||
count: data.data?.length || 0,
|
||||
included: data.included?.length || 0
|
||||
});
|
||||
res.json(data);
|
||||
} catch (error) {
|
||||
console.error('[Events Route] Error:', error);
|
||||
res.status(500).json({
|
||||
status: 'error',
|
||||
message: error.message,
|
||||
details: error.response?.data || null
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Get events by time range
|
||||
router.get('/by-time/:timeRange', async (req, res) => {
|
||||
try {
|
||||
const { timeRange } = req.params;
|
||||
const { metricId, startDate, endDate } = req.query;
|
||||
|
||||
let result;
|
||||
if (timeRange === 'custom') {
|
||||
if (!startDate || !endDate) {
|
||||
return res.status(400).json({ error: 'Custom range requires startDate and endDate' });
|
||||
}
|
||||
|
||||
const range = timeManager.getCustomRange(startDate, endDate);
|
||||
if (!range) {
|
||||
return res.status(400).json({ error: 'Invalid date range' });
|
||||
}
|
||||
|
||||
result = await eventsService.getEvents({
|
||||
metricId,
|
||||
startDate: range.start.toISO(),
|
||||
endDate: range.end.toISO()
|
||||
});
|
||||
} else {
|
||||
result = await eventsService.getEventsByTimeRange(
|
||||
timeRange,
|
||||
{ metricId }
|
||||
);
|
||||
}
|
||||
|
||||
res.json(result);
|
||||
} catch (error) {
|
||||
console.error("[Events Route] Error:", error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Get comprehensive statistics for a time period
|
||||
router.get('/stats', async (req, res) => {
|
||||
try {
|
||||
const { timeRange, startDate, endDate } = req.query;
|
||||
console.log('[Events Route] Stats request:', {
|
||||
timeRange,
|
||||
startDate,
|
||||
endDate
|
||||
});
|
||||
|
||||
let range;
|
||||
if (startDate && endDate) {
|
||||
range = timeManager.getCustomRange(startDate, endDate);
|
||||
} else if (timeRange) {
|
||||
range = timeManager.getDateRange(timeRange);
|
||||
} else {
|
||||
return res.status(400).json({ error: 'Must provide either timeRange or startDate and endDate' });
|
||||
}
|
||||
|
||||
if (!range) {
|
||||
return res.status(400).json({ error: 'Invalid time range' });
|
||||
}
|
||||
|
||||
const params = {
|
||||
timeRange,
|
||||
startDate: range.start.toISO(),
|
||||
endDate: range.end.toISO()
|
||||
};
|
||||
|
||||
console.log('[Events Route] Calculating period stats with params:', params);
|
||||
const stats = await eventsService.calculatePeriodStats(params);
|
||||
console.log('[Events Route] Stats response:', {
|
||||
timeRange: {
|
||||
start: range.start.toISO(),
|
||||
end: range.end.toISO()
|
||||
},
|
||||
shippedCount: stats?.shipping?.shippedCount,
|
||||
totalOrders: stats?.orderCount
|
||||
});
|
||||
|
||||
res.json({
|
||||
timeRange: {
|
||||
start: range.start.toISO(),
|
||||
end: range.end.toISO(),
|
||||
displayStart: timeManager.formatForDisplay(range.start),
|
||||
displayEnd: timeManager.formatForDisplay(range.end)
|
||||
},
|
||||
stats
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("[Events Route] Error:", error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Add new route for smart revenue projection
|
||||
router.get('/projection', async (req, res) => {
|
||||
try {
|
||||
const { timeRange, startDate, endDate } = req.query;
|
||||
console.log('[Events Route] Projection request:', {
|
||||
timeRange,
|
||||
startDate,
|
||||
endDate
|
||||
});
|
||||
|
||||
let range;
|
||||
if (startDate && endDate) {
|
||||
range = timeManager.getCustomRange(startDate, endDate);
|
||||
} else if (timeRange) {
|
||||
range = timeManager.getDateRange(timeRange);
|
||||
} else {
|
||||
return res.status(400).json({ error: 'Must provide either timeRange or startDate and endDate' });
|
||||
}
|
||||
|
||||
if (!range) {
|
||||
return res.status(400).json({ error: 'Invalid time range' });
|
||||
}
|
||||
|
||||
const params = {
|
||||
timeRange,
|
||||
startDate: range.start.toISO(),
|
||||
endDate: range.end.toISO()
|
||||
};
|
||||
|
||||
// Try to get from cache first with a short TTL
|
||||
const cacheKey = redisService._getCacheKey('projection', params);
|
||||
const cachedData = await redisService.get(cacheKey);
|
||||
|
||||
if (cachedData) {
|
||||
console.log('[Events Route] Cache hit for projection');
|
||||
return res.json(cachedData);
|
||||
}
|
||||
|
||||
console.log('[Events Route] Calculating smart projection with params:', params);
|
||||
const projection = await eventsService.calculateSmartProjection(params);
|
||||
|
||||
// Cache the results with a short TTL (5 minutes)
|
||||
await redisService.set(cacheKey, projection, 300);
|
||||
|
||||
res.json(projection);
|
||||
} catch (error) {
|
||||
console.error("[Events Route] Error calculating projection:", error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Add new route for detailed stats
|
||||
router.get('/stats/details', async (req, res) => {
|
||||
try {
|
||||
const { timeRange, startDate, endDate, metric, daily = false } = req.query;
|
||||
|
||||
let range;
|
||||
if (startDate && endDate) {
|
||||
range = timeManager.getCustomRange(startDate, endDate);
|
||||
} else if (timeRange) {
|
||||
range = timeManager.getDateRange(timeRange);
|
||||
} else {
|
||||
return res.status(400).json({ error: 'Must provide either timeRange or startDate and endDate' });
|
||||
}
|
||||
|
||||
if (!range) {
|
||||
return res.status(400).json({ error: 'Invalid time range' });
|
||||
}
|
||||
|
||||
const params = {
|
||||
timeRange,
|
||||
startDate: range.start.toISO(),
|
||||
endDate: range.end.toISO(),
|
||||
metric,
|
||||
daily: daily === 'true' || daily === true
|
||||
};
|
||||
|
||||
// Try to get from cache first
|
||||
const cacheKey = redisService._getCacheKey('stats:details', params);
|
||||
const cachedData = await redisService.get(cacheKey);
|
||||
|
||||
if (cachedData) {
|
||||
console.log('[Events Route] Cache hit for detailed stats');
|
||||
return res.json({
|
||||
timeRange: {
|
||||
start: range.start.toISO(),
|
||||
end: range.end.toISO(),
|
||||
displayStart: timeManager.formatForDisplay(range.start),
|
||||
displayEnd: timeManager.formatForDisplay(range.end)
|
||||
},
|
||||
stats: cachedData
|
||||
});
|
||||
}
|
||||
|
||||
const stats = await eventsService.calculateDetailedStats(params);
|
||||
|
||||
// Cache the results
|
||||
const ttl = redisService._getTTL(timeRange);
|
||||
await redisService.set(cacheKey, stats, ttl);
|
||||
|
||||
res.json({
|
||||
timeRange: {
|
||||
start: range.start.toISO(),
|
||||
end: range.end.toISO(),
|
||||
displayStart: timeManager.formatForDisplay(range.start),
|
||||
displayEnd: timeManager.formatForDisplay(range.end)
|
||||
},
|
||||
stats
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("[Events Route] Error:", error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Get product statistics for a time period
|
||||
router.get('/products', async (req, res) => {
|
||||
try {
|
||||
const { timeRange, startDate, endDate } = req.query;
|
||||
|
||||
let range;
|
||||
if (startDate && endDate) {
|
||||
range = timeManager.getCustomRange(startDate, endDate);
|
||||
} else if (timeRange) {
|
||||
range = timeManager.getDateRange(timeRange);
|
||||
} else {
|
||||
return res.status(400).json({ error: 'Must provide either timeRange or startDate and endDate' });
|
||||
}
|
||||
|
||||
if (!range) {
|
||||
return res.status(400).json({ error: 'Invalid time range' });
|
||||
}
|
||||
|
||||
const params = {
|
||||
timeRange,
|
||||
startDate: range.start.toISO(),
|
||||
endDate: range.end.toISO()
|
||||
};
|
||||
|
||||
// Try to get from cache first
|
||||
const cacheKey = redisService._getCacheKey('events', params);
|
||||
const cachedData = await redisService.getEventData('products', params);
|
||||
|
||||
if (cachedData) {
|
||||
console.log('[Events Route] Cache hit for products');
|
||||
return res.json({
|
||||
timeRange: {
|
||||
start: range.start.toISO(),
|
||||
end: range.end.toISO(),
|
||||
displayStart: timeManager.formatForDisplay(range.start),
|
||||
displayEnd: timeManager.formatForDisplay(range.end)
|
||||
},
|
||||
stats: {
|
||||
products: cachedData
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
const stats = await eventsService.calculatePeriodStats(params);
|
||||
|
||||
res.json({
|
||||
timeRange: {
|
||||
start: range.start.toISO(),
|
||||
end: range.end.toISO(),
|
||||
displayStart: timeManager.formatForDisplay(range.start),
|
||||
displayEnd: timeManager.formatForDisplay(range.end)
|
||||
},
|
||||
stats
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("[Events Route] Error:", error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Get event feed (multiple event types sorted by time)
|
||||
router.get('/feed', async (req, res) => {
|
||||
try {
|
||||
const { timeRange, startDate, endDate, metricIds } = req.query;
|
||||
|
||||
let range;
|
||||
if (startDate && endDate) {
|
||||
range = timeManager.getCustomRange(startDate, endDate);
|
||||
} else if (timeRange) {
|
||||
range = timeManager.getDateRange(timeRange);
|
||||
} else {
|
||||
return res.status(400).json({ error: 'Must provide either timeRange or startDate and endDate' });
|
||||
}
|
||||
|
||||
if (!range) {
|
||||
return res.status(400).json({ error: 'Invalid time range' });
|
||||
}
|
||||
|
||||
const params = {
|
||||
timeRange,
|
||||
startDate: range.start.toISO(),
|
||||
endDate: range.end.toISO(),
|
||||
metricIds: metricIds ? JSON.parse(metricIds) : null
|
||||
};
|
||||
|
||||
const result = await eventsService.getMultiMetricEvents(params);
|
||||
|
||||
res.json({
|
||||
timeRange: {
|
||||
start: range.start.toISO(),
|
||||
end: range.end.toISO(),
|
||||
displayStart: timeManager.formatForDisplay(range.start),
|
||||
displayEnd: timeManager.formatForDisplay(range.end)
|
||||
},
|
||||
...result
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("[Events Route] Error:", error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Get aggregated events data
|
||||
router.get('/aggregate', async (req, res) => {
|
||||
try {
|
||||
const { timeRange, startDate, endDate, interval = 'day', metricId, property } = req.query;
|
||||
|
||||
let range;
|
||||
if (startDate && endDate) {
|
||||
range = timeManager.getCustomRange(startDate, endDate);
|
||||
} else if (timeRange) {
|
||||
range = timeManager.getDateRange(timeRange);
|
||||
} else {
|
||||
return res.status(400).json({ error: 'Must provide either timeRange or startDate and endDate' });
|
||||
}
|
||||
|
||||
if (!range) {
|
||||
return res.status(400).json({ error: 'Invalid time range' });
|
||||
}
|
||||
|
||||
const params = {
|
||||
timeRange,
|
||||
startDate: range.start.toISO(),
|
||||
endDate: range.end.toISO(),
|
||||
metricId,
|
||||
interval,
|
||||
property
|
||||
};
|
||||
|
||||
const result = await eventsService.getEvents(params);
|
||||
const groupedData = timeManager.groupEventsByInterval(result.data, interval, property);
|
||||
|
||||
res.json({
|
||||
timeRange: {
|
||||
start: range.start.toISO(),
|
||||
end: range.end.toISO(),
|
||||
displayStart: timeManager.formatForDisplay(range.start),
|
||||
displayEnd: timeManager.formatForDisplay(range.end)
|
||||
},
|
||||
data: groupedData
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("[Events Route] Error:", error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Get date range for a given time period
|
||||
router.get("/dateRange", async (req, res) => {
|
||||
try {
|
||||
const { timeRange, startDate, endDate } = req.query;
|
||||
|
||||
let range;
|
||||
if (startDate && endDate) {
|
||||
range = timeManager.getCustomRange(startDate, endDate);
|
||||
} else {
|
||||
range = timeManager.getDateRange(timeRange || 'today');
|
||||
}
|
||||
|
||||
if (!range) {
|
||||
return res.status(400).json({
|
||||
error: "Invalid time range parameters"
|
||||
});
|
||||
}
|
||||
|
||||
res.json({
|
||||
start: range.start.toISO(),
|
||||
end: range.end.toISO(),
|
||||
displayStart: timeManager.formatForDisplay(range.start),
|
||||
displayEnd: timeManager.formatForDisplay(range.end)
|
||||
});
|
||||
} catch (error) {
|
||||
console.error('Error getting date range:', error);
|
||||
res.status(500).json({
|
||||
error: "Failed to get date range"
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Clear cache for a specific time range
|
||||
router.post("/clearCache", async (req, res) => {
|
||||
try {
|
||||
const { timeRange, startDate, endDate } = req.body;
|
||||
await redisService.clearCache({ timeRange, startDate, endDate });
|
||||
res.json({ message: "Cache cleared successfully" });
|
||||
} catch (error) {
|
||||
console.error('Error clearing cache:', error);
|
||||
res.status(500).json({ error: "Failed to clear cache" });
|
||||
}
|
||||
});
|
||||
|
||||
// Add new batch metrics endpoint
|
||||
router.get('/batch', async (req, res) => {
|
||||
try {
|
||||
const { timeRange, startDate, endDate, metrics } = req.query;
|
||||
|
||||
// Parse metrics array from query
|
||||
const metricsList = metrics ? JSON.parse(metrics) : [];
|
||||
|
||||
const params = timeRange === 'custom'
|
||||
? { startDate, endDate, metrics: metricsList }
|
||||
: { timeRange, metrics: metricsList };
|
||||
|
||||
const results = await eventsService.getBatchMetrics(params);
|
||||
|
||||
res.json(results);
|
||||
} catch (error) {
|
||||
console.error('[Events Route] Error in batch request:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
// Klaviyo router factory. Phase 4 merge: takes the injected redis client and
|
||||
// the env-resolved API key/revision, returns the mounted /api/klaviyo router
|
||||
// (matches Caddy proxy path; no rewrite needed).
|
||||
|
||||
import express from 'express';
|
||||
import rateLimit from 'express-rate-limit';
|
||||
import { requirePermission } from '../../../shared/auth/middleware.js';
|
||||
|
||||
import { createEventsRouter } from './events.routes.js';
|
||||
import { createMetricsRouter } from './metrics.routes.js';
|
||||
import { createCampaignsRouter } from './campaigns.routes.js';
|
||||
import { createReportingRouter } from './reporting.routes.js';
|
||||
|
||||
export function createKlaviyoRouter({ redis }) {
|
||||
const apiKey = process.env.KLAVIYO_API_KEY;
|
||||
const apiRevision = process.env.KLAVIYO_API_REVISION || '2024-02-15';
|
||||
|
||||
if (!apiKey) {
|
||||
// Loud at startup; the routes themselves will 500 on every call without it.
|
||||
console.warn('[klaviyo] KLAVIYO_API_KEY not set — Klaviyo endpoints will fail');
|
||||
}
|
||||
|
||||
const router = express.Router();
|
||||
|
||||
// Phase 4 carryover from klaviyo-server: throttle the heavy /reporting/campaign-values-reports
|
||||
// endpoint. authenticate() already runs upstream so we don't add a per-user limiter here.
|
||||
const reportingLimiter = rateLimit({
|
||||
windowMs: 10 * 60 * 1000,
|
||||
max: 10,
|
||||
message: 'Too many requests to reporting endpoint, please try again later',
|
||||
keyGenerator: (req) => `${req.ip}-klaviyo-reporting`,
|
||||
skip: (req) => !req.path.includes('campaign-values-reports'),
|
||||
standardHeaders: true,
|
||||
legacyHeaders: false,
|
||||
});
|
||||
router.use('/reporting', reportingLimiter);
|
||||
|
||||
router.use('/events', createEventsRouter(apiKey, apiRevision, redis));
|
||||
router.use('/metrics', createMetricsRouter(apiKey, apiRevision));
|
||||
router.use('/campaigns', createCampaignsRouter(apiKey, apiRevision, redis));
|
||||
router.use('/reporting', createReportingRouter(apiKey, apiRevision, redis));
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
// Re-exported so the dashboard server / future tests can attach the
|
||||
// klaviyo_admin gate without reaching into the events router file.
|
||||
export { requirePermission };
|
||||
@@ -0,0 +1,28 @@
|
||||
import express from 'express';
|
||||
import { MetricsService } from '../../services/klaviyo/metrics.service.js';
|
||||
|
||||
export function createMetricsRouter(apiKey, apiRevision) {
|
||||
const router = express.Router();
|
||||
const metricsService = new MetricsService(apiKey, apiRevision);
|
||||
|
||||
// Get all metrics
|
||||
router.get('/', async (req, res) => {
|
||||
try {
|
||||
console.log('[Metrics Route] Fetching metrics');
|
||||
const data = await metricsService.getMetrics();
|
||||
console.log('[Metrics Route] Success:', {
|
||||
count: data.data?.length || 0
|
||||
});
|
||||
res.json(data);
|
||||
} catch (error) {
|
||||
console.error('[Metrics Route] Error:', error);
|
||||
res.status(500).json({
|
||||
status: 'error',
|
||||
message: error.message,
|
||||
details: error.response?.data || null
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import express from 'express';
|
||||
import { ReportingService } from '../../services/klaviyo/reporting.service.js';
|
||||
import { TimeManager } from '../../utils/time.utils.js';
|
||||
|
||||
export function createReportingRouter(apiKey, apiRevision, redis) {
|
||||
const router = express.Router();
|
||||
const reportingService = new ReportingService(apiKey, apiRevision, redis);
|
||||
const timeManager = new TimeManager();
|
||||
|
||||
// Get campaign reports by time range
|
||||
router.get('/campaigns/:timeRange', async (req, res) => {
|
||||
try {
|
||||
const { timeRange } = req.params;
|
||||
const { channel } = req.query;
|
||||
|
||||
const reports = await reportingService.getCampaignReports({
|
||||
timeRange,
|
||||
channel
|
||||
});
|
||||
|
||||
res.json(reports);
|
||||
} catch (error) {
|
||||
console.error('[ReportingRoutes] Error fetching campaign reports:', error);
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
// Meta router factory — ESM conversion of meta-server/routes/campaigns.routes.js.
|
||||
// Phase 6.2: mutations (PATCH /campaigns/:id/budget, POST /campaigns/:id/:action)
|
||||
// require the `meta_write` permission. Reads (GET) stay authenticated-only.
|
||||
|
||||
import express from 'express';
|
||||
import { requirePermission } from '../../../shared/auth/middleware.js';
|
||||
import {
|
||||
fetchCampaigns,
|
||||
fetchAccountInsights,
|
||||
updateCampaignBudget,
|
||||
updateCampaignStatus,
|
||||
} from '../../services/meta/meta.service.js';
|
||||
|
||||
export function createMetaRouter() {
|
||||
const router = express.Router();
|
||||
|
||||
// Reads — authenticated-only
|
||||
router.get('/campaigns', async (req, res, next) => {
|
||||
try {
|
||||
const { since, until } = req.query;
|
||||
if (!since || !until) {
|
||||
return res.status(400).json({ error: 'Date range is required (since, until)' });
|
||||
}
|
||||
const campaigns = await fetchCampaigns(since, until);
|
||||
res.json(campaigns);
|
||||
} catch (error) {
|
||||
console.error('Campaign fetch error:', error);
|
||||
res.status(500).json({
|
||||
error: 'Failed to fetch campaigns',
|
||||
details: error.response?.data?.error?.message || error.message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/account-insights', async (req, res) => {
|
||||
try {
|
||||
const { since, until } = req.query;
|
||||
if (!since || !until) {
|
||||
return res.status(400).json({ error: 'Date range is required (since, until)' });
|
||||
}
|
||||
const insights = await fetchAccountInsights(since, until);
|
||||
res.json(insights);
|
||||
} catch (error) {
|
||||
console.error('Account insights fetch error:', error);
|
||||
res.status(500).json({
|
||||
error: 'Failed to fetch account insights',
|
||||
details: error.response?.data?.error?.message || error.message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Writes — meta_write
|
||||
router.patch('/campaigns/:campaignId/budget', requirePermission('meta_write'), async (req, res) => {
|
||||
try {
|
||||
const { campaignId } = req.params;
|
||||
const { budget } = req.body;
|
||||
if (!budget) {
|
||||
return res.status(400).json({ error: 'Budget is required' });
|
||||
}
|
||||
const result = await updateCampaignBudget(campaignId, budget);
|
||||
res.json(result);
|
||||
} catch (error) {
|
||||
console.error('Budget update error:', error);
|
||||
res.status(500).json({
|
||||
error: 'Failed to update campaign budget',
|
||||
details: error.response?.data?.error?.message || error.message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/campaigns/:campaignId/:action', requirePermission('meta_write'), async (req, res) => {
|
||||
try {
|
||||
const { campaignId, action } = req.params;
|
||||
if (!['pause', 'unpause'].includes(action)) {
|
||||
return res.status(400).json({ error: 'Invalid action. Use "pause" or "unpause"' });
|
||||
}
|
||||
const result = await updateCampaignStatus(campaignId, action);
|
||||
res.json(result);
|
||||
} catch (error) {
|
||||
console.error('Status update error:', error);
|
||||
res.status(500).json({
|
||||
error: 'Failed to update campaign status',
|
||||
details: error.response?.data?.error?.message || error.message,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
// Typeform router — ESM conversion of typeform-server/routes/typeform.routes.js.
|
||||
// All routes read-only — authenticated-only is sufficient; typeform_write reserved
|
||||
// for future write endpoints (per migration 005).
|
||||
|
||||
import express from 'express';
|
||||
import { TypeformService } from '../../services/typeform/typeform.service.js';
|
||||
|
||||
export function createTypeformRouter({ redis }) {
|
||||
const router = express.Router();
|
||||
const typeform = new TypeformService(redis);
|
||||
|
||||
router.get('/forms/:formId/responses', async (req, res) => {
|
||||
try {
|
||||
const { formId } = req.params;
|
||||
const filters = req.query;
|
||||
if (!formId) {
|
||||
return res.status(400).json({ error: 'Missing form ID', details: 'The form ID parameter is required' });
|
||||
}
|
||||
const data = await typeform.getFormResponsesWithFilters(formId, filters);
|
||||
if (!data) {
|
||||
return res.status(404).json({ error: 'No data found', details: `No responses found for form ${formId}` });
|
||||
}
|
||||
res.json(data);
|
||||
} catch (error) {
|
||||
console.error('Form responses error:', {
|
||||
formId: req.params.formId,
|
||||
filters: req.query,
|
||||
error: error.message,
|
||||
response: error.response?.data,
|
||||
});
|
||||
if (error.response?.status === 401) {
|
||||
return res.status(401).json({ error: 'Authentication failed', details: 'Invalid Typeform API credentials' });
|
||||
}
|
||||
if (error.response?.status === 404) {
|
||||
return res.status(404).json({ error: 'Not found', details: `Form '${req.params.formId}' not found` });
|
||||
}
|
||||
if (error.response?.status === 400) {
|
||||
return res.status(400).json({
|
||||
error: 'Invalid request',
|
||||
details: error.response?.data?.message || 'The request was invalid',
|
||||
data: error.response?.data,
|
||||
});
|
||||
}
|
||||
res.status(500).json({
|
||||
error: 'Failed to fetch form responses',
|
||||
details: error.response?.data?.message || error.message,
|
||||
data: error.response?.data,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/forms/:formId/insights', async (req, res) => {
|
||||
try {
|
||||
const { formId } = req.params;
|
||||
if (!formId) {
|
||||
return res.status(400).json({ error: 'Missing form ID', details: 'The form ID parameter is required' });
|
||||
}
|
||||
const data = await typeform.getFormInsights(formId);
|
||||
if (!data) {
|
||||
return res.status(404).json({ error: 'No data found', details: `No insights found for form ${formId}` });
|
||||
}
|
||||
res.json(data);
|
||||
} catch (error) {
|
||||
console.error('Form insights error:', {
|
||||
formId: req.params.formId,
|
||||
error: error.message,
|
||||
response: error.response?.data,
|
||||
});
|
||||
if (error.response?.status === 401) {
|
||||
return res.status(401).json({ error: 'Authentication failed', details: 'Invalid Typeform API credentials' });
|
||||
}
|
||||
if (error.response?.status === 404) {
|
||||
return res.status(404).json({ error: 'Not found', details: `Form '${req.params.formId}' not found` });
|
||||
}
|
||||
res.status(500).json({
|
||||
error: 'Failed to fetch form insights',
|
||||
details: error.response?.data?.message || error.message,
|
||||
data: error.response?.data,
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
-- Stores individual product links found in Klaviyo campaign emails
|
||||
CREATE TABLE IF NOT EXISTS klaviyo_campaign_products (
|
||||
id SERIAL PRIMARY KEY,
|
||||
campaign_id TEXT NOT NULL,
|
||||
campaign_name TEXT,
|
||||
sent_at TIMESTAMPTZ,
|
||||
pid BIGINT NOT NULL,
|
||||
product_url TEXT,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(campaign_id, pid)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kcp_campaign_id ON klaviyo_campaign_products(campaign_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_kcp_pid ON klaviyo_campaign_products(pid);
|
||||
CREATE INDEX IF NOT EXISTS idx_kcp_sent_at ON klaviyo_campaign_products(sent_at);
|
||||
|
||||
-- Stores non-product shop links (categories, filters, etc.) found in campaigns
|
||||
CREATE TABLE IF NOT EXISTS klaviyo_campaign_links (
|
||||
id SERIAL PRIMARY KEY,
|
||||
campaign_id TEXT NOT NULL,
|
||||
campaign_name TEXT,
|
||||
sent_at TIMESTAMPTZ,
|
||||
link_url TEXT NOT NULL,
|
||||
link_type TEXT, -- 'category', 'brand', 'filter', 'clearance', 'deals', 'other'
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(campaign_id, link_url)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_kcl_campaign_id ON klaviyo_campaign_links(campaign_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_kcl_sent_at ON klaviyo_campaign_links(sent_at);
|
||||
@@ -0,0 +1,279 @@
|
||||
/**
|
||||
* Extract products featured in Klaviyo campaign emails and store in DB.
|
||||
*
|
||||
* - Fetches recent sent campaigns from Klaviyo API
|
||||
* - Gets template HTML for each campaign message
|
||||
* - Parses out product links (/shop/{id}) and other shop links
|
||||
* - Inserts into klaviyo_campaign_products and klaviyo_campaign_links tables
|
||||
*
|
||||
* Usage: node scripts/poc-campaign-products.js [limit] [offset]
|
||||
* limit: number of sent campaigns to process (default: 10)
|
||||
* offset: number of sent campaigns to skip before processing (default: 0)
|
||||
*
|
||||
* Requires DB_* env vars (from inventory-server .env) and KLAVIYO_API_KEY.
|
||||
*/
|
||||
|
||||
import fetch from 'node-fetch';
|
||||
import pg from 'pg';
|
||||
import dotenv from 'dotenv';
|
||||
import path from 'path';
|
||||
import fs from 'fs';
|
||||
import { fileURLToPath } from 'url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
// Load klaviyo .env for API key
|
||||
dotenv.config({ path: path.resolve(__dirname, '../.env') });
|
||||
// Also load the main inventory-server .env for DB credentials
|
||||
const mainEnvPath = '/var/www/inventory/.env';
|
||||
if (fs.existsSync(mainEnvPath)) {
|
||||
dotenv.config({ path: mainEnvPath });
|
||||
}
|
||||
|
||||
const API_KEY = process.env.KLAVIYO_API_KEY;
|
||||
const REVISION = process.env.KLAVIYO_API_REVISION || '2026-01-15';
|
||||
const BASE_URL = 'https://a.klaviyo.com/api';
|
||||
|
||||
if (!API_KEY) {
|
||||
console.error('KLAVIYO_API_KEY not set in .env');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// ── Klaviyo API helpers ──────────────────────────────────────────────
|
||||
|
||||
const headers = {
|
||||
'Accept': 'application/json',
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Klaviyo-API-Key ${API_KEY}`,
|
||||
'revision': REVISION,
|
||||
};
|
||||
|
||||
async function klaviyoGet(endpoint, params = {}) {
|
||||
const url = new URL(`${BASE_URL}${endpoint}`);
|
||||
for (const [k, v] of Object.entries(params)) {
|
||||
url.searchParams.append(k, v);
|
||||
}
|
||||
return klaviyoFetch(url.toString());
|
||||
}
|
||||
|
||||
async function klaviyoFetch(url) {
|
||||
const res = await fetch(url, { headers });
|
||||
if (!res.ok) {
|
||||
const body = await res.text();
|
||||
throw new Error(`Klaviyo ${res.status} on ${url}: ${body}`);
|
||||
}
|
||||
return res.json();
|
||||
}
|
||||
|
||||
async function getRecentCampaigns(limit, offset = 0) {
|
||||
const campaigns = [];
|
||||
const messageMap = {};
|
||||
let skipped = 0;
|
||||
|
||||
let data = await klaviyoGet('/campaigns', {
|
||||
'filter': 'equals(messages.channel,"email")',
|
||||
'sort': '-scheduled_at',
|
||||
'include': 'campaign-messages',
|
||||
});
|
||||
|
||||
while (true) {
|
||||
for (const c of (data.data || [])) {
|
||||
if (c.attributes?.status === 'Sent') {
|
||||
if (skipped < offset) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
campaigns.push(c);
|
||||
if (campaigns.length >= limit) break;
|
||||
}
|
||||
}
|
||||
|
||||
for (const inc of (data.included || [])) {
|
||||
if (inc.type === 'campaign-message') {
|
||||
messageMap[inc.id] = inc;
|
||||
}
|
||||
}
|
||||
|
||||
const nextUrl = data.links?.next;
|
||||
if (campaigns.length >= limit || !nextUrl) break;
|
||||
|
||||
const progress = skipped < offset
|
||||
? `Skipped ${skipped}/${offset}...`
|
||||
: `Fetched ${campaigns.length}/${limit} sent campaigns, loading next page...`;
|
||||
console.log(` ${progress}`);
|
||||
await new Promise(r => setTimeout(r, 200));
|
||||
data = await klaviyoFetch(nextUrl);
|
||||
}
|
||||
|
||||
return { campaigns: campaigns.slice(0, limit), messageMap };
|
||||
}
|
||||
|
||||
async function getTemplateHtml(messageId) {
|
||||
const data = await klaviyoGet(`/campaign-messages/${messageId}/template`, {
|
||||
'fields[template]': 'html,name',
|
||||
});
|
||||
return {
|
||||
templateId: data.data?.id,
|
||||
templateName: data.data?.attributes?.name,
|
||||
html: data.data?.attributes?.html || '',
|
||||
};
|
||||
}
|
||||
|
||||
// ── HTML parsing ─────────────────────────────────────────────────────
|
||||
|
||||
function parseProductsFromHtml(html) {
|
||||
const seen = new Set();
|
||||
const products = [];
|
||||
|
||||
const linkRegex = /href="([^"]*acherryontop\.com\/shop\/(\d+))[^"]*"/gi;
|
||||
let match;
|
||||
while ((match = linkRegex.exec(html)) !== null) {
|
||||
const productId = match[2];
|
||||
if (!seen.has(productId)) {
|
||||
seen.add(productId);
|
||||
products.push({
|
||||
siteProductId: productId,
|
||||
url: match[1],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const categoryLinks = [];
|
||||
const catRegex = /href="([^"]*acherryontop\.com\/shop\/[^"]+)"/gi;
|
||||
while ((match = catRegex.exec(html)) !== null) {
|
||||
const url = match[1];
|
||||
if (/\/shop\/\d+$/.test(url)) continue;
|
||||
if (!categoryLinks.includes(url)) categoryLinks.push(url);
|
||||
}
|
||||
|
||||
return { products, categoryLinks };
|
||||
}
|
||||
|
||||
function classifyLink(url) {
|
||||
if (/\/shop\/(new|pre-order|backinstock)/.test(url)) return 'filter';
|
||||
if (/\/shop\/company\//.test(url)) return 'brand';
|
||||
if (/\/shop\/clearance/.test(url)) return 'clearance';
|
||||
if (/\/shop\/daily_deals/.test(url)) return 'deals';
|
||||
if (/\/shop\/category\//.test(url)) return 'category';
|
||||
return 'other';
|
||||
}
|
||||
|
||||
// ── Database ─────────────────────────────────────────────────────────
|
||||
|
||||
function createPool() {
|
||||
return new pg.Pool({
|
||||
host: process.env.DB_HOST,
|
||||
user: process.env.DB_USER,
|
||||
password: process.env.DB_PASSWORD,
|
||||
database: process.env.DB_NAME,
|
||||
port: process.env.DB_PORT || 5432,
|
||||
ssl: process.env.DB_SSL === 'true' ? { rejectUnauthorized: false } : false,
|
||||
});
|
||||
}
|
||||
|
||||
async function insertProducts(pool, campaignId, campaignName, sentAt, products) {
|
||||
if (products.length === 0) return 0;
|
||||
|
||||
let inserted = 0;
|
||||
for (const p of products) {
|
||||
try {
|
||||
await pool.query(
|
||||
`INSERT INTO klaviyo_campaign_products
|
||||
(campaign_id, campaign_name, sent_at, pid, product_url)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (campaign_id, pid) DO NOTHING`,
|
||||
[campaignId, campaignName, sentAt, parseInt(p.siteProductId), p.url]
|
||||
);
|
||||
inserted++;
|
||||
} catch (err) {
|
||||
console.error(` Error inserting product ${p.siteProductId}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
return inserted;
|
||||
}
|
||||
|
||||
async function insertLinks(pool, campaignId, campaignName, sentAt, links) {
|
||||
if (links.length === 0) return 0;
|
||||
|
||||
let inserted = 0;
|
||||
for (const url of links) {
|
||||
try {
|
||||
await pool.query(
|
||||
`INSERT INTO klaviyo_campaign_links
|
||||
(campaign_id, campaign_name, sent_at, link_url, link_type)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (campaign_id, link_url) DO NOTHING`,
|
||||
[campaignId, campaignName, sentAt, url, classifyLink(url)]
|
||||
);
|
||||
inserted++;
|
||||
} catch (err) {
|
||||
console.error(` Error inserting link: ${err.message}`);
|
||||
}
|
||||
}
|
||||
return inserted;
|
||||
}
|
||||
|
||||
// ── Main ─────────────────────────────────────────────────────────────
|
||||
|
||||
async function main() {
|
||||
const limit = parseInt(process.argv[2]) || 10;
|
||||
const offset = parseInt(process.argv[3]) || 0;
|
||||
const pool = createPool();
|
||||
|
||||
try {
|
||||
// Fetch campaigns
|
||||
console.log(`Fetching up to ${limit} recent campaigns (offset: ${offset})...\n`);
|
||||
const { campaigns, messageMap } = await getRecentCampaigns(limit, offset);
|
||||
console.log(`Found ${campaigns.length} sent campaigns.\n`);
|
||||
|
||||
let totalProducts = 0;
|
||||
let totalLinks = 0;
|
||||
|
||||
for (const campaign of campaigns) {
|
||||
const name = campaign.attributes?.name || 'Unnamed';
|
||||
const sentAt = campaign.attributes?.send_time;
|
||||
|
||||
console.log(`━━━ ${name} (${sentAt?.slice(0, 10) || 'no date'}) ━━━`);
|
||||
|
||||
const msgIds = (campaign.relationships?.['campaign-messages']?.data || [])
|
||||
.map(r => r.id);
|
||||
|
||||
if (msgIds.length === 0) {
|
||||
console.log(' No messages.\n');
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const msgId of msgIds) {
|
||||
const msg = messageMap[msgId];
|
||||
const subject = msg?.attributes?.definition?.content?.subject;
|
||||
if (subject) console.log(` Subject: ${subject}`);
|
||||
|
||||
try {
|
||||
const template = await getTemplateHtml(msgId);
|
||||
const { products, categoryLinks } = parseProductsFromHtml(template.html);
|
||||
|
||||
const pInserted = await insertProducts(pool, campaign.id, name, sentAt, products);
|
||||
const lInserted = await insertLinks(pool, campaign.id, name, sentAt, categoryLinks);
|
||||
|
||||
console.log(` ${products.length} products (${pInserted} new), ${categoryLinks.length} links (${lInserted} new)`);
|
||||
totalProducts += pInserted;
|
||||
totalLinks += lInserted;
|
||||
|
||||
await new Promise(r => setTimeout(r, 200));
|
||||
} catch (err) {
|
||||
console.log(` Error: ${err.message}`);
|
||||
}
|
||||
}
|
||||
console.log('');
|
||||
}
|
||||
|
||||
console.log(`Done. Inserted ${totalProducts} product rows, ${totalLinks} link rows.`);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('Fatal error:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,127 @@
|
||||
// dashboard-server — Phase 4 of CONSOLIDATION_PLAN.md.
|
||||
// Merges the four per-vendor PM2 apps (klaviyo, meta, google-analytics, typeform)
|
||||
// into a single ESM service on DASHBOARD_PORT (default 3015).
|
||||
//
|
||||
// Mount points (matches Caddy proxy paths):
|
||||
// /api/klaviyo/* → routes/klaviyo (was klaviyo-server :3004)
|
||||
// /api/meta/* → routes/meta (was meta-server :3005)
|
||||
// /api/dashboard-analytics/* → routes/google (was google-server :3007 via Caddy /api/analytics rewrite)
|
||||
// /api/typeform/* → routes/typeform (was typeform-server :3008)
|
||||
//
|
||||
// Shared infrastructure (Phase 2 + Phase 6):
|
||||
// - shared/auth/middleware.js authenticate() guards /api/* (Phase 6.1/6.2 — second line of defense)
|
||||
// - shared/cors/policy.js explicit allowed-origins list (Phase 6.6)
|
||||
// - shared/logging/request-log.js pino-http, Authorization/Cookie redacted (Phase 6.5/6.9)
|
||||
// - shared/errors/handler.js consistent error envelope, no leak in prod
|
||||
// - shared/db/pg.js / shared/db/redis.js one Pool + one ioredis client for all vendors
|
||||
//
|
||||
// Per-route permission gates (Phase 6.2):
|
||||
// - meta_write PATCH/POST mutations to Meta campaigns
|
||||
// - klaviyo_admin POST /api/klaviyo/events/clearCache (operational maintenance)
|
||||
// Read-only Google + Typeform endpoints stay authenticated-only.
|
||||
|
||||
import { config as loadEnv } from 'dotenv';
|
||||
import express from 'express';
|
||||
import cors from 'cors';
|
||||
import path from 'node:path';
|
||||
import fs from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { authenticate } from '../shared/auth/middleware.js';
|
||||
import { corsOptions } from '../shared/cors/policy.js';
|
||||
import { createPool } from '../shared/db/pg.js';
|
||||
import { createRedis } from '../shared/db/redis.js';
|
||||
import { errorHandler } from '../shared/errors/handler.js';
|
||||
import { logger } from '../shared/logging/logger.js';
|
||||
import { requestLog } from '../shared/logging/request-log.js';
|
||||
|
||||
import { createKlaviyoRouter } from './routes/klaviyo/index.js';
|
||||
import { createMetaRouter } from './routes/meta/index.js';
|
||||
import { createGoogleRouter } from './routes/google/index.js';
|
||||
import { createTypeformRouter } from './routes/typeform/index.js';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
// Layer envs: shared inventory .env wins on collisions (security-critical vars come
|
||||
// from one place); vendor-specific keys come from the per-service .env.
|
||||
//
|
||||
// dotenv defaults to override:false so the first file loaded wins. Order matters.
|
||||
const sharedEnvPath = '/var/www/inventory/.env';
|
||||
const dashboardEnvPath = path.resolve(__dirname, '.env');
|
||||
|
||||
if (fs.existsSync(sharedEnvPath)) loadEnv({ path: sharedEnvPath });
|
||||
if (fs.existsSync(dashboardEnvPath)) loadEnv({ path: dashboardEnvPath });
|
||||
|
||||
// Phase 6.4 — refuse to start without JWT_SECRET. Without it authenticate() falls
|
||||
// back to res.status(401) on every request and the service is useless anyway.
|
||||
if (!process.env.JWT_SECRET) {
|
||||
logger.error('JWT_SECRET is not set; refusing to start (per Phase 6.4)');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const app = express();
|
||||
const PORT = Number(process.env.DASHBOARD_PORT) || 3015;
|
||||
|
||||
// Single Postgres pool — used by authenticate() to load user permissions.
|
||||
// All four vendors share this pool (auth lookups are the only DB hits at runtime).
|
||||
const pool = createPool('DB');
|
||||
|
||||
// Single ioredis client shared across all vendors. lazyConnect:true means the
|
||||
// first .get/.set triggers the actual connect — keeps startup non-blocking even
|
||||
// if Redis is temporarily unavailable, and aligns with shared/db/redis.js defaults.
|
||||
const redis = createRedis();
|
||||
|
||||
app.use(requestLog());
|
||||
app.use(cors(corsOptions));
|
||||
app.use(express.json({ limit: '10mb' }));
|
||||
|
||||
// Phase 6.1/6.2: every /api request requires a valid JWT. authenticate() also
|
||||
// loads user permissions, which the per-route requirePermission() checks rely on.
|
||||
app.use('/api', authenticate({ pool, secret: process.env.JWT_SECRET }));
|
||||
|
||||
app.use('/api/klaviyo', createKlaviyoRouter({ redis }));
|
||||
app.use('/api/meta', createMetaRouter());
|
||||
// Note: frontend calls /api/dashboard-analytics (Caddy used to rewrite it to
|
||||
// /api/analytics for the standalone google-server). Mount at the public path so
|
||||
// Caddy can drop the rewrite — see Caddyfile.proposed.
|
||||
app.use('/api/dashboard-analytics', createGoogleRouter({ redis }));
|
||||
app.use('/api/typeform', createTypeformRouter({ redis }));
|
||||
|
||||
app.get('/health', (req, res) => {
|
||||
res.json({
|
||||
status: 'ok',
|
||||
timestamp: new Date().toISOString(),
|
||||
service: 'dashboard-server',
|
||||
redis: redis.status,
|
||||
});
|
||||
});
|
||||
|
||||
app.use(errorHandler);
|
||||
|
||||
// Connect Redis up front so the first request doesn't pay the connect cost.
|
||||
// Failures here are non-fatal — vendors degrade to cache-miss → upstream fetch.
|
||||
redis.connect().catch((err) => {
|
||||
logger.error({ err: { message: err.message, code: err.code } }, 'redis lazy-connect failed');
|
||||
});
|
||||
|
||||
const server = app.listen(PORT, '0.0.0.0', () => {
|
||||
logger.info({ port: PORT, mode: process.env.NODE_ENV || 'development' }, 'dashboard-server listening');
|
||||
});
|
||||
|
||||
const shutdown = async (signal) => {
|
||||
logger.info({ signal }, 'dashboard-server shutting down');
|
||||
server.close();
|
||||
try { await redis.quit(); } catch { /* ignore */ }
|
||||
try { await pool.end(); } catch { /* ignore */ }
|
||||
process.exit(0);
|
||||
};
|
||||
process.on('SIGTERM', () => shutdown('SIGTERM'));
|
||||
process.on('SIGINT', () => shutdown('SIGINT'));
|
||||
|
||||
process.on('uncaughtException', (err) => {
|
||||
logger.error({ err: { message: err.message, stack: err.stack } }, 'uncaughtException');
|
||||
process.exit(1);
|
||||
});
|
||||
process.on('unhandledRejection', (reason) => {
|
||||
logger.error({ reason }, 'unhandledRejection');
|
||||
});
|
||||
@@ -0,0 +1,195 @@
|
||||
// Google Analytics (GA4) service — ESM conversion of google-server/services/analytics.service.js.
|
||||
// Phase 4: accepts injected ioredis client (was self-constructing node-redis v4 before).
|
||||
// node-redis v4 set syntax `{ EX: 300 }` is translated to ioredis `setex(key, 300, val)`.
|
||||
|
||||
import { BetaAnalyticsDataClient } from '@google-analytics/data';
|
||||
|
||||
const CACHE_DURATIONS = {
|
||||
REALTIME_BASIC: 60,
|
||||
REALTIME_DETAILED: 300,
|
||||
BASIC_METRICS: 3600,
|
||||
USER_BEHAVIOR: 3600,
|
||||
};
|
||||
|
||||
export class AnalyticsService {
|
||||
constructor(redis) {
|
||||
if (!redis) {
|
||||
throw new Error('AnalyticsService requires an ioredis client (Phase 4: injected)');
|
||||
}
|
||||
this.redis = redis;
|
||||
|
||||
const credentials = process.env.GOOGLE_APPLICATION_CREDENTIALS_JSON;
|
||||
this.analyticsClient = new BetaAnalyticsDataClient({
|
||||
credentials: typeof credentials === 'string' ? JSON.parse(credentials) : credentials,
|
||||
});
|
||||
this.propertyId = process.env.GA_PROPERTY_ID;
|
||||
}
|
||||
|
||||
get _redisReady() {
|
||||
return this.redis.status === 'ready' || this.redis.status === 'connect';
|
||||
}
|
||||
|
||||
async _cacheGet(key) {
|
||||
if (!this._redisReady) return null;
|
||||
try {
|
||||
const raw = await this.redis.get(key);
|
||||
return raw ? JSON.parse(raw) : null;
|
||||
} catch (err) {
|
||||
console.warn('[AnalyticsService] cache get failed:', err.message);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async _cacheSet(key, value, ttlSec) {
|
||||
if (!this._redisReady) return;
|
||||
try {
|
||||
await this.redis.setex(key, ttlSec, JSON.stringify(value));
|
||||
} catch (err) {
|
||||
console.warn('[AnalyticsService] cache set failed:', err.message);
|
||||
}
|
||||
}
|
||||
|
||||
async getBasicMetrics(startDate = '7daysAgo') {
|
||||
const cacheKey = `analytics:basic_metrics:${startDate}`;
|
||||
const cached = await this._cacheGet(cacheKey);
|
||||
if (cached) return cached;
|
||||
|
||||
const [response] = await this.analyticsClient.runReport({
|
||||
property: `properties/${this.propertyId}`,
|
||||
dateRanges: [{ startDate, endDate: 'today' }],
|
||||
dimensions: [{ name: 'date' }],
|
||||
metrics: [
|
||||
{ name: 'activeUsers' },
|
||||
{ name: 'newUsers' },
|
||||
{ name: 'averageSessionDuration' },
|
||||
{ name: 'screenPageViews' },
|
||||
{ name: 'bounceRate' },
|
||||
{ name: 'conversions' },
|
||||
],
|
||||
returnPropertyQuota: true,
|
||||
});
|
||||
|
||||
await this._cacheSet(cacheKey, response, CACHE_DURATIONS.BASIC_METRICS);
|
||||
return response;
|
||||
}
|
||||
|
||||
async getRealTimeBasicData() {
|
||||
const cacheKey = 'analytics:realtime:basic';
|
||||
const cached = await this._cacheGet(cacheKey);
|
||||
if (cached) return cached;
|
||||
|
||||
const [userResponse] = await this.analyticsClient.runRealtimeReport({
|
||||
property: `properties/${this.propertyId}`,
|
||||
metrics: [{ name: 'activeUsers' }],
|
||||
returnPropertyQuota: true,
|
||||
});
|
||||
|
||||
const [fiveMinResponse] = await this.analyticsClient.runRealtimeReport({
|
||||
property: `properties/${this.propertyId}`,
|
||||
metrics: [{ name: 'activeUsers' }],
|
||||
minuteRanges: [{ startMinutesAgo: 5, endMinutesAgo: 0 }],
|
||||
});
|
||||
|
||||
const [timeSeriesResponse] = await this.analyticsClient.runRealtimeReport({
|
||||
property: `properties/${this.propertyId}`,
|
||||
dimensions: [{ name: 'minutesAgo' }],
|
||||
metrics: [{ name: 'activeUsers' }],
|
||||
});
|
||||
|
||||
const response = {
|
||||
userResponse,
|
||||
fiveMinResponse,
|
||||
timeSeriesResponse,
|
||||
quotaInfo: {
|
||||
projectHourly: userResponse.propertyQuota.tokensPerProjectPerHour,
|
||||
daily: userResponse.propertyQuota.tokensPerDay,
|
||||
serverErrors: userResponse.propertyQuota.serverErrorsPerProjectPerHour,
|
||||
thresholdedRequests: userResponse.propertyQuota.potentiallyThresholdedRequestsPerHour,
|
||||
},
|
||||
};
|
||||
|
||||
await this._cacheSet(cacheKey, response, CACHE_DURATIONS.REALTIME_BASIC);
|
||||
return response;
|
||||
}
|
||||
|
||||
async getRealTimeDetailedData() {
|
||||
const cacheKey = 'analytics:realtime:detailed';
|
||||
const cached = await this._cacheGet(cacheKey);
|
||||
if (cached) return cached;
|
||||
|
||||
const [pageResponse] = await this.analyticsClient.runRealtimeReport({
|
||||
property: `properties/${this.propertyId}`,
|
||||
dimensions: [{ name: 'unifiedScreenName' }],
|
||||
metrics: [{ name: 'screenPageViews' }],
|
||||
orderBy: [{ metric: { metricName: 'screenPageViews' }, desc: true }],
|
||||
limit: 25,
|
||||
});
|
||||
|
||||
const [eventResponse] = await this.analyticsClient.runRealtimeReport({
|
||||
property: `properties/${this.propertyId}`,
|
||||
dimensions: [{ name: 'eventName' }],
|
||||
metrics: [{ name: 'eventCount' }],
|
||||
orderBy: [{ metric: { metricName: 'eventCount' }, desc: true }],
|
||||
limit: 25,
|
||||
});
|
||||
|
||||
const [deviceResponse] = await this.analyticsClient.runRealtimeReport({
|
||||
property: `properties/${this.propertyId}`,
|
||||
dimensions: [{ name: 'deviceCategory' }],
|
||||
metrics: [{ name: 'activeUsers' }],
|
||||
orderBy: [{ metric: { metricName: 'activeUsers' }, desc: true }],
|
||||
limit: 10,
|
||||
returnPropertyQuota: true,
|
||||
});
|
||||
|
||||
const response = {
|
||||
pageResponse,
|
||||
eventResponse,
|
||||
sourceResponse: deviceResponse,
|
||||
};
|
||||
|
||||
await this._cacheSet(cacheKey, response, CACHE_DURATIONS.REALTIME_DETAILED);
|
||||
return response;
|
||||
}
|
||||
|
||||
async getUserBehavior(timeRange = '30') {
|
||||
const cacheKey = `analytics:user_behavior:${timeRange}`;
|
||||
const cached = await this._cacheGet(cacheKey);
|
||||
if (cached) return cached;
|
||||
|
||||
const [pageResponse] = await this.analyticsClient.runReport({
|
||||
property: `properties/${this.propertyId}`,
|
||||
dateRanges: [{ startDate: `${timeRange}daysAgo`, endDate: 'today' }],
|
||||
dimensions: [{ name: 'pagePath' }],
|
||||
metrics: [
|
||||
{ name: 'screenPageViews' },
|
||||
{ name: 'averageSessionDuration' },
|
||||
{ name: 'bounceRate' },
|
||||
{ name: 'sessions' },
|
||||
],
|
||||
orderBy: [{ metric: { metricName: 'screenPageViews' }, desc: true }],
|
||||
limit: 25,
|
||||
});
|
||||
|
||||
const [deviceResponse] = await this.analyticsClient.runReport({
|
||||
property: `properties/${this.propertyId}`,
|
||||
dateRanges: [{ startDate: `${timeRange}daysAgo`, endDate: 'today' }],
|
||||
dimensions: [{ name: 'deviceCategory' }],
|
||||
metrics: [{ name: 'screenPageViews' }, { name: 'sessions' }],
|
||||
});
|
||||
|
||||
const [sourceResponse] = await this.analyticsClient.runReport({
|
||||
property: `properties/${this.propertyId}`,
|
||||
dateRanges: [{ startDate: `${timeRange}daysAgo`, endDate: 'today' }],
|
||||
dimensions: [{ name: 'sessionSource' }],
|
||||
metrics: [{ name: 'sessions' }, { name: 'conversions' }],
|
||||
orderBy: [{ metric: { metricName: 'sessions' }, desc: true }],
|
||||
limit: 25,
|
||||
returnPropertyQuota: true,
|
||||
});
|
||||
|
||||
const response = { pageResponse, deviceResponse, sourceResponse };
|
||||
await this._cacheSet(cacheKey, response, CACHE_DURATIONS.USER_BEHAVIOR);
|
||||
return response;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
import fetch from 'node-fetch';
|
||||
import { TimeManager } from '../../utils/time.utils.js';
|
||||
import { RedisService } from './redis.service.js';
|
||||
|
||||
export class CampaignsService {
|
||||
constructor(apiKey, apiRevision, redis) {
|
||||
this.apiKey = apiKey;
|
||||
this.apiRevision = apiRevision;
|
||||
this.baseUrl = 'https://a.klaviyo.com/api';
|
||||
this.timeManager = new TimeManager();
|
||||
this.redisService = new RedisService(redis);
|
||||
}
|
||||
|
||||
async getCampaigns(params = {}) {
|
||||
try {
|
||||
// Add request debouncing
|
||||
const requestKey = JSON.stringify(params);
|
||||
if (this._pendingRequests && this._pendingRequests[requestKey]) {
|
||||
return this._pendingRequests[requestKey];
|
||||
}
|
||||
|
||||
// Try to get from cache first
|
||||
const cacheKey = this.redisService._getCacheKey('campaigns', params);
|
||||
let cachedData = null;
|
||||
try {
|
||||
cachedData = await this.redisService.get(`${cacheKey}:raw`);
|
||||
if (cachedData) {
|
||||
return cachedData;
|
||||
}
|
||||
} catch (cacheError) {
|
||||
console.warn('[CampaignsService] Cache error:', cacheError);
|
||||
}
|
||||
|
||||
this._pendingRequests = this._pendingRequests || {};
|
||||
this._pendingRequests[requestKey] = (async () => {
|
||||
let allCampaigns = [];
|
||||
let nextCursor = params.pageCursor;
|
||||
let pageCount = 0;
|
||||
|
||||
const filter = params.filter || this._buildFilter(params);
|
||||
|
||||
do {
|
||||
const queryParams = new URLSearchParams();
|
||||
if (filter) {
|
||||
queryParams.append('filter', filter);
|
||||
}
|
||||
queryParams.append('sort', params.sort || '-send_time');
|
||||
|
||||
if (nextCursor) {
|
||||
queryParams.append('page[cursor]', nextCursor);
|
||||
}
|
||||
|
||||
const url = `${this.baseUrl}/campaigns?${queryParams.toString()}`;
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
'Accept': 'application/json',
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Klaviyo-API-Key ${this.apiKey}`,
|
||||
'revision': this.apiRevision
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.json();
|
||||
console.error('[CampaignsService] API Error:', errorData);
|
||||
throw new Error(`Klaviyo API error: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const responseData = await response.json();
|
||||
allCampaigns = allCampaigns.concat(responseData.data || []);
|
||||
pageCount++;
|
||||
|
||||
nextCursor = responseData.links?.next ?
|
||||
new URL(responseData.links.next).searchParams.get('page[cursor]') : null;
|
||||
|
||||
if (nextCursor) {
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
}
|
||||
} catch (fetchError) {
|
||||
console.error('[CampaignsService] Fetch error:', fetchError);
|
||||
throw fetchError;
|
||||
}
|
||||
|
||||
} while (nextCursor);
|
||||
|
||||
const transformedCampaigns = this._transformCampaigns(allCampaigns);
|
||||
|
||||
const result = {
|
||||
data: transformedCampaigns,
|
||||
meta: {
|
||||
total_count: transformedCampaigns.length,
|
||||
page_count: pageCount
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
const ttl = this.redisService._getTTL(params.timeRange);
|
||||
await this.redisService.set(`${cacheKey}:raw`, result, ttl);
|
||||
} catch (cacheError) {
|
||||
console.warn('[CampaignsService] Cache set error:', cacheError);
|
||||
}
|
||||
|
||||
delete this._pendingRequests[requestKey];
|
||||
return result;
|
||||
})();
|
||||
|
||||
return await this._pendingRequests[requestKey];
|
||||
} catch (error) {
|
||||
console.error('[CampaignsService] Error fetching campaigns:', error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
_buildFilter(params) {
|
||||
const filters = [];
|
||||
|
||||
if (params.startDate && params.endDate) {
|
||||
const startUtc = this.timeManager.formatForAPI(params.startDate);
|
||||
const endUtc = this.timeManager.formatForAPI(params.endDate);
|
||||
|
||||
filters.push(`greater-or-equal(send_time,${startUtc})`);
|
||||
filters.push(`less-than(send_time,${endUtc})`);
|
||||
}
|
||||
|
||||
if (params.status) {
|
||||
filters.push(`equals(status,"${params.status}")`);
|
||||
}
|
||||
|
||||
if (params.customFilters) {
|
||||
filters.push(...params.customFilters);
|
||||
}
|
||||
|
||||
return filters.length > 0 ? (filters.length > 1 ? `and(${filters.join(',')})` : filters[0]) : null;
|
||||
}
|
||||
|
||||
async getCampaignsByTimeRange(timeRange, options = {}) {
|
||||
const range = this.timeManager.getDateRange(timeRange);
|
||||
if (!range) {
|
||||
throw new Error('Invalid time range specified');
|
||||
}
|
||||
|
||||
const params = {
|
||||
timeRange,
|
||||
startDate: range.start.toISO(),
|
||||
endDate: range.end.toISO(),
|
||||
...options
|
||||
};
|
||||
|
||||
// Try to get from cache first
|
||||
const cacheKey = this.redisService._getCacheKey('campaigns', params);
|
||||
let cachedData = null;
|
||||
try {
|
||||
cachedData = await this.redisService.get(`${cacheKey}:raw`);
|
||||
if (cachedData) {
|
||||
return cachedData;
|
||||
}
|
||||
} catch (cacheError) {
|
||||
console.warn('[CampaignsService] Cache error:', cacheError);
|
||||
}
|
||||
|
||||
return this.getCampaigns(params);
|
||||
}
|
||||
|
||||
_transformCampaigns(campaigns) {
|
||||
if (!Array.isArray(campaigns)) {
|
||||
console.warn('[CampaignsService] Campaigns is not an array:', campaigns);
|
||||
return [];
|
||||
}
|
||||
|
||||
return campaigns.map(campaign => {
|
||||
try {
|
||||
const stats = campaign.attributes?.campaign_message?.stats || {};
|
||||
|
||||
return {
|
||||
id: campaign.id,
|
||||
name: campaign.attributes?.name || "Unnamed Campaign",
|
||||
subject: campaign.attributes?.campaign_message?.subject || "",
|
||||
send_time: campaign.attributes?.send_time,
|
||||
stats: {
|
||||
delivery_rate: stats.delivery_rate || 0,
|
||||
delivered: stats.delivered || 0,
|
||||
recipients: stats.recipients || 0,
|
||||
open_rate: stats.open_rate || 0,
|
||||
opens_unique: stats.opens_unique || 0,
|
||||
opens: stats.opens || 0,
|
||||
clicks_unique: stats.clicks_unique || 0,
|
||||
click_rate: stats.click_rate || 0,
|
||||
click_to_open_rate: stats.click_to_open_rate || 0,
|
||||
conversion_value: stats.conversion_value || 0,
|
||||
conversion_uniques: stats.conversion_uniques || 0
|
||||
}
|
||||
};
|
||||
} catch (error) {
|
||||
console.error('[CampaignsService] Error transforming campaign:', error, campaign);
|
||||
return {
|
||||
id: campaign.id || 'unknown',
|
||||
name: 'Error Processing Campaign',
|
||||
stats: {}
|
||||
};
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user