matt/inventory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: matt:add-company-specific-ai-prompts
Branches Tags
matt:master matt:merge-dashboard matt:fix-numbers matt:check-numbers matt:move-to-postgresql matt:add-company-specific-ai-prompts matt:add-permissions matt:add-product-upload-page matt:Improve-calcs matt:Improve-data-import-and-calcs
..
compare: matt:d936d50f831a45c66a8c1e6b55b3193bf50cc99a
Branches Tags
matt:master matt:merge-dashboard matt:fix-numbers matt:check-numbers matt:move-to-postgresql matt:add-company-specific-ai-prompts matt:add-permissions matt:add-product-upload-page matt:Improve-calcs matt:Improve-data-import-and-calcs

4 Commits
add-compan ... d936d50f83

Author SHA1 Message Date
Matt
d936d50f83 Vendor calculate script fix 2025-02-10 09:26:24 -05:00
Matt
610e26689c Try to speed up calculate script + fixes 2025-02-10 01:29:01 -05:00
Matt
7ff757203f Calculate script fixes 2025-02-09 15:40:57 -05:00
Matt
843ce71506 Make calculations incremental 2025-02-09 13:35:44 -05:00
224 changed files with 6830 additions and 38456 deletions
Expand all files Collapse all files

41
.VSCodeCounter/2025-03-17_16-24-17/details.md
View File

@@ -1,41 +0,0 @@
# Details
Date : 2025-03-17 16:24:17
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
Total : 26 files, 6193 codes, 1008 comments, 1017 blanks, all 8218 lines
[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
## Files
| filename | language | code | comment | blank | total |
| :--- | :--- | ---: | ---: | ---: | ---: |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md) | Markdown | 39 | 0 | 19 | 58 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx) | TypeScript JSX | 230 | 10 | 8 | 248 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx) | TypeScript JSX | 18 | 0 | 3 | 21 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx) | TypeScript JSX | 273 | 19 | 37 | 329 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx) | TypeScript JSX | 374 | 42 | 44 | 460 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx) | TypeScript JSX | 730 | 126 | 106 | 962 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx) | TypeScript JSX | 499 | 48 | 54 | 601 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx) | TypeScript JSX | 112 | 12 | 21 | 145 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx) | TypeScript JSX | 232 | 31 | 32 | 295 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx) | TypeScript JSX | 407 | 56 | 52 | 515 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx) | TypeScript JSX | 193 | 23 | 22 | 238 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx) | TypeScript JSX | 289 | 36 | 31 | 356 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx) | TypeScript JSX | 500 | 75 | 89 | 664 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx) | TypeScript JSX | 248 | 69 | 74 | 391 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx) | TypeScript JSX | 204 | 26 | 33 | 263 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx) | TypeScript JSX | 209 | 49 | 50 | 308 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx) | TypeScript JSX | 219 | 39 | 47 | 305 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx) | TypeScript JSX | 1,060 | 228 | 229 | 1,517 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx) | TypeScript JSX | 20 | 6 | 2 | 28 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts) | TypeScript | 4 | 0 | 1 | 5 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts) | TypeScript | 16 | 4 | 4 | 24 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts) | TypeScript | 124 | 4 | 14 | 142 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts) | TypeScript | 21 | 15 | 5 | 41 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts) | TypeScript | 43 | 24 | 7 | 74 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js) | JavaScript | 28 | 7 | 9 | 44 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts) | TypeScript | 101 | 59 | 24 | 184 |
[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md)

20
.VSCodeCounter/2025-03-17_16-24-17/diff-details.md
View File

@@ -1,20 +0,0 @@
# Diff Details
Date : 2025-03-17 16:24:17
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
Total : 5 files, -358 codes, -15 comments, -33 blanks, all -406 lines
[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details
## Files
| filename | language | code | comment | blank | total |
| :--- | :--- | ---: | ---: | ---: | ---: |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SaveTemplateDialog.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SaveTemplateDialog.tsx) | TypeScript JSX | -83 | 0 | -4 | -87 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/TemplateManager.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/TemplateManager.tsx) | TypeScript JSX | -193 | -4 | -15 | -212 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx) | TypeScript JSX | -241 | -68 | -72 | -381 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useFilters.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useFilters.tsx) | TypeScript JSX | -89 | -12 | -16 | -117 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx) | TypeScript JSX | 248 | 69 | 74 | 391 |
[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details

23
.VSCodeCounter/2025-03-17_16-24-17/diff.md
View File

@@ -1,23 +0,0 @@
# Diff Summary
Date : 2025-03-17 16:24:17
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
Total : 5 files, -358 codes, -15 comments, -33 blanks, all -406 lines
[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md)
## Languages
| language | files | code | comment | blank | total |
| :--- | ---: | ---: | ---: | ---: | ---: |
| TypeScript JSX | 5 | -358 | -15 | -33 | -406 |
## Directories
| path | files | code | comment | blank | total |
| :--- | ---: | ---: | ---: | ---: | ---: |
| . | 5 | -358 | -15 | -33 | -406 |
| components | 3 | -517 | -72 | -91 | -680 |
| hooks | 2 | 159 | 57 | 58 | 274 |
[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md)

31
.VSCodeCounter/2025-03-17_16-24-17/diff.txt
View File

@@ -1,31 +0,0 @@
Date : 2025-03-17 16:24:17
Directory : /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
Total : 5 files, -358 codes, -15 comments, -33 blanks, all -406 lines
Languages
+----------------+------------+------------+------------+------------+------------+
| language | files | code | comment | blank | total |
+----------------+------------+------------+------------+------------+------------+
| TypeScript JSX | 5 | -358 | -15 | -33 | -406 |
+----------------+------------+------------+------------+------------+------------+
Directories
+-------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
| path | files | code | comment | blank | total |
+-------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
| . | 5 | -358 | -15 | -33 | -406 |
| components | 3 | -517 | -72 | -91 | -680 |
| hooks | 2 | 159 | 57 | 58 | 274 |
+-------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
Files
+-------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
| filename | language | code | comment | blank | total |
+-------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SaveTemplateDialog.tsx | TypeScript JSX | -83 | 0 | -4 | -87 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/TemplateManager.tsx | TypeScript JSX | -193 | -4 | -15 | -212 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx | TypeScript JSX | -241 | -68 | -72 | -381 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useFilters.tsx | TypeScript JSX | -89 | -12 | -16 | -117 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx | TypeScript JSX | 248 | 69 | 74 | 391 |
| Total | | -358 | -15 | -33 | -406 |
+-------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+

1
.VSCodeCounter/2025-03-17_16-24-17/results.json
View File

File diff suppressed because one or more lines are too long

31
.VSCodeCounter/2025-03-17_16-24-17/results.md
View File

@@ -1,31 +0,0 @@
# Summary
Date : 2025-03-17 16:24:17
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
Total : 26 files, 6193 codes, 1008 comments, 1017 blanks, all 8218 lines
Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
## Languages
| language | files | code | comment | blank | total |
| :--- | ---: | ---: | ---: | ---: | ---: |
| TypeScript JSX | 18 | 5,817 | 895 | 934 | 7,646 |
| TypeScript | 6 | 309 | 106 | 55 | 470 |
| Markdown | 1 | 39 | 0 | 19 | 58 |
| JavaScript | 1 | 28 | 7 | 9 | 44 |
## Directories
| path | files | code | comment | blank | total |
| :--- | ---: | ---: | ---: | ---: | ---: |
| . | 26 | 6,193 | 1,008 | 1,017 | 8,218 |
| . (Files) | 3 | 63 | 6 | 22 | 91 |
| components | 11 | 3,357 | 403 | 410 | 4,170 |
| components (Files) | 6 | 2,124 | 245 | 252 | 2,621 |
| components/cells | 5 | 1,233 | 158 | 158 | 1,549 |
| hooks | 6 | 2,440 | 486 | 522 | 3,448 |
| types | 1 | 16 | 4 | 4 | 24 |
| utils | 5 | 317 | 109 | 59 | 485 |
Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md)

60
.VSCodeCounter/2025-03-17_16-24-17/results.txt
View File

@@ -1,60 +0,0 @@
Date : 2025-03-17 16:24:17
Directory : /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
Total : 26 files, 6193 codes, 1008 comments, 1017 blanks, all 8218 lines
Languages
+----------------+------------+------------+------------+------------+------------+
| language | files | code | comment | blank | total |
+----------------+------------+------------+------------+------------+------------+
| TypeScript JSX | 18 | 5,817 | 895 | 934 | 7,646 |
| TypeScript | 6 | 309 | 106 | 55 | 470 |
| Markdown | 1 | 39 | 0 | 19 | 58 |
| JavaScript | 1 | 28 | 7 | 9 | 44 |
+----------------+------------+------------+------------+------------+------------+
Directories
+------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
| path | files | code | comment | blank | total |
+------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
| . | 26 | 6,193 | 1,008 | 1,017 | 8,218 |
| . (Files) | 3 | 63 | 6 | 22 | 91 |
| components | 11 | 3,357 | 403 | 410 | 4,170 |
| components (Files) | 6 | 2,124 | 245 | 252 | 2,621 |
| components/cells | 5 | 1,233 | 158 | 158 | 1,549 |
| hooks | 6 | 2,440 | 486 | 522 | 3,448 |
| types | 1 | 16 | 4 | 4 | 24 |
| utils | 5 | 317 | 109 | 59 | 485 |
+------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
Files
+------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
| filename | language | code | comment | blank | total |
+------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md | Markdown | 39 | 0 | 19 | 58 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx | TypeScript JSX | 230 | 10 | 8 | 248 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx | TypeScript JSX | 18 | 0 | 3 | 21 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx | TypeScript JSX | 273 | 19 | 37 | 329 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx | TypeScript JSX | 374 | 42 | 44 | 460 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx | TypeScript JSX | 730 | 126 | 106 | 962 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx | TypeScript JSX | 499 | 48 | 54 | 601 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx | TypeScript JSX | 112 | 12 | 21 | 145 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx | TypeScript JSX | 232 | 31 | 32 | 295 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx | TypeScript JSX | 407 | 56 | 52 | 515 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx | TypeScript JSX | 193 | 23 | 22 | 238 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx | TypeScript JSX | 289 | 36 | 31 | 356 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx | TypeScript JSX | 500 | 75 | 89 | 664 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx | TypeScript JSX | 248 | 69 | 74 | 391 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx | TypeScript JSX | 204 | 26 | 33 | 263 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx | TypeScript JSX | 209 | 49 | 50 | 308 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx | TypeScript JSX | 219 | 39 | 47 | 305 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx | TypeScript JSX | 1,060 | 228 | 229 | 1,517 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx | TypeScript JSX | 20 | 6 | 2 | 28 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts | TypeScript | 4 | 0 | 1 | 5 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts | TypeScript | 16 | 4 | 4 | 24 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts | TypeScript | 124 | 4 | 14 | 142 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts | TypeScript | 21 | 15 | 5 | 41 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts | TypeScript | 43 | 24 | 7 | 74 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js | JavaScript | 28 | 7 | 9 | 44 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts | TypeScript | 101 | 59 | 24 | 184 |
| Total | | 6,193 | 1,008 | 1,017 | 8,218 |
+------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+

42
.VSCodeCounter/2025-03-18_12-39-04/details.md
View File

@@ -1,42 +0,0 @@
# Details
Date : 2025-03-18 12:39:04
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
Total : 27 files, 6925 codes, 1247 comments, 1248 blanks, all 9420 lines
[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
## Files
| filename | language | code | comment | blank | total |
| :--- | :--- | ---: | ---: | ---: | ---: |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md) | Markdown | 39 | 0 | 19 | 58 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx) | TypeScript JSX | 230 | 10 | 8 | 248 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx) | TypeScript JSX | 18 | 0 | 3 | 21 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx) | TypeScript JSX | 273 | 19 | 37 | 329 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx) | TypeScript JSX | 113 | 17 | 10 | 140 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx) | TypeScript JSX | 377 | 49 | 54 | 480 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx) | TypeScript JSX | 969 | 182 | 158 | 1,309 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx) | TypeScript JSX | 509 | 50 | 57 | 616 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx) | TypeScript JSX | 112 | 12 | 21 | 145 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx) | TypeScript JSX | 233 | 34 | 33 | 300 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx) | TypeScript JSX | 420 | 66 | 59 | 545 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx) | TypeScript JSX | 193 | 23 | 22 | 238 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx) | TypeScript JSX | 227 | 36 | 32 | 295 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx) | TypeScript JSX | 500 | 75 | 89 | 664 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx) | TypeScript JSX | 264 | 75 | 81 | 420 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx) | TypeScript JSX | 204 | 26 | 33 | 263 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx) | TypeScript JSX | 337 | 88 | 92 | 517 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx) | TypeScript JSX | 360 | 78 | 85 | 523 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx) | TypeScript JSX | 1,190 | 288 | 289 | 1,767 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx) | TypeScript JSX | 20 | 6 | 2 | 28 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts) | TypeScript | 4 | 0 | 1 | 5 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts) | TypeScript | 16 | 4 | 4 | 24 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts) | TypeScript | 124 | 4 | 14 | 142 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts) | TypeScript | 21 | 15 | 5 | 41 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts) | TypeScript | 43 | 24 | 7 | 74 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js) | JavaScript | 28 | 7 | 9 | 44 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts) | TypeScript | 101 | 59 | 24 | 184 |
[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md)

26
.VSCodeCounter/2025-03-18_12-39-04/diff-details.md
View File

@@ -1,26 +0,0 @@
# Diff Details
Date : 2025-03-18 12:39:04
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
Total : 11 files, 732 codes, 239 comments, 231 blanks, all 1202 lines
[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details
## Files
| filename | language | code | comment | blank | total |
| :--- | :--- | ---: | ---: | ---: | ---: |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx) | TypeScript JSX | 113 | 17 | 10 | 140 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx) | TypeScript JSX | 3 | 7 | 10 | 20 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx) | TypeScript JSX | 239 | 56 | 52 | 347 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx) | TypeScript JSX | 10 | 2 | 3 | 15 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx) | TypeScript JSX | 1 | 3 | 1 | 5 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx) | TypeScript JSX | 13 | 10 | 7 | 30 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx) | TypeScript JSX | -62 | 0 | 1 | -61 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx) | TypeScript JSX | 16 | 6 | 7 | 29 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx) | TypeScript JSX | 128 | 39 | 42 | 209 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx) | TypeScript JSX | 141 | 39 | 38 | 218 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx) | TypeScript JSX | 130 | 60 | 60 | 250 |
[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details

25
.VSCodeCounter/2025-03-18_12-39-04/diff.md
View File

@@ -1,25 +0,0 @@
# Diff Summary
Date : 2025-03-18 12:39:04
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
Total : 11 files, 732 codes, 239 comments, 231 blanks, all 1202 lines
[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md)
## Languages
| language | files | code | comment | blank | total |
| :--- | ---: | ---: | ---: | ---: | ---: |
| TypeScript JSX | 11 | 732 | 239 | 231 | 1,202 |
## Directories
| path | files | code | comment | blank | total |
| :--- | ---: | ---: | ---: | ---: | ---: |
| . | 11 | 732 | 239 | 231 | 1,202 |
| components | 7 | 317 | 95 | 84 | 496 |
| components (Files) | 4 | 365 | 82 | 75 | 522 |
| components/cells | 3 | -48 | 13 | 9 | -26 |
| hooks | 4 | 415 | 144 | 147 | 706 |
[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md)

39
.VSCodeCounter/2025-03-18_12-39-04/diff.txt
View File

@@ -1,39 +0,0 @@
Date : 2025-03-18 12:39:04
Directory : /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
Total : 11 files, 732 codes, 239 comments, 231 blanks, all 1202 lines
Languages
+----------------+------------+------------+------------+------------+------------+
| language | files | code | comment | blank | total |
+----------------+------------+------------+------------+------------+------------+
| TypeScript JSX | 11 | 732 | 239 | 231 | 1,202 |
+----------------+------------+------------+------------+------------+------------+
Directories
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
| path | files | code | comment | blank | total |
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
| . | 11 | 732 | 239 | 231 | 1,202 |
| components | 7 | 317 | 95 | 84 | 496 |
| components (Files) | 4 | 365 | 82 | 75 | 522 |
| components/cells | 3 | -48 | 13 | 9 | -26 |
| hooks | 4 | 415 | 144 | 147 | 706 |
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
Files
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
| filename | language | code | comment | blank | total |
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx | TypeScript JSX | 113 | 17 | 10 | 140 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx | TypeScript JSX | 3 | 7 | 10 | 20 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx | TypeScript JSX | 239 | 56 | 52 | 347 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx | TypeScript JSX | 10 | 2 | 3 | 15 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx | TypeScript JSX | 1 | 3 | 1 | 5 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx | TypeScript JSX | 13 | 10 | 7 | 30 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx | TypeScript JSX | -62 | 0 | 1 | -61 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx | TypeScript JSX | 16 | 6 | 7 | 29 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx | TypeScript JSX | 128 | 39 | 42 | 209 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx | TypeScript JSX | 141 | 39 | 38 | 218 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx | TypeScript JSX | 130 | 60 | 60 | 250 |
| Total | | 732 | 239 | 231 | 1,202 |
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+

1
.VSCodeCounter/2025-03-18_12-39-04/results.json
View File

File diff suppressed because one or more lines are too long

31
.VSCodeCounter/2025-03-18_12-39-04/results.md
View File

@@ -1,31 +0,0 @@
# Summary
Date : 2025-03-18 12:39:04
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
Total : 27 files, 6925 codes, 1247 comments, 1248 blanks, all 9420 lines
Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
## Languages
| language | files | code | comment | blank | total |
| :--- | ---: | ---: | ---: | ---: | ---: |
| TypeScript JSX | 19 | 6,549 | 1,134 | 1,165 | 8,848 |
| TypeScript | 6 | 309 | 106 | 55 | 470 |
| Markdown | 1 | 39 | 0 | 19 | 58 |
| JavaScript | 1 | 28 | 7 | 9 | 44 |
## Directories
| path | files | code | comment | blank | total |
| :--- | ---: | ---: | ---: | ---: | ---: |
| . | 27 | 6,925 | 1,247 | 1,248 | 9,420 |
| . (Files) | 3 | 63 | 6 | 22 | 91 |
| components | 12 | 3,674 | 498 | 494 | 4,666 |
| components (Files) | 7 | 2,489 | 327 | 327 | 3,143 |
| components/cells | 5 | 1,185 | 171 | 167 | 1,523 |
| hooks | 6 | 2,855 | 630 | 669 | 4,154 |
| types | 1 | 16 | 4 | 4 | 24 |
| utils | 5 | 317 | 109 | 59 | 485 |
Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md)

61
.VSCodeCounter/2025-03-18_12-39-04/results.txt
View File

@@ -1,61 +0,0 @@
Date : 2025-03-18 12:39:04
Directory : /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
Total : 27 files, 6925 codes, 1247 comments, 1248 blanks, all 9420 lines
Languages
+----------------+------------+------------+------------+------------+------------+
| language | files | code | comment | blank | total |
+----------------+------------+------------+------------+------------+------------+
| TypeScript JSX | 19 | 6,549 | 1,134 | 1,165 | 8,848 |
| TypeScript | 6 | 309 | 106 | 55 | 470 |
| Markdown | 1 | 39 | 0 | 19 | 58 |
| JavaScript | 1 | 28 | 7 | 9 | 44 |
+----------------+------------+------------+------------+------------+------------+
Directories
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
| path | files | code | comment | blank | total |
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
| . | 27 | 6,925 | 1,247 | 1,248 | 9,420 |
| . (Files) | 3 | 63 | 6 | 22 | 91 |
| components | 12 | 3,674 | 498 | 494 | 4,666 |
| components (Files) | 7 | 2,489 | 327 | 327 | 3,143 |
| components/cells | 5 | 1,185 | 171 | 167 | 1,523 |
| hooks | 6 | 2,855 | 630 | 669 | 4,154 |
| types | 1 | 16 | 4 | 4 | 24 |
| utils | 5 | 317 | 109 | 59 | 485 |
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
Files
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
| filename | language | code | comment | blank | total |
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md | Markdown | 39 | 0 | 19 | 58 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx | TypeScript JSX | 230 | 10 | 8 | 248 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx | TypeScript JSX | 18 | 0 | 3 | 21 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx | TypeScript JSX | 273 | 19 | 37 | 329 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx | TypeScript JSX | 113 | 17 | 10 | 140 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx | TypeScript JSX | 377 | 49 | 54 | 480 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx | TypeScript JSX | 969 | 182 | 158 | 1,309 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx | TypeScript JSX | 509 | 50 | 57 | 616 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx | TypeScript JSX | 112 | 12 | 21 | 145 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx | TypeScript JSX | 233 | 34 | 33 | 300 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx | TypeScript JSX | 420 | 66 | 59 | 545 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx | TypeScript JSX | 193 | 23 | 22 | 238 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx | TypeScript JSX | 227 | 36 | 32 | 295 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx | TypeScript JSX | 500 | 75 | 89 | 664 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx | TypeScript JSX | 264 | 75 | 81 | 420 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx | TypeScript JSX | 204 | 26 | 33 | 263 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx | TypeScript JSX | 337 | 88 | 92 | 517 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx | TypeScript JSX | 360 | 78 | 85 | 523 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx | TypeScript JSX | 1,190 | 288 | 289 | 1,767 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx | TypeScript JSX | 20 | 6 | 2 | 28 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts | TypeScript | 4 | 0 | 1 | 5 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts | TypeScript | 16 | 4 | 4 | 24 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts | TypeScript | 124 | 4 | 14 | 142 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts | TypeScript | 21 | 15 | 5 | 41 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts | TypeScript | 43 | 24 | 7 | 74 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js | JavaScript | 28 | 7 | 9 | 44 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts | TypeScript | 101 | 59 | 24 | 184 |
| Total | | 6,925 | 1,247 | 1,248 | 9,420 |
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+

42
.VSCodeCounter/2025-03-18_13-49-23/details.md
View File

@@ -1,42 +0,0 @@
# Details
Date : 2025-03-18 13:49:23
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
Total : 27 files, 6961 codes, 1254 comments, 1252 blanks, all 9467 lines
[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
## Files
| filename | language | code | comment | blank | total |
| :--- | :--- | ---: | ---: | ---: | ---: |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md) | Markdown | 39 | 0 | 19 | 58 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx) | TypeScript JSX | 230 | 10 | 8 | 248 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx) | TypeScript JSX | 18 | 0 | 3 | 21 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx) | TypeScript JSX | 273 | 19 | 37 | 329 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx) | TypeScript JSX | 113 | 17 | 10 | 140 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx) | TypeScript JSX | 395 | 51 | 55 | 501 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx) | TypeScript JSX | 969 | 182 | 158 | 1,309 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx) | TypeScript JSX | 527 | 55 | 60 | 642 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx) | TypeScript JSX | 112 | 12 | 21 | 145 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx) | TypeScript JSX | 233 | 34 | 33 | 300 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx) | TypeScript JSX | 420 | 66 | 59 | 545 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx) | TypeScript JSX | 193 | 23 | 22 | 238 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx) | TypeScript JSX | 227 | 36 | 32 | 295 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx) | TypeScript JSX | 500 | 75 | 89 | 664 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx) | TypeScript JSX | 264 | 75 | 81 | 420 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx) | TypeScript JSX | 204 | 26 | 33 | 263 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx) | TypeScript JSX | 337 | 88 | 92 | 517 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx) | TypeScript JSX | 360 | 78 | 85 | 523 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx) | TypeScript JSX | 1,190 | 288 | 289 | 1,767 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx) | TypeScript JSX | 20 | 6 | 2 | 28 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts) | TypeScript | 4 | 0 | 1 | 5 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts) | TypeScript | 16 | 4 | 4 | 24 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts) | TypeScript | 124 | 4 | 14 | 142 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts) | TypeScript | 21 | 15 | 5 | 41 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts) | TypeScript | 43 | 24 | 7 | 74 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js) | JavaScript | 28 | 7 | 9 | 44 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts) | TypeScript | 101 | 59 | 24 | 184 |
[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md)

17
.VSCodeCounter/2025-03-18_13-49-23/diff-details.md
View File

@@ -1,17 +0,0 @@
# Diff Details
Date : 2025-03-18 13:49:23
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
Total : 2 files, 36 codes, 7 comments, 4 blanks, all 47 lines
[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details
## Files
| filename | language | code | comment | blank | total |
| :--- | :--- | ---: | ---: | ---: | ---: |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx) | TypeScript JSX | 18 | 2 | 1 | 21 |
| [inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx](/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx) | TypeScript JSX | 18 | 5 | 3 | 26 |
[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details

22
.VSCodeCounter/2025-03-18_13-49-23/diff.md
View File

@@ -1,22 +0,0 @@
# Diff Summary
Date : 2025-03-18 13:49:23
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
Total : 2 files, 36 codes, 7 comments, 4 blanks, all 47 lines
[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md)
## Languages
| language | files | code | comment | blank | total |
| :--- | ---: | ---: | ---: | ---: | ---: |
| TypeScript JSX | 2 | 36 | 7 | 4 | 47 |
## Directories
| path | files | code | comment | blank | total |
| :--- | ---: | ---: | ---: | ---: | ---: |
| . | 2 | 36 | 7 | 4 | 47 |
| components | 2 | 36 | 7 | 4 | 47 |
[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md)

27
.VSCodeCounter/2025-03-18_13-49-23/diff.txt
View File

@@ -1,27 +0,0 @@
Date : 2025-03-18 13:49:23
Directory : /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
Total : 2 files, 36 codes, 7 comments, 4 blanks, all 47 lines
Languages
+----------------+------------+------------+------------+------------+------------+
| language | files | code | comment | blank | total |
+----------------+------------+------------+------------+------------+------------+
| TypeScript JSX | 2 | 36 | 7 | 4 | 47 |
+----------------+------------+------------+------------+------------+------------+
Directories
+---------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
| path | files | code | comment | blank | total |
+---------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
| . | 2 | 36 | 7 | 4 | 47 |
| components | 2 | 36 | 7 | 4 | 47 |
+---------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
Files
+---------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
| filename | language | code | comment | blank | total |
+---------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx | TypeScript JSX | 18 | 2 | 1 | 21 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx | TypeScript JSX | 18 | 5 | 3 | 26 |
| Total | | 36 | 7 | 4 | 47 |
+---------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+

1
.VSCodeCounter/2025-03-18_13-49-23/results.json
View File

File diff suppressed because one or more lines are too long

31
.VSCodeCounter/2025-03-18_13-49-23/results.md
View File

@@ -1,31 +0,0 @@
# Summary
Date : 2025-03-18 13:49:23
Directory /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
Total : 27 files, 6961 codes, 1254 comments, 1252 blanks, all 9467 lines
Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md)
## Languages
| language | files | code | comment | blank | total |
| :--- | ---: | ---: | ---: | ---: | ---: |
| TypeScript JSX | 19 | 6,585 | 1,141 | 1,169 | 8,895 |
| TypeScript | 6 | 309 | 106 | 55 | 470 |
| Markdown | 1 | 39 | 0 | 19 | 58 |
| JavaScript | 1 | 28 | 7 | 9 | 44 |
## Directories
| path | files | code | comment | blank | total |
| :--- | ---: | ---: | ---: | ---: | ---: |
| . | 27 | 6,961 | 1,254 | 1,252 | 9,467 |
| . (Files) | 3 | 63 | 6 | 22 | 91 |
| components | 12 | 3,710 | 505 | 498 | 4,713 |
| components (Files) | 7 | 2,525 | 334 | 331 | 3,190 |
| components/cells | 5 | 1,185 | 171 | 167 | 1,523 |
| hooks | 6 | 2,855 | 630 | 669 | 4,154 |
| types | 1 | 16 | 4 | 4 | 24 |
| utils | 5 | 317 | 109 | 59 | 485 |
Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md)

61
.VSCodeCounter/2025-03-18_13-49-23/results.txt
View File

@@ -1,61 +0,0 @@
Date : 2025-03-18 13:49:23
Directory : /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew
Total : 27 files, 6961 codes, 1254 comments, 1252 blanks, all 9467 lines
Languages
+----------------+------------+------------+------------+------------+------------+
| language | files | code | comment | blank | total |
+----------------+------------+------------+------------+------------+------------+
| TypeScript JSX | 19 | 6,585 | 1,141 | 1,169 | 8,895 |
| TypeScript | 6 | 309 | 106 | 55 | 470 |
| Markdown | 1 | 39 | 0 | 19 | 58 |
| JavaScript | 1 | 28 | 7 | 9 | 44 |
+----------------+------------+------------+------------+------------+------------+
Directories
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
| path | files | code | comment | blank | total |
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
| . | 27 | 6,961 | 1,254 | 1,252 | 9,467 |
| . (Files) | 3 | 63 | 6 | 22 | 91 |
| components | 12 | 3,710 | 505 | 498 | 4,713 |
| components (Files) | 7 | 2,525 | 334 | 331 | 3,190 |
| components/cells | 5 | 1,185 | 171 | 167 | 1,523 |
| hooks | 6 | 2,855 | 630 | 669 | 4,154 |
| types | 1 | 16 | 4 | 4 | 24 |
| utils | 5 | 317 | 109 | 59 | 485 |
+-------------------------------------------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+
Files
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
| filename | language | code | comment | blank | total |
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/README.md | Markdown | 39 | 0 | 19 | 58 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx | TypeScript JSX | 230 | 10 | 8 | 248 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/BaseCellContent.tsx | TypeScript JSX | 18 | 0 | 3 | 21 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx | TypeScript JSX | 273 | 19 | 37 | 329 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/UpcValidationTableAdapter.tsx | TypeScript JSX | 113 | 17 | 10 | 140 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx | TypeScript JSX | 395 | 51 | 55 | 501 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx | TypeScript JSX | 969 | 182 | 158 | 1,309 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx | TypeScript JSX | 527 | 55 | 60 | 642 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/CheckboxCell.tsx | TypeScript JSX | 112 | 12 | 21 | 145 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx | TypeScript JSX | 233 | 34 | 33 | 300 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiSelectCell.tsx | TypeScript JSX | 420 | 66 | 59 | 545 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultilineInput.tsx | TypeScript JSX | 193 | 23 | 22 | 238 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx | TypeScript JSX | 227 | 36 | 32 | 295 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx | TypeScript JSX | 500 | 75 | 89 | 664 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useProductLinesFetching.tsx | TypeScript JSX | 264 | 75 | 81 | 420 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx | TypeScript JSX | 204 | 26 | 33 | 263 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx | TypeScript JSX | 337 | 88 | 92 | 517 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx | TypeScript JSX | 360 | 78 | 85 | 523 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx | TypeScript JSX | 1,190 | 288 | 289 | 1,767 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/index.tsx | TypeScript JSX | 20 | 6 | 2 | 28 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts | TypeScript | 4 | 0 | 1 | 5 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types/index.ts | TypeScript | 16 | 4 | 4 | 24 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts | TypeScript | 124 | 4 | 14 | 142 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts | TypeScript | 21 | 15 | 5 | 41 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts | TypeScript | 43 | 24 | 7 | 74 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js | JavaScript | 28 | 7 | 9 | 44 |
| /Users/matt/Dev/inventory/inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts | TypeScript | 101 | 59 | 24 | 184 |
| Total | | 6,961 | 1,254 | 1,252 | 9,467 |
+-------------------------------------------------------------------------------------------------------------------------------------------+----------------+------------+------------+------------+------------+

9
.gitignore vendored
View File

@@ -50,11 +50,6 @@ dashboard-server/meta-server/._package-lock.json
dashboard-server/meta-server/._services
*.tsbuildinfo
uploads/*
uploads/**/*
**/uploads/*
**/uploads/**/*
# CSV data files
*.csv
csv/*
@@ -64,7 +59,3 @@ csv/**/*
!csv/.gitkeep
inventory/tsconfig.tsbuildinfo
inventory-server/scripts/.fuse_hidden00000fa20000000a
.VSCodeCounter/
.VSCodeCounter/*
.VSCodeCounter/**/*

172
docs/PERMISSIONS.md
View File

@@ -1,172 +0,0 @@
# Permission System Documentation
This document outlines the permission system implemented in the Inventory Manager application.
## Permission Structure
Permissions follow this naming convention:
- Page access: `access:{page_name}`
- Actions: `{action}:{resource}`
Examples:
- `access:products` - Can access the Products page
- `create:products` - Can create new products
- `edit:users` - Can edit user accounts
## Permission Components
### PermissionGuard
The core component that conditionally renders content based on permissions.
```tsx
<PermissionGuard
permission="create:products"
fallback={<p>No permission</p>}
>
<button>Create Product</button>
</PermissionGuard>
```
Options:
- `permission`: Single permission code
- `anyPermissions`: Array of permissions (ANY match grants access)
- `allPermissions`: Array of permissions (ALL required)
- `adminOnly`: For admin-only sections
- `page`: Page name (checks `access:{page}` permission)
- `fallback`: Content to show if permission check fails
### PermissionProtectedRoute
Protects entire pages based on page access permissions.
```tsx
<Route path="/products" element={
<PermissionProtectedRoute page="products">
<Products />
</PermissionProtectedRoute>
} />
```
### ProtectedSection
Protects sections within a page based on action permissions.
```tsx
<ProtectedSection page="products" action="create">
<button>Add Product</button>
</ProtectedSection>
```
### PermissionButton
Button that automatically handles permissions.
```tsx
<PermissionButton
page="products"
action="create"
onClick={handleCreateProduct}
>
Add Product
</PermissionButton>
```
### SettingsSection
Specific component for settings with built-in permission checks.
```tsx
<SettingsSection
title="System Settings"
description="Configure global settings"
permission="edit:system_settings"
>
{/* Settings content */}
</SettingsSection>
```
## Permission Hooks
### usePermissions
Core hook for checking any permission.
```tsx
const { hasPermission, hasPageAccess, isAdmin } = usePermissions();
if (hasPermission('delete:products')) {
// Can delete products
}
```
### usePagePermission
Specialized hook for page-level permissions.
```tsx
const { canView, canCreate, canEdit, canDelete } = usePagePermission('products');
if (canEdit()) {
// Can edit products
}
```
## Database Schema
Permissions are stored in the database:
- `permissions` table: Stores all available permissions
- `user_permissions` junction table: Maps permissions to users
Admin users automatically have all permissions.
## Common Permission Codes
| Code | Description |
|------|-------------|
| `access:dashboard` | Access to Dashboard page |
| `access:products` | Access to Products page |
| `create:products` | Create new products |
| `edit:products` | Edit existing products |
| `delete:products` | Delete products |
| `view:users` | View user accounts |
| `edit:users` | Edit user accounts |
| `manage:permissions` | Assign permissions to users |
## Implementation Examples
### Page Protection
In `App.tsx`:
```tsx
<Route path="/products" element={
<PermissionProtectedRoute page="products">
<Products />
</PermissionProtectedRoute>
} />
```
### Component Level Protection
```tsx
const { canEdit } = usePagePermission('products');
function handleEdit() {
if (!canEdit()) {
toast.error("You don't have permission");
return;
}
// Edit logic
}
```
### UI Element Protection
```tsx
<PermissionButton
page="products"
action="delete"
onClick={handleDelete}
>
Delete
</PermissionButton>
```

396
docs/ValidationStep-Refactoring-Plan.md
View File

@@ -1,396 +0,0 @@
# ValidationStep Component Refactoring Plan
## Overview
This document outlines a comprehensive plan to refactor the current ValidationStep component (4000+ lines) into a more maintainable, modular structure. The new implementation will be developed alongside the existing component without modifying the original code. Once completed, the previous step in the workflow will offer the option to continue to either the original ValidationStep or the new implementation.
## Table of Contents
1. [Current Component Analysis](#current-component-analysis)
2. [New Architecture Design](#new-architecture-design)
3. [Component Structure](#component-structure)
4. [State Management](#state-management)
5. [Key Features Implementation](#key-features-implementation)
6. [Integration Plan](#integration-plan)
7. [Testing Strategy](#testing-strategy)
8. [Project Timeline](#project-timeline)
9. [Design Principles](#design-principles)
10. [Appendix: Function Reference](#appendix-function-reference)
## Current Component Analysis
The current ValidationStep component has several issues:
- **Size**: At over 4000 lines, it's difficult to maintain and understand
- **Multiple responsibilities**: Handles validation, UI rendering, template management, and more
- **Special cases**: Contains numerous special case handlers and exceptions
- **Complex state management**: State is distributed across multiple useState calls
- **Tightly coupled concerns**: UI, validation logic, and business rules are intertwined
### Key Features to Preserve
1. **Data Validation**
- Field-level validation (required, regex, unique)
- Row-level validation (supplier, company fields)
- UPC validation with API integration
- AI-assisted validation
2. **Template Management**
- Saving, loading, and applying templates
- Template-based validation
3. **UI Components**
- Editable table with specialized cell renderers
- Error display and management
- Filtering and sorting capabilities
- Status indicators and progress tracking
4. **Special Field Handling**
- Input fields with price formatting
- Multi-input fields with separator configuration
- Select fields with dropdown options
- Checkbox fields with boolean value mapping
- UPC fields with specialized validation
5. **User Interaction Flows**
- Tab and keyboard navigation
- Bulk operations (select all, apply template)
- Row validation on value change
- Error reporting and display
## New Architecture Design
The new architecture will follow these principles:
1. **Separation of Concerns**
- UI rendering separate from business logic
- Validation logic isolated from state management
- Clear interfaces between components
2. **Composable Components**
- Small, focused components with single responsibilities
- Reusable pattern for different field types
3. **Centralized State Management**
- Custom hooks for state management
- Clear data flow patterns
- Reduced prop drilling
4. **Consistent Error Handling**
- Standardized error structure
- Predictable error propagation
- User-friendly error display
5. **Performance Optimization**
- Virtualized table rendering
- Memoization of expensive computations
- Deferred validation for better user experience
## Component Structure
The new ValidationStepNew folder has the following structure:
```
ValidationStepNew/
├── index.tsx # Main entry point that composes all pieces
├── components/ # UI Components
│ ├── ValidationContainer.tsx # Main wrapper component
│ ├── ValidationTable.tsx # Table implementation
│ ├── ValidationCell.tsx # Cell component
│ ├── ValidationSidebar.tsx # Sidebar with controls
│ ├── ValidationToolbar.tsx # Top toolbar (removed as unnecessary)
│ ├── TemplateManager.tsx # Template management
│ ├── FilterPanel.tsx # Filtering interface (integrated into Container)
│ └── cells/ # Specialized cell renderers
│ ├── InputCell.tsx
│ ├── SelectCell.tsx
│ ├── MultiInputCell.tsx
│ └── CheckboxCell.tsx
├── hooks/ # Custom hooks
│ ├── useValidationState.tsx # Main state management
│ ├── useTemplates.tsx # Template-related logic (integrated into ValidationState)
│ ├── useFilters.tsx # Filtering logic (integrated into ValidationState)
│ └── useUpcValidation.tsx # UPC-specific validation
└── utils/ # Utility functions
├── validationUtils.ts # Validation helper functions
├── formatters.ts # Value formatting utilities
└── constants.ts # Constant values and configuration
```
### Component Responsibilities
#### ValidationContainer
- Main container component
- Coordinates between subcomponents
- Manages global state
- Handles navigation events (next, back)
- Contains filter controls
#### ValidationTable
- Displays the data in tabular form
- Manages selection state
- Handles keyboard navigation
- Integrates with TanStack Table
- Displays properly styled rows and cells
#### ValidationCell
- Factory component that renders appropriate cell type
- Manages cell-level state
- Handles validation errors display
- Manages edit mode
- Shows consistent error indicators
#### TemplateManager
- Handles template selection UI
- Provides template save/load functionality
- Manages template application to rows
#### Cell Components
- **InputCell**: Handles text input with multiline and price support
- **MultiInputCell**: Handles multiple values with separator configuration
- **SelectCell**: Command/popover component for single selection
- **CheckboxCell**: Boolean value selection with mapping support
## State Management
### Core State Interface
```typescript
interface ValidationState<T extends string> {
// Core data
data: RowData<T>[];
filteredData: RowData<T>[];
// Validation state
isValidating: boolean;
validationErrors: Map<number, Record<string, Error[]>>;
rowValidationStatus: Map<number, 'pending' | 'validating' | 'validated' | 'error'>;
// Selection state
rowSelection: RowSelectionState;
// Template state
templates: Template[];
selectedTemplateId: string | null;
// Filter state
filters: FilterState;
// Methods
updateRow: (rowIndex: number, key: T, value: any) => void;
validateRow: (rowIndex: number) => Promise<void>;
validateUpc: (rowIndex: number, upcValue: string) => Promise<void>;
applyTemplate: (templateId: string, rowIndexes: number[]) => void;
saveTemplate: (name: string, type: string) => void;
setFilters: (newFilters: Partial<FilterState>) => void;
// Additional methods...
}
```
### useValidationState Hook
The main state management hook handles:
- Data manipulation (update, sort, filter)
- Selection management
- Validation coordination
- Integration with validation utilities
- Template management
- Filtering and sorting
## Key Features Implementation
### 1. Field Type Handling
Implemented a strategy pattern for different field types:
```typescript
// In ValidationCell
const renderCellContent = () => {
const fieldType = field.fieldType.type
switch (fieldType) {
case 'input':
return <InputCell<T> field={field} value={value} onChange={onChange} ... />
case 'multi-input':
return <MultiInputCell<T> field={field} value={value} onChange={onChange} ... />
case 'select':
return <SelectCell<T> field={field} value={value} onChange={onChange} ... />
// etc.
}
}
```
### 2. Validation Logic
Validation is broken down into clear steps:
1. **Field Validation**: Apply field-level validations (required, regex, etc.)
2. **Row Validation**: Apply row-level validations and rowHook
3. **Table Validation**: Apply table-level validations (unique) and tableHook
Validation now happens automatically without explicit buttons, with immediate feedback on field blur.
### 3. UI Components
UI components follow these principles:
1. **Consistent Styling**: All components use shadcn UI for consistent look and feel
2. **Visual Feedback**: Errors are clearly indicated with icons and border styling
3. **Intuitive Editing**: Fields show outlines even when not in focus, and edit on click
4. **Proper Command Pattern**: Select and multi-select fields use command/popover pattern for better UX
5. **Focus Management**: Fields close when clicking away and perform validation on blur
## Design Principles
Based on user preferences and best practices, the following design principles guide this refactoring:
1. **Automatic Validation**
- Validation should happen automatically without explicit buttons
- All validation should run on initial data load
- Fields should validate on blur (when user clicks away)
2. **Modern UI Patterns**
- Command/popover components for all selects and multi-selects
- Consistent field outlines and borders even when not in focus
- Badge patterns for multi-select items
- Clear visual indicators for errors
3. **Reduced Complexity**
- Remove unnecessary UI elements like "validate all" buttons
- Eliminate redundant state and toast notifications
- Simplify component hierarchy where possible
- Find root causes rather than adding special cases
4. **Consistent Component Behavior**
- Fields should close when clicking away
- All inputs should follow the same editing pattern
- Error handling should be consistent across all field types
- Multi-select fields should allow selecting multiple items with clear visual feedback
## Integration Plan
### 1. Creating the New Component Structure
Folder structure has been created without modifying the existing code:
```bash
mkdir -p inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/{components,hooks,utils}
mkdir -p inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells
```
### 2. Implementing Basic Components
Core components have been implemented:
1. Created index.tsx as the main entry point
2. Implemented ValidationContainer with basic state management
3. Created ValidationTable for data display
4. Implemented basic cell rendering with specialized cell types
### 3. Implementing State Management
State management has been implemented:
1. Created useValidationState hook
2. Implemented data transformation utilities
3. Added validation logic
### 4. Integrating with Previous Step
The previous step component allows choosing between validation implementations, enabling gradual testing and adoption.
## Testing Strategy
1. **Unit Tests**
- Test individual utility functions
- Test hooks in isolation
- Test individual UI components
2. **Integration Tests**
- Test component interactions
- Test state management flow
- Test validation logic integration
3. **Comparison Tests**
- Compare output of new component with original
- Verify that all functionality works the same
4. **Performance Tests**
- Measure render times
- Evaluate memory usage
- Compare against original component
## Project Timeline
1. **Phase 1: Initial Structure (Completed)**
- Set up folder structure
- Implement basic components
- Create core state management
2. **Phase 2: Core Functionality (In Progress)**
- Implement validation logic (completed)
- Create cell renderers (completed)
- Add template management (in progress)
3. **Phase 3: Special Features (Upcoming)**
- Implement UPC validation
- Add AI validation
- Handle special cases
4. **Phase 4: UI Refinement (Ongoing)**
- Improve error display (completed)
- Enhance user interactions (completed)
- Optimize performance (in progress)
5. **Phase 5: Testing and Integration (Upcoming)**
- Write tests
- Fix bugs
- Integrate with previous step
## Appendix: Function Reference
This section documents the core functions from the original ValidationStep that need to be preserved in the new implementation.
### Validation Functions
1. **validateRegex** - Validates values against regex patterns
2. **getValidationError** - Determines field-level validation errors
3. **validateAndCommit** - Validates and commits new values
4. **validateData** - Validates all data rows
5. **validateUpcAndGenerateItemNumbers** - Validates UPC codes and generates item numbers
### Formatting Functions
1. **formatPrice** - Formats price values
2. **getDisplayValue** - Gets formatted display value based on field type
3. **isMultiInputType** - Checks if field is multi-input type
4. **getMultiInputSeparator** - Gets separator for multi-input fields
5. **isPriceField** - Checks if field should be formatted as price
### Template Functions
1. **loadTemplates** - Loads templates from storage
2. **saveTemplate** - Saves a new template
3. **applyTemplate** - Applies a template to selected rows
4. **getTemplateDisplayText** - Gets display text for a template
### AI Validation Functions
1. **handleAiValidation** - Triggers AI validation
2. **showCurrentPrompt** - Shows current AI prompt
3. **getFieldDisplayValue** - Gets display value for a field
4. **highlightDifferences** - Highlights differences between original and corrected values
5. **getFieldDisplayValueWithHighlight** - Gets display value with highlighted differences
6. **revertAiChange** - Reverts an AI-suggested change
7. **isChangeReverted** - Checks if an AI change has been reverted
### Event Handlers
1. **handleUpcValueUpdate** - Handles UPC value updates
2. **handleBlur** - Handles input blur events
3. **handleWheel** - Handles wheel events for navigation
4. **copyValueDown** - Copies a value to cells below
5. **handleSkuGeneration** - Generates SKUs
By following this refactoring plan, we continue to transform the monolithic ValidationStep component into a modular, maintainable set of components while preserving all existing functionality and aligning with user preferences for design and behavior.

137
docs/ValidationStepNew-Implementation-Status.md
View File

@@ -1,137 +0,0 @@
# ValidationStepNew Implementation Status
## Overview
This document outlines the current status of the ValidationStepNew implementation, a refactored version of the original ValidationStep component. The goal is to create a more maintainable, modular component that preserves all functionality of the original while eliminating technical debt and implementing modern UI patterns.
## Design Principles
Based on the user's preferences, we're following these core design principles:
1. **Automatic Validation**
- ✅ Validation runs automatically on data load
- ✅ No explicit "validate all" button needed
- ✅ Fields validate on blur when user clicks away
- ✅ Immediate visual feedback for validation errors
2. **Modern UI Patterns**
- ✅ Command/popover components for selects and multi-selects
- ✅ Consistent field outlines and borders even when not in focus
- ✅ Badge pattern for multi-select field items
- ✅ Visual indicators for errors with appropriate styling
3. **Reduced Complexity**
- ✅ Removed unnecessary UI elements like "validate all" button
- ✅ Eliminated redundant toast notifications
- ✅ Simplified component hierarchy
- ✅ Fixed root causes rather than adding special cases
4. **Consistent Behavior**
- ✅ Fields close when clicking away
- ✅ All inputs follow the same editing pattern
- ✅ Error handling is consistent across field types
- ✅ Multi-select fields allow selecting multiple items
## Completed Components
### Core Structure
- ✅ Main component structure
- ✅ Directory organization
- ✅ TypeScript interfaces
- ✅ Props definition and passing
### State Management
- ✅ `useValidationState` hook for centralized state
- ✅ Data validation logic
- ✅ Integration with rowHook and tableHook
- ✅ Error tracking and management
- ✅ Row selection
- ✅ Automatic validation on data load
### UI Components
- ✅ ValidationContainer with appropriate layout
- ✅ ValidationTable with shadcn UI components
- ✅ ValidationCell factory component
- ✅ Row select/deselect functionality
- ✅ Error display and indicators
- ✅ Selection action bar
### Cell Components
- ✅ InputCell with price and multiline support
- ✅ MultiInputCell with separator configuration
- ✅ SelectCell using command/popover pattern
- ✅ CheckboxCell with boolean mapping
- ✅ Consistent styling across all field types
- ✅ Proper edit/view state management
- ✅ Outlined borders in both edit and view modes
### Utility Functions
- ✅ Value formatting for display
- ✅ Field type detection
- ✅ Error creation and management
- ✅ Price formatting
### UI Improvements
- ✅ Consistent borders and field outlines
- ✅ Fields that properly close when clicking away
- ✅ Multi-select with badge UI pattern
- ✅ Command pattern for searchable select menus
- ✅ Better visual error indication
## Pending Tasks
### Enhanced Validation
- ⏳ AI validation system
- ⏳ Custom validation hooks
- ⏳ Enhanced UPC validation with API integration
- ⏳ Validation visualizations
### Advanced UI Features
- ⏳ Table virtualization for performance
- ⏳ Drag-and-drop reordering
- ⏳ Bulk operations (copy down, fill all, etc.)
- ⏳ Keyboard navigation improvements
- ⏳ Template dialogs and management UI
### Special Features
- ⏳ Image preview integration
- ⏳ SKU generation system
- ⏳ Item number generation
- ⏳ Dependent dropdown values
### Testing
- ⏳ Unit tests for utility functions
- ⏳ Component tests
- ⏳ Integration tests
- ⏳ Performance benchmarks
## Known Issues
1. TypeScript error for `validationDisabled` property in ValidationCell.tsx
2. Some type casting is needed due to complex generic types
3. Need to address edge cases for multi-select fields validation
4. Proper error handling for API calls needs implementation
## Next Steps
1. Fix TypeScript errors in ValidationCell and related components
2. Complete template management functionality
3. Implement UPC validation with API integration
4. Make multi-select field validation more robust
5. Add comprehensive tests
## Performance Improvements
We've already implemented several performance optimizations:
1. ✅ More efficient state updates by removing unnecessary re-renders
2. ✅ Better error handling to prevent cascading validations
3. ✅ Improved component isolation to prevent unnecessary re-renders
4. ✅ Automatic validation that doesn't block the UI
Additional planned improvements:
1. Virtualized table rendering for large datasets
2. Memoization of expensive calculations
3. Optimized state updates to minimize re-renders
4. Batched API calls for validation

185
docs/calculate-issues.md
View File

@@ -1,185 +0,0 @@
1. **Missing Updates for Reorder Point and Safety Stock** [RESOLVED - product-metrics.js]
- **Problem:** In the **product_metrics** table (used by the inventory health view), the fields **reorder_point** and **safety_stock** are never updated in the product metrics calculations. Although a helper function (`calculateReorderQuantities`) exists and computes these values, the update query in the `calculateProductMetrics` function does not assign any values to these columns.
- **Effect:** The inventory health view relies on these fields (using COALESCE to default them to 0), which means that stock might never be classified as "Reorder" or "Healthy" based on the proper reorder point or safety stock calculations.
- **Example:** Even if a product's base metrics would require a reorder (for example, if its days of inventory are low), the view always shows a value of 0 for reorder_point and safety_stock.
- **Fix:** Update the product metrics query (or add a subsequent update) so that **pm.reorder_point** and **pm.safety_stock** are calculated (for instance, by integrating the logic from `calculateReorderQuantities`) and stored in the table.
2. **Overwritten Module Exports When Combining Scripts** [RESOLVED - calculate-metrics.js]
- **Problem:** The code provided shows two distinct exports. The main metrics calculation module exports `calculateMetrics` (along with cancel and getProgress helpers), but later in the same concatenated file the module exports are overwritten.
- **Effect:** If these two code sections end up in a single module file, the export for the main calculation will be lost. This would break any code that calls the overall metrics calculation.
- **Example:** An external caller expecting to run `calculateMetrics` would instead receive the `calculateProductMetrics` function.
- **Fix:** Make sure each script resides in its own module file. Verify that the module boundaries and exports are not accidentally merged or overwritten when deployed.
3. **Potential Formula Issue in EOQ Calculation (Reorder Qty)** [RESOLVED - product-metrics.js]
- **Problem:** The helper function `calculateReorderQuantities` uses an EOQ formula with a holding cost expressed as a percentage (0.25) rather than a per‐unit cost.
- **Effect:** If the intent was to use the traditional EOQ formula (which expects a holding cost per unit rather than a percentage), this could lead to an incorrect reorder quantity.
- **Example:** For a given annual demand and fixed order cost, the computed reorder quantity might be higher or lower than expected.
- **Fix:** Double-check the EOQ formula. If the intention is to compute based on a percentage, then document that clearly; otherwise, adjust the formula to use the proper holding cost value.
4. **Potential Overlap or Redundancy in GMROI Calculation** [RESOLVED - time-aggregates.js]
- **Problem:** In the time aggregates function, GMROI is calculated in two steps. The initial INSERT query computes GMROI as
`CASE WHEN s.inventory_value > 0 THEN (s.total_revenue - s.total_cost) / s.inventory_value ELSE 0 END`
and then a subsequent UPDATE query recalculates it as an annualized value using gross profit and active days.
- **Effect:** Overwriting a computed value may be intentional to refine the metric, but if not coordinated it can cause confusion or unexpected output in the `product_time_aggregates` table.
- **Example:** A product's GMROI might first appear as a simple ratio but then be updated to a scaled value based on the number of active days, which could lead to inconsistent reporting if not documented.
- **Fix:** Consolidated the GMROI calculation into a single step in the initial INSERT query, properly handling annualization and NULL values.
5. **Handling of Products Without Orders or Purchase Data** [RESOLVED - time-aggregates.js]
- **Problem:** In the INSERT query of the time aggregates function, the UNION covers two cases: one for products with order data (from `monthly_sales`) and one for products that have entries in `monthly_stock` but no matching order data.
- **Effect:** If a product has neither orders nor purchase orders, it won't get an entry in `product_time_aggregates`. Depending on business rules, this might be acceptable or might mean missing data.
- **Example:** A product that's new or rarely ordered might not appear in the time aggregates view, potentially affecting downstream calculations.
- **Fix:** Added an `all_products` CTE and modified the JOIN structure to ensure every product gets an entry with appropriate default values, even if it has no orders or purchase orders.
6. **Redundant Recalculation of Vendor Metrics**
- **Problem:** Similar concepts from prior scripts where cumulative metrics (like **total_revenue** and **total_cost**) are calculated in multiple query steps without necessary validation or optimization. In the vendor metrics script, calculations for total revenue and margin are performed within a `WITH` clause, which is then used in other parts of the process, making it more complex than needed.
- **Effect:** There's unnecessary duplication in querying the same data multiple times across subqueries. It could result in decreased performance and may even lead to excess computation if the subqueries are not optimized or correctly indexed.
- **Example:** Vendor sales and vendor purchase orders (PO) metrics are calculated in separate `WITH` clauses, leading to repeated calculations.
- **Fix:** Synthesize the required metrics into fewer queries or reuse the results within the `WITH` clause itself. Avoid redundant calculations of **revenue** and **cost** unless truly necessary.
7. **Handling Products Without Orders or Purchase Orders**
- **Problem:** In your `calculateVendorMetrics` script, the initial insert for vendor sales doesn't fully address the products that might not have matching orders or purchase orders. If a vendor has products without any sales within the last 12 months, the results may not be fully accurate unless handled explicitly.
- **Effect:** If no orders exist for a product associated with a particular vendor, that product will not contribute to the vendor's metrics, potentially omitting important data when calculating **total_orders** or **total_revenue**.
- **Example:** The scripted statistics fill gaps, but products with no recent purchase or sales orders might not be counted accurately.
- **Fix:** Include logic to handle scenarios where these products still need to be part of the vendor calculation. Use a `LEFT JOIN` wherever possible to account for cases without sales or purchase orders.
8. **Redundant `ON DUPLICATE KEY UPDATE`**
- **Problem:** Multiple queries in the `calculateVendorMetrics` script use `ON DUPLICATE KEY UPDATE` clauses to handle repeated metrics updates. This is useful for ensuring the most up-to-date calculations but can cause inconsistencies if multiple calculations happen for the same product or vendor simultaneously.
- **Effect:** This approach can lead to an inaccurate update of brand-specific data when insertion and update overlap. Each time you add a new batch, an existing entry could be overwritten if not handled correctly.
- **Example:** Vendor country, category, or sales-related metrics could unintentionally update during processing.
- **Fix:** Match on current status more robustly in case of existing rows to avoid unnecessary updates. Ensure that the key used for `ON DUPLICATE KEY` aligns with any foreign key relationships that might indicate an already processed entry.
9. **SQL Query Performance with Multiple Nested `WITH` Clauses**
- **Problem:** Heavily nested queries (especially **WITH** clauses) may lead to slow performance depending on the size of the dataset.
- **Effect:** Computational burden could be high when the database is large, e.g., querying **purchase orders**, **vendor sales**, and **product info** simultaneously. Even with proper indexes, the deployment might struggle in production environments.
- **Example:** Multiple `WITH` clauses in the vendor and brand metrics calculation scripts might work fine in small datasets but degrade performance in production.
- **Fix:** Combine some subqueries and reduce the layer of computations needed for calculating final metrics. Test performance on a production-sized dataset to see how nested queries are handled.
10. **Missing Updates for Reorder Metrics (Vendor/Brand)**
- **Previously Identified Issue:** Inconsistent updates for **reorder_point** and **safety_stock** across earlier scripts.
- **Current Impact on This Script:** The vendor and brand metrics do not have explicit updates for reorder point or safety stock, which are essential for inventory evaluation.
- **Effect:** The correct thresholds and reorder logic for vendor product inventory aren't fully accounted for in these scripts.
- **Fix:** Integrate relevant logic to update **reorder_point** or **safety_stock** within the vendor and brand metrics calculations. Ensure that it's consistently computed and stored.
11. **Data Integrity and Consistency**
**w**hen tracking sales growth or performance
- **Problem:** Brand metrics include a sales growth clause where negative results can sometimes be skewed severely if period data varies considerably.
- **Effect:** If period boundaries are incorrect or records are missing, this can create drastic growth rate calculations.
- **Example:** If the "previous" period has no sales but "current" has a substantial increase, the growth rate will show as **100%**.
- **Fix:** Implement checks that ensure both periods are valid and that the system calculates growth accurately, avoiding growth rates based solely on potential outliers. Replace consistent gaps with a no-growth rate or a meaningful zero.
12. **Exclusion of Vendors With No Sales**
The vendor metrics query is driven by the `vendor_sales` CTE, which aggregates data only for vendors that have orders in the past 12 months.
- **Impact:** Vendors that have purchase activity (or simply exist in vendor_details) but no recent sales won't show up in vendor_metrics. This could cause the frontend to miss metrics for vendors that might still be important.
- **Fix:** Consider adding a UNION or changing the driving set so that all vendors (for example, from vendor_details) are included—even if they have zero sales.
13. **Identical Formulas for On-Time Delivery and Order Fill Rates**
Both metrics are calculated as `(received_orders / total_orders) * 100`.
- **Impact:** If the business expects these to be distinct (for example, one might factor in on-time receipt versus mere receipt), then showing identical values on the frontend could be misleading.
- **Fix:** Verify and adjust the formulas if on-time delivery and order fill rates should be computed differently.
14. **Handling Nulls and Defaults in Aggregations**
The query uses COALESCE in most places, but be sure that every aggregated value (like average lead time) correctly defaults when no data is present.
- **Impact:** Incorrect defaults might cause odd or missing numbers on the production interface.
- **Fix:** Double-check that all numeric aggregates reliably default to 0 where needed.
15. **Inconsistent Stock Filtering Conditions**
In the main brand metrics query the CTE filters products with the condition
`p.stock_quantity <= 5000 AND p.stock_quantity >= 0`
whereas in the brand time-based metrics query the condition is only `p.stock_quantity <= 5000`.
- **Impact:** This discrepancy may lead to inconsistent numbers (for example, if any products have negative stock, which might be due to data issues) between overall brand metrics and time-based metrics on the frontend.
- **Fix:** Standardize the filtering criteria so that both queries treat out-of-range stock values in the same way.
16. **Growth Rate Calculation Periods**
The growth rate is computed by comparing revenue from the last 3 months ("current") against a period from 15–12 months ago ("previous").
- **Impact:** This narrow window may not reflect typical year-over-year performance and could lead to volatile or unexpected growth percentages on the frontend.
- **Fix:** Revisit the business logic for growth—if a longer or different comparison period is preferred, adjust the date intervals accordingly.
17. **Potential NULLs in Aggregated Time-Based Metrics**
In the brand time-based metrics query, aggregate expressions such as `SUM(o.quantity * o.price)` aren't wrapped with COALESCE.
- **Impact:** If there are no orders for a given brand/month, these sums might return NULL rather than 0, which could propagate into the frontend display.
- **Fix:** Wrap such aggregates in COALESCE (e.g. `COALESCE(SUM(o.quantity * o.price), 0)`) to ensure a default numeric value.
18. **Grouping by Category Status in Base Metrics Insert**
- **Problem:** The INSERT for base category metrics groups by both `c.cat_id` and `c.status` even though the table's primary key is just `category_id`.
- **Effect:** If a category's status changes over time, the grouping may produce unexpected updates (or even multiple groups before the duplicate key update kicks in), possibly causing the wrong status or aggregated figures to be stored.
- **Example:** A category that toggles between "active" and "inactive" might have its metrics calculated differently on different runs.
- **Fix:** Ensure that the grouping keys match the primary key (or that the status update logic is exactly as intended) so that a single row per category is maintained.
19. **Potential Null Handling in Margin Calculations**
- **Problem:** In the query for category time metrics, the calculation of average margin uses expressions such as `SUM(o.quantity * (o.price - GREATEST(p.cost_price, 0)))` without using `COALESCE` on `p.cost_price`.
- **Effect:** If any product's `cost_price` is `NULL`, then `GREATEST(p.cost_price, 0)` returns `NULL` and the resulting sum (and thus the margin) could become `NULL` rather than defaulting to 0. This might lead to missing or misleading margin figures on the frontend.
- **Example:** A product with a missing cost price would make the entire margin expression evaluate to `NULL` even when sales exist.
- **Fix:** Replace `GREATEST(p.cost_price, 0)` with `GREATEST(COALESCE(p.cost_price, 0), 0)` (or simply use `COALESCE(p.cost_price, 0)`) to ensure that missing values are handled.
20. **Data Coverage in Growth Rate Calculation**
- **Problem:** The growth rate update depends on multiple CTEs (current period, previous period, and trend analysis) that require a minimum amount of data (for instance, `HAVING COUNT(*) >= 6` in the trend_stats CTE).
- **Effect:** Categories with insufficient historical data will fall into the "ELSE" branch (or may even be skipped if no revenue is present), which might result in a growth rate of 0.0 or an unexpected value.
- **Example:** A newly created category that has only two months of data won't have trend analysis, so its growth rate will be calculated solely by the simple difference, which might not reflect true performance.
- **Fix:** Confirm that this fallback behavior is acceptable for production; if not, adjust the logic so that every category receives a consistent growth rate even with sparse data.
21. **Omission of Forecasts for Zero–Sales Categories**
- **Observation:** The category–sales metrics query uses a `HAVING AVG(cs.daily_quantity) > 0` clause.
- **Effect:** Categories without any average daily sales will not receive a forecast record in `category_sales_metrics`. If the frontend expects a row (even with zeros) for every category, this will lead to missing data.
- **Fix:** Verify that it's acceptable for categories with no sales to have no forecast entry. If not, adjust the query so that a default forecast (with zeros) is inserted.
22. **Randomness in Category-Level Forecast Revenue Calculation**
- **Problem:** In the category-level forecasts query, the forecast revenue is multiplied by a factor of `(0.95 + (RAND() * 0.1))`.
- **Effect:** This introduces randomness into the forecast figures so that repeated runs could yield slightly different values. If deterministic forecasts are expected on the production frontend, this could lead to inconsistent displays.
- **Example:** The same category might show a 5% higher forecast on one run and 3% on another because of the random multiplier.
- **Fix:** Confirm that this randomness is intentional for your forecasting model; if forecasts are meant to be reproducible, remove or replace the `RAND()` factor with a fixed multiplier.
23. **Multi-Statement Cleanup of Temporary Tables**
- **Problem:** The cleanup query drops multiple temporary tables in one call (separated by semicolons).
- **Effect:** If your Node.js MySQL driver isn't configured to allow multi-statement execution, this query may fail, leaving temporary tables behind. Leftover temporary tables might eventually cause conflicts or resource issues.
- **Example:** Running the cleanup query could produce an error like "multi-statement queries not enabled," preventing proper cleanup.
- **Fix:** Either configure your database connection to allow multi-statements or issue separate queries for each temporary table drop to ensure that the cleanup runs successfully.
24. **Handling Products with No Sales Data**
- **Problem:** In the product-level forecast calculation, the CTE `daily_stats` includes a `HAVING AVG(ds.daily_quantity) > 0` clause.
- **Effect:** Products that have no sales (or a zero average daily quantity) will be excluded from the forecasts. This means the frontend won't show forecasts for non–selling products, which might be acceptable but could also be a completeness issue.
- **Example:** A product that has never sold will not appear in the `sales_forecasts` table.
- **Fix:** Confirm that it is intended for forecasts to be generated only for products with some sales activity. If forecasts are required for all products, adjust the query to insert default forecast records for products with zero sales.
25. **Complexity of the Forecast Formula Involving the Seasonality Factor**
- **Issue:**
The sales forecast calculations incorporate an adjustment factor using `COALESCE(sf.seasonality_factor, 0)` to modify forecast units and revenue. This means that if the seasonality data is missing (or not populated), the factor defaults to 0.
- **Potential Problem:**
A default value of 0 will drastically alter the forecast calculations—often leading to a forecast of 0 or an overly dampened forecast—when in reality the intended behavior might be to use a neutral multiplier (typically 1.0). This could result in forecasts that are not reflective of the actual seasonal impact, thereby skewing the figures that reach the frontend.
- **Fix:**
Review your data source for seasonality (the `sales_seasonality` table) and ensure it's consistently populated. Alternatively, if missing seasonality data is possible, consider using a more neutral default (such as 1.0) in your COALESCE. This change would prevent the forecast formulas from over-simplifying (or even nullifying) the forecast output due to missing seasonality factors.
26. **Group By with Seasonality Factor Variability**
- **Observation:** In the forecast insertion query, the GROUP BY clause includes `sf.seasonality_factor` along with other fields.
- **Effect:** If the seasonality factor differs (or is `NULL` versus a value) for different forecast dates, this might result in multiple rows for the same product and forecast date. However, the `ON DUPLICATE KEY UPDATE` clause will merge them—but only if the primary key (pid, forecast_date) is truly unique.
- **Fix:** Verify that the grouping produces exactly one row per product per forecast date. If there's potential for multiple rows due to seasonality variability, consider applying a COALESCE or an aggregation on the seasonality factor so that it does not affect grouping.
27. **Memory Management for Temporary Tables** [RESOLVED - calculate-metrics.js]
- **Problem:** In metrics calculations, temporary tables aren't always properly cleaned up if the process fails between creation and the DROP statement.
- **Effect:** If a process fails after creating temporary tables but before dropping them, these tables remain in memory until the connection is closed. In a production environment with multiple calculation runs, this could lead to memory leaks or table name conflicts.
- **Example:** The `temp_revenue_ranks` table creation in ABC classification could remain if the process fails before reaching the DROP statement.
- **Fix:** Implement proper cleanup in a finally block or use transaction management that ensures temporary tables are always cleaned up, even in failure scenarios.

72
docs/fix-multi-select.md
View File

@@ -1,72 +0,0 @@
# Solution: Keeping Dropdowns Open During Multiple Selections
## The Problem
When implementing a multi-select dropdown in React, a common issue occurs:
1. You select an item in the dropdown
2. The `onChange` handler is called, updating the data
3. This triggers a re-render of the parent component (in this case, the entire table)
4. During the re-render, the dropdown is unmounted and remounted
5. This causes the dropdown to close before you can make multiple selections
## The Solution: Deferred State Updates
The key insight is to **separate local state management from parent state updates**:
```typescript
// Step 1: Add local state to track selections
const [internalValue, setInternalValue] = useState<string[]>(value)
// Step 2: Handle popover open state changes
const handleOpenChange = useCallback((newOpen: boolean) => {
if (open && !newOpen) {
// Only update parent state when dropdown closes
if (JSON.stringify(internalValue) !== JSON.stringify(value)) {
onChange(internalValue);
}
}
setOpen(newOpen);
if (newOpen) {
// Sync internal state with external state when opening
setInternalValue(value);
}
}, [open, internalValue, value, onChange]);
// Step 3: Toggle selection only updates internal state
const toggleSelection = useCallback((selectedValue: string) => {
setInternalValue(prev => {
if (prev.includes(selectedValue)) {
return prev.filter(v => v !== selectedValue);
} else {
return [...prev, selectedValue];
}
});
}, []);
```
## Why This Works
1. **No parent re-renders during selection**: Since we're only updating local state, the parent component doesn't re-render during selection.
2. **Consistent UI**: The dropdown shows accurate selected states using the internal value.
3. **Data integrity**: The final selections are properly synchronized back to the parent when done.
4. **Resilient to external changes**: Initial state is synchronized when opening the dropdown.
## Implementation Steps
1. Create a local state variable to track selections inside the component
2. Only make selections against this local state while the dropdown is open
3. Defer updating the parent until the dropdown is explicitly closed
4. When opening, synchronize the internal state with the external value
## Benefits
This pattern:
- Avoids re-render cycles that would unmount the dropdown
- Maintains UI consistency during multi-selection
- Simplifies the component's interaction with parent components
- Works with existing component lifecycles rather than fighting against them
This solution is much simpler than trying to prevent event propagation or manipulating DOM events, and addresses the root cause of the issue: premature re-rendering.

239
docs/validate-table-changes-implementation-issue4.md
View File

@@ -1,239 +0,0 @@
# Validation Display Issue Implementation
## Issue Being Addressed
**Validation Display Issue**: Validation isn't happening beyond checking if a cell is required or not. All validation rules defined in import.tsx need to be respected.
* Required fields correctly show a red border when empty (✅ ALREADY WORKING)
* Non-empty fields with validation errors (regex, unique, etc.) should show a red border AND an alert circle icon with tooltip explaining the error (❌ NOT WORKING)
## Implementation Attempts
!!!!**NOTE** All previous attempts have been reverted and are no longer part of the code, please take this into account when trying a new solution. !!!!
### Attempt 1: Fix Validation Display Logic
**Approach**: Modified `processErrors` function to separate required errors from validation errors and show alert icons only for non-empty fields with validation errors.
**Changes Made**:
```typescript
function processErrors(value: any, errors: ErrorObject[]) {
// ...existing code...
// Separate required errors from other validation errors
const requiredErrors = errors.filter(error =>
error.message?.toLowerCase().includes('required')
);
const validationErrors = errors.filter(error =>
!error.message?.toLowerCase().includes('required')
);
const isRequiredButEmpty = valueIsEmpty && requiredErrors.length > 0;
const hasValidationErrors = validationErrors.length > 0;
const shouldShowErrorIcon = hasValidationErrors && !valueIsEmpty;
// ...more code...
}
```
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
### Attempt 2: Comprehensive Fix for Validation Display
**Approach**: Completely rewrote `processErrors` function with consistent empty value detection, clear error separation, and improved error message extraction.
**Changes Made**:
```typescript
function processErrors(value: any, errors: ErrorObject[]) {
if (!errors || errors.length === 0) {
return { filteredErrors: [], hasError: false, isRequiredButEmpty: false,
shouldShowErrorIcon: false, errorMessages: '' };
}
const valueIsEmpty = isEmpty(value);
const requiredErrors = errors.filter(error =>
error.message?.toLowerCase().includes('required')
);
const validationErrors = errors.filter(error =>
!error.message?.toLowerCase().includes('required')
);
let filteredErrors = valueIsEmpty ? requiredErrors : validationErrors;
const isRequiredButEmpty = valueIsEmpty && requiredErrors.length > 0;
const hasValidationErrors = validationErrors.length > 0;
const hasError = isRequiredButEmpty || hasValidationErrors;
const shouldShowErrorIcon = hasValidationErrors && !valueIsEmpty;
let errorMessages = '';
if (shouldShowErrorIcon) {
errorMessages = validationErrors.map(getErrorMessage).join('\n');
}
return { filteredErrors, hasError, isRequiredButEmpty, shouldShowErrorIcon, errorMessages };
}
```
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
### Attempt 3: Simplified Error Processing Logic
**Approach**: Refactored `processErrors` to use shared `isEmpty` function, simplified error icon logic, and made error message extraction more direct.
**Changes Made**:
```typescript
function processErrors(value: any, errors: ErrorObject[]) {
if (!errors || errors.length === 0) {
return { filteredErrors: [], hasError: false, isRequiredButEmpty: false,
shouldShowErrorIcon: false, errorMessages: '' };
}
const valueIsEmpty = isEmpty(value);
const requiredErrors = errors.filter(error =>
error.message?.toLowerCase().includes('required')
);
const validationErrors = errors.filter(error =>
!error.message?.toLowerCase().includes('required')
);
let filteredErrors = valueIsEmpty ? requiredErrors : validationErrors;
const isRequiredButEmpty = valueIsEmpty && requiredErrors.length > 0;
const hasValidationErrors = !valueIsEmpty && validationErrors.length > 0;
const hasError = isRequiredButEmpty || hasValidationErrors;
const shouldShowErrorIcon = hasValidationErrors;
let errorMessages = '';
if (shouldShowErrorIcon) {
errorMessages = validationErrors.map(getErrorMessage).join('\n');
}
return { filteredErrors, hasError, isRequiredButEmpty, shouldShowErrorIcon, errorMessages };
}
```
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
### Attempt 4: Consistent Error Processing Across Components
**Approach**: Updated both `processErrors` function and `ValidationCell` component to ensure consistent error handling between components.
**Changes Made**:
```typescript
// In processErrors function
function processErrors(value: any, errors: ErrorObject[]) {
// Similar to Attempt 3 with consistent error handling
}
// In ValidationCell component
const ValidationCell = ({ field, value, onChange, errors, /* other props */ }) => {
// ...existing code...
// Use the processErrors function to handle validation errors
const { hasError, isRequiredButEmpty, shouldShowErrorIcon, errorMessages } =
React.useMemo(() => processErrors(value, errors), [value, errors]);
// ...rest of the component...
}
```
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
### Attempt 5: Unified Error Processing with ItemNumberCell
**Approach**: Replaced custom error processing in `ValidationCell` with the same `processErrors` utility used by `ItemNumberCell`.
**Changes Made**:
```typescript
const ValidationCell = ({ field, value, onChange, errors, /* other props */ }) => {
// State and context setup...
// For item_number fields, use the specialized component
if (fieldKey === 'item_number') {
return <ItemNumberCell {...props} />;
}
// Use the same processErrors utility function that ItemNumberCell uses
const { hasError, isRequiredButEmpty, shouldShowErrorIcon, errorMessages } =
React.useMemo(() => processErrors(value, errors), [value, errors]);
// Rest of component...
}
```
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
### Attempt 6: Standardize Error Processing Across Cell Types
**Approach**: Standardized error handling across all cell types using the shared `processErrors` utility function.
**Changes Made**: Similar to Attempt 5, with focus on standardizing the approach for determining when to show validation error icons.
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
### Attempt 7: Replace Custom Error Processing with Shared Utility
**Approach**: Ensured consistent error handling between `ItemNumberCell` and regular `ValidationCell` components.
**Changes Made**: Similar to Attempts 5 and 6, with focus on using the shared utility function consistently.
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
### Attempt 8: Improved Error Normalization and Deep Comparison
**Approach**: Modified `MemoizedCell` in `ValidationTable.tsx` to use deep comparison for error objects and improved error normalization.
**Changes Made**:
```typescript
// Create a memoized cell component
const MemoizedCell = React.memo(({ field, value, onChange, errors, /* other props */ }) => {
return <ValidationCell {...props} />;
}, (prev, next) => {
// Basic prop comparison
if (prev.value !== next.value) return false;
if (prev.isValidating !== next.isValidating) return false;
if (prev.itemNumber !== next.itemNumber) return false;
// Deep compare errors - critical for validation display
if (!prev.errors && next.errors) return false;
if (prev.errors && !next.errors) return false;
if (prev.errors && next.errors) {
if (prev.errors.length !== next.errors.length) return false;
// Compare each error object
for (let i = 0; i < prev.errors.length; i++) {
if (prev.errors[i].message !== next.errors[i].message) return false;
if (prev.errors[i].level !== next.errors[i].level) return false;
if (prev.errors[i].source !== next.errors[i].source) return false;
}
}
// Compare options...
return true;
});
// In the field columns definition:
cell: ({ row }) => {
const rowErrors = validationErrors.get(row.index);
const cellErrors = rowErrors?.[fieldKey] || [];
// Ensure cellErrors is always an array
const normalizedErrors = Array.isArray(cellErrors) ? cellErrors : [cellErrors];
return <MemoizedCell {...props} errors={normalizedErrors} />;
}
```
**Result**: Non-empty fields with validation errors still aren't displaying the alert icon with tooltip.
## Root Causes (Revised Hypothesis)
After multiple attempts, the issue appears more complex than initially thought. Possible root causes:
1. **Error Object Structure**: Error objects might not have the expected structure or properties
2. **Error Propagation**: Errors might be getting filtered out before reaching cell components
3. **Validation Rules Configuration**: Validation rules in import.tsx might be incorrectly configured
4. **Error State Management**: Error state might not be properly updated or might be reset incorrectly
5. **Component Rendering Logic**: Components might not re-render when validation state changes
6. **CSS/Styling Issues**: Validation icons might be rendered but hidden due to styling issues
7. **Validation Timing**: Validation might be happening at the wrong time or getting overridden

138
docs/validate-table-changes-implementation-issue8.md
View File

@@ -1,138 +0,0 @@
# Multiple Cell Edit Issue Implementation
## Issue Being Addressed
**Multiple Cell Edit Issue**: When you enter values in 2+ cells before validation finishes, contents from all edited cells get erased when validation finishes.
## Implementation Attempts
### Attempt 1: Fix Multiple Cell Edit Issue (First Approach)
**Approach**:
- Added a tracking mechanism using a Set to keep track of cells that are currently being edited
- Modified the `flushPendingUpdates` function to preserve values of cells being edited
- Added cleanup of editing state after validation completes
**Changes Made**:
```typescript
// Add ref to track cells currently being edited
const currentlyEditingCellsRef = useRef(new Set<string>());
// Update a row's field value
const updateRow = useCallback((rowIndex: number, key: T, value: any) => {
// Add this cell to currently editing cells
const cellKey = `${rowIndex}-${key}`;
currentlyEditingCellsRef.current.add(cellKey);
// ...existing code...
// After validation completes, remove this cell from currently editing list
setTimeout(() => {
currentlyEditingCellsRef.current.delete(cellKey);
}, 100);
}, []);
// Modify flushPendingUpdates to respect currently editing cells
const flushPendingUpdates = useCallback(() => {
// ...existing code...
if (dataUpdates.length > 0) {
setData(prev => {
// ...existing code...
dataUpdates.forEach((row, index) => {
if (index < newData.length) {
const updatedRow = { ...row };
// Check if any fields in this row are currently being edited
// If so, preserve their current values in the previous data
Object.keys(prev[index] || {}).forEach(key => {
const cellKey = `${index}-${key}`;
if (currentlyEditingCellsRef.current.has(cellKey)) {
// Keep the value from the previous state for this field
updatedRow[key] = prev[index][key];
}
});
newData[index] = updatedRow;
}
});
return newData;
});
}
}, []);
```
**Result**:
- Slight improvement - the first value entered was saved, but any subsequent values still got erased
### Attempt 2: Fix Multiple Cell Edit Issue (Second Approach)
**Approach**:
- Completely revised the cell editing tracking system
- Used a Map with timestamps to track editing cells more accurately
- Added proper Promise-based tracking for cell validation
- Increased timeout from 100ms to 1000ms
- Made cleanup more robust by checking if it's still the same editing session
**Changes Made**:
```typescript
// Add ref to track cells currently being edited with timestamps
const currentlyEditingCellsRef = useRef(new Map<string, number>());
// Add ref to track validation promises
const validationPromisesRef = useRef<Map<string, Promise<void>>>(new Map());
// Update a row's field value
const updateRow = useCallback((rowIndex: number, key: T, value: any) => {
// Mark this cell as being edited with the current timestamp
const cellKey = `${rowIndex}-${key}`;
currentlyEditingCellsRef.current.set(cellKey, Date.now());
// ...existing code...
// Create a validation promise
const validationPromise = new Promise<void>((resolve) => {
setTimeout(() => {
try {
validateRow(rowIndex);
} finally {
resolve();
}
}, 0);
});
validationPromisesRef.current.set(cellKey, validationPromise);
// When validation is complete, remove from validating cells
validationPromise.then(() => {
// ...existing code...
// Keep this cell in the editing state for a longer time
setTimeout(() => {
if (currentlyEditingCellsRef.current.has(cellKey)) {
currentlyEditingCellsRef.current.delete(cellKey);
}
}, 1000); // Keep as "editing" for 1 second
});
}, []);
```
**Result**:
- Worse than the first approach - now all values get erased, including the first one
## Root Causes (Hypothesized)
- The validation process might be updating the entire data state, causing race conditions with cell edits
- The timing of validation completions might be problematic
- State updates might be happening in a way that overwrites user changes
- The cell state tracking system is not robust enough to prevent overwrites
## Next Steps
The issue requires a more fundamental approach than just tweaking the editing logic. We need to:
1. Implement a more robust state management system for cell edits that can survive validation cycles
2. Consider disabling validation during active editing
3. Implement a proper "dirty state" tracking system for cells

305
docs/validate-table-changes.md
View File

@@ -1,305 +0,0 @@
# Current Issues to Address
4. Validation isn't happening beyond checking if a cell is required or not - needs to respect rules in import.tsx
* Red cell outline if cell is required and it's empty
* Red outline + alert circle icon with tooltip if cell is NOT empty and isn't valid
8. When you enter a value in 2+ cells before validation finishes, contents from all edited cells get erased when validation finishes
## Do NOT change or edit
* Anything related to AI validation
* Anything about how templates or UPC validation work (only focus on specific issues described above)
* Anything outside of the ValidationStepNew folder
## Issues already fixed - do not work on these
✅FIXED 1. The red row background should go away when all cells in the row are valid and all required cells are populated
✅FIXED 2. Columns alignment with header is slightly off, gets worse the further right you go
✅FIXED 3. The copy down button is in the way of the validation error icon and the select open trigger - all three need to be in unique locations
✅FIXED 5. Description column needs to have an expanded view of some sort, maybe a popover to allow for easier editing
* Don't distort table to make it happen
✅FIXED 6. Need to ensure all cell's contents don't overflow the input (truncate). COO does this currently, probably more
✅FIXED 7. The template select cell is expanding, needs to be fixed size and truncate
✅FIXED 9. Import dialog state not fully reset when closing? (validate data step appears scrolled to the middle of the table where I left it)
✅FIXED 10. UPC column doesn't need to show loading state when Item Number is being processed, only show on item number column
✅FIXED 11. Copy down needs to show a loading state on the cells that it will copy to
✅FIXED 12. Shipping restrictions/tax category should default to ID 0 if we didn't get it elsewhere
✅FIXED 13. Header row should be sticky (both up/down and left/right)
✅FIXED 14. Need a way to scroll around table if user doesn't have mouse wheel for left/right
✅FIXED 15. Enhance copy down feature by allowing user to choose the last cell to copy to, instead of going all the way to the bottom
---------
# Validation Step Components Overview
## Core Components
### ValidationContainer
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx`
- Main wrapper component for the validation step
- Manages global state and coordinates between subcomponents
- Handles navigation events (next, back)
- Manages template application and validation state
- Coordinates UPC validation and product line loading
- Manages row selection and filtering
- Contains cache management for UPC validation results
- Maintains item number references separate from main data
### ValidationTable
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx`
- Handles data display and column configuration
- Uses TanStack Table for core functionality
- Features:
- Sticky header (both vertical and horizontal) - currently doesn't work properly
- Row selection with checkboxes
- Template selection column
- Dynamic column widths based on field types - specified in import.tsx component
- Copy down functionality for cell values
- Error highlighting for rows and cells
- Loading states for cells being validated
### ValidationCell
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx`
- Base cell component that renders different cell types based on field configuration
- Handles error display with tooltips
- Manages copy down button visibility
- Supports loading states during validation
- Cell Types:
1. InputCell: For single-value text input
2. SelectCell: For dropdown selection
3. MultiInputCell: For multiple value inputs
4. Template selection cells with SearchableTemplateSelect component
### SearchableTemplateSelect
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SearchableTemplateSelect.tsx`
- Advanced template selection component with search functionality
- Features:
- Real-time search filtering of templates
- Customizable display text for templates
- Support for default brand selection
- Accessible popover interface
- Keyboard navigation support
- Custom styling through className props
- Scroll event handling for nested scrollable areas
### TemplateManager
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/TemplateManager.tsx`
- Comprehensive template management interface
- Features:
- Template selection with search functionality
- Save template dialog with name and type inputs
- Batch template application to selected rows
- Template count tracking
- Toast notifications for user feedback
- Dialog-based interface for template operations
### AiValidationDialogs
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/AiValidationDialogs.tsx`
- Manages AI-assisted validation dialogs and interactions
### SaveTemplateDialog
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/SaveTemplateDialog.tsx`
- Dialog component for saving new templates
## Cell Components
### InputCell
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/InputCell.tsx`
- Handles single value text input
- Features:
- Inline/edit mode switching
- Multiline support
- Price formatting
- Error state display
- Loading state during validation
- Width constraints
- Automated cleanPriceFields processing for "$" formatting
### SelectCell
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/SelectCell.tsx`
- Handles dropdown selection
- Features:
- Searchable dropdown
- Custom option rendering
- Error state display
- Loading state during validation
- Width constraints
- Disabled state support
- Deferred search query handling for performance
### MultiInputCell
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/cells/MultiInputCell.tsx`
- Handles multiple value inputs
- Features:
- Comma-separated input support
- Multi-select dropdown for predefined options
- Custom separators
- Badge display for selected count
- Truncation for long values
- Width constraints
- Price formatting support
- Internal state management to avoid excessive re-renders
## Validation System
### useValidation Hook
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidation.tsx`
- Provides core validation logic
- Validates at multiple levels:
1. Field-level validation (required, regex, unique)
2. Row-level validation (supplier, company fields)
3. Table-level validation
4. Custom validation hooks support
- Error object structure includes message, level, and source properties
- Handles debounced validation updates to avoid UI freezing
### useAiValidation Hook
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useAiValidation.tsx`
- Manages AI-assisted validation logic and state
- Features:
- Tracks detailed changes per product
- Manages validation progress with estimated completion time
- Handles warnings and change suggestions
- Supports diff generation for changes
- Progress tracking with step indicators
- Prompt management for AI interactions
- Timer management for long-running operations
### useTemplates Hook
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useTemplates.tsx`
- Comprehensive template management system
- Features:
- Template CRUD operations
- Template application logic
- Default value handling
- Template search and filtering
- Batch template operations
- Template validation
### useUpcValidation Hook
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useUpcValidation.tsx`
- Dedicated UPC validation management
- Features:
- UPC format validation
- Supplier data validation
- Cache management for validation results
- Batch processing of UPC validations
- Item number generation logic
- Loading state management
### useFilters Hook
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useFilters.tsx`
- Advanced filtering system for table data
- Features:
- Multiple filter criteria support
- Dynamic filter updates
- Filter persistence
- Filter combination logic
- Performance optimized filtering
### useValidationState Hook
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx`
- Manages global validation state
- Handles:
- Data updates
- Template management
- Error tracking using Map objects
- Row selection
- Filtering
- UPC validation with caching to prevent duplicate API calls
- Product line loading
- Batch processing of updates
- Default value application for tax_cat and ship_restrictions (defaulting to "0")
- Price field auto-formatting to remove "$" symbols
### Utility Files
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validationUtils.ts`
- Core validation utility functions
- Includes:
- Field validation logic
- Error message formatting
- Validation rule processing
- Type checking utilities
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/errorUtils.ts`
- Error handling and formatting utilities
- Includes:
- Error object creation
- Error message formatting
- Error source tracking
- Error level management
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/dataMutations.ts`
- Data transformation and mutation utilities
- Includes:
- Row data updates
- Batch data processing
- Data structure conversions
- Change tracking
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/validation-helper.js`
- Helper functions for validation
- Includes:
- Common validation patterns
- Validation state management
- Validation result processing
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/utils/upcValidation.ts`
- UPC-specific validation utilities
- Includes:
- UPC format checking
- Checksum validation
- Supplier data matching
- Cache management
### Types
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/types.ts`
- Core type definitions for the validation step
### Validation Types
1. Required field validation
2. Regex pattern validation
3. Unique value validation
4. Custom field validation
5. Row-level validation
6. Table-level validation
## State Management
### useValidationState Hook
`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx`
- Manages global validation state
- Handles:
- Data updates
- Template management
- Error tracking using Map objects
- Row selection
- Filtering
- UPC validation with caching to prevent duplicate API calls
- Product line loading
- Batch processing of updates
- Default value application for tax_cat and ship_restrictions (defaulting to "0")
- Price field auto-formatting to remove "$" symbols
## UPC Validation System
### UPC Processing
- Validates UPCs against supplier data
- Cache system for UPC validation results
- Batch processing of UPC validation requests
- Auto-generation of item numbers based on UPC
- Special loading states for UPC/item number fields
- Separate state tracking to avoid unnecessary data structure updates
## Template System
### Template Management
- Supports saving and loading templates
- Template application to single/multiple rows
- Default template values
- Template search and filtering
## Performance Optimizations
1. Memoized components to prevent unnecessary renders
2. Virtualized table for large datasets
3. Deferred value updates for search inputs
4. Efficient error state management
5. Optimized cell update handling

131
docs/validation-hook-refactor.md
View File

@@ -1,131 +0,0 @@
# Refactoring Plan for Validation Code
## Current Structure Analysis
- **useValidationState.tsx**: ~1650 lines - Core validation state management
- **useValidation.tsx**: ~425 lines - Field/data validation utility
- **useUpcValidation.tsx**: ~410 lines - UPC-specific validation
## Proposed New Structure
### 1. Core Types & Utilities (150-200 lines)
**File: `validation/types.ts`**
- All interfaces and types (RowData, ValidationError, FilterState, Template, etc.)
- Shared utility functions (isEmpty, getCellKey, etc.)
**File: `validation/utils.ts`**
- Generic validation utility functions
- Caching mechanism and cache clearing helpers
- API URL helpers
### 2. Field Validation (300-350 lines)
**File: `validation/hooks/useFieldValidation.ts`**
- `validateField` function
- Field-level validation logic
- Required, regex, and other field validations
### 3. Uniqueness Validation (250-300 lines)
**File: `validation/hooks/useUniquenessValidation.ts`**
- `validateUniqueField` function
- `validateUniqueItemNumbers` function
- All uniqueness checking logic
### 4. UPC Validation (300-350 lines)
**File: `validation/hooks/useUpcValidation.ts`**
- `fetchProductByUpc` function
- `validateUpc` function
- `applyItemNumbersToData` function
- UPC validation state management
### 5. Validation Status Management (300-350 lines)
**File: `validation/hooks/useValidationStatus.ts`**
- Error state management
- Row validation status tracking
- Validation indicators and refs
- Batch validation processing
### 6. Data Management (300-350 lines)
**File: `validation/hooks/useValidationData.ts`**
- Data state management
- Row updates
- Data filtering
- Initial data processing
### 7. Template Management (250-300 lines)
**File: `validation/hooks/useTemplateManagement.ts`**
- Template saving
- Template application
- Template loading
- Template display helpers
### 8. Main Validation Hook (300-350 lines)
**File: `validation/hooks/useValidation.ts`**
- Main hook that composes all other hooks
- Public API export
- Initialization logic
- Core validation flow
## Function Distribution
### Core Types & Utilities
- All interfaces (InfoWithSource, ValidationState, etc.)
- `isEmpty` utility
- `getApiUrl` helper
### Field Validation
- `validateField`
- `validateRow`
- `validateData` (partial)
- All validation result caching
### Uniqueness Validation
- `validateUniqueField`
- `validateUniqueItemNumbers`
- Uniqueness caching mechanisms
### UPC Validation
- `fetchProductByUpc`
- `validateUpc`
- `validateAllUPCs`
- `applyItemNumbersToData`
- UPC validation state tracking (cells, rows)
### Validation Status Management
- `startValidatingCell`/`stopValidatingCell`
- `startValidatingRow`/`stopValidatingRow`
- `isValidatingCell`/`isRowValidatingUpc`
- Error state management
- `revalidateRows`
### Data Management
- Initial data cleaning/processing
- `updateRow`
- `copyDown`
- Search/filter functionality
- `filteredData` calculation
### Template Management
- `saveTemplate`
- `applyTemplate`
- `applyTemplateToSelected`
- `getTemplateDisplayText`
- `loadTemplates`/`refreshTemplates`
### Main Validation Hook
- Composition of all other hooks
- Initialization logic
- Button/navigation handling
- Field options management
## Implementation Approach
1. **Start with Types**: Create the types file first, as all other files will depend on it
2. **Create Utility Functions**: Move shared utilities next
3. **Build Core Validation**: Extract the field validation and uniqueness validation
4. **Separate UPC Logic**: Move all UPC-specific code to its own module
5. **Extract State Management**: Move data and status management to separate files
6. **Move Template Logic**: Extract template functionality
7. **Create Composition Hook**: Build the main hook that uses all other hooks
This approach will give you more maintainable code with clearer separation of concerns, making it easier to understand, test, and modify each component independently.

354
docs/validation-process-issues.md
View File

@@ -1,354 +0,0 @@
## 1. ✅ Error Filtering Logic Inconsistency (RESOLVED)
> **Note: This issue has been resolved by implementing a type-based error system.**
The filtering logic in `ValidationCell.tsx` previously relied on string matching, which was fragile:
```typescript
// Old implementation (string-based matching)
const filteredErrors = React.useMemo(() => {
return !isEmpty(value)
? errors.filter(error => !error.message?.toLowerCase().includes('required'))
: errors;
}, [value, errors]);
// New implementation (type-based filtering)
const filteredErrors = React.useMemo(() => {
return !isEmpty(value)
? errors.filter(error => error.type !== ErrorType.Required)
: errors;
}, [value, errors]);
```
The solution implemented:
- Added an `ErrorType` enum in `types.ts` to standardize error categorization
- Updated all error creation to include the appropriate error type
- Modified error filtering to use the type property instead of string matching
- Ensured consistent error handling across the application
**Guidelines for future development:**
- Always use the `ErrorType` enum when creating errors
- Never rely on string matching for error filtering
- Ensure all error objects include the `type` property
- Use the appropriate error type for each validation rule:
- `ErrorType.Required` for required field validations
- `ErrorType.Regex` for regex validations
- `ErrorType.Unique` for uniqueness validations
- `ErrorType.Custom` for custom validations
- `ErrorType.Api` for API-based validations
## 2. ⚠️ Redundant Error Processing (PARTIALLY RESOLVED)
> **Note: This issue has been partially resolved by the re-rendering optimizations.**
The system still processes errors in multiple places:
- In `ValidationCell.tsx`, errors are filtered by the optimized `processErrors` function
- In `useValidation.tsx`, errors are generated at the field level
- In `ValidationContainer.tsx`, errors are manipulated at the container level
While the error processing has been optimized to be more efficient, there is still some redundancy in how errors are handled across components. However, the current implementation has mitigated the performance impact.
**Improvements made:**
- Created a central `processErrors` function in ValidationCell that efficiently handles error filtering
- Implemented a batched update system to reduce redundant error processing
- Added better memoization to avoid reprocessing errors when not needed
**Future improvement opportunities:**
- Further consolidate error processing logic into a single location
- Create a dedicated error handling service or hook
- Implement a more declarative approach to error handling
## 3. Race Conditions in Async Validation
async validations could create race conditions:
- If a user types quickly, multiple validation requests might be in flight
- Later responses could overwrite more recent ones if they complete out of order
- The debouncing helps but doesn't fully solve this issue
## 4. Memory Leaks in Timeout Management
The validation timeouts are stored in refs:
```typescript
const validationTimeoutsRef = useRef<Record<number, NodeJS.Timeout>>({});
```
While there is cleanup on unmount, if rows are added/removed dynamically, timeouts for deleted rows might not be properly cleared.
## 5. ✅ Inefficient Error Storage (RESOLVED)
**Status: RESOLVED**
### Problem
Previously, validation errors were stored in multiple locations:
- In the `validationErrors` Map in `useValidationState`
- In the row data itself as `__errors`
This redundancy caused several issues:
- Inconsistent error states between the two storage locations
- Increased memory usage by storing the same information twice
- Complex state management to keep both sources in sync
- Difficulty reasoning about where errors should be accessed from
### Solution
We've implemented a unified error storage approach by:
- Making the `validationErrors` Map in `useValidationState` the single source of truth for all validation errors
- Removed the `__errors` property from row data
- Updated all validation functions to interact with the central error store instead of modifying row data
- Modified UPC validation to use the central error store
- Updated all components to read errors from the `validationErrors` Map instead of row data
### Key Changes
1. Modified `dataMutations.ts` to stop storing errors in row data
2. Updated the `Meta` type to remove the `__errors` property
3. Modified the `RowData` type to remove the `__errors` property
4. Updated the `useValidation` hook to return errors separately from row data
5. Modified the `useAiValidation` hook to work with the central error store
6. Updated the `useFilters` hook to check for errors in the `validationErrors` Map
7. Modified the `ValidationTable` and `ValidationCell` components to read errors from the `validationErrors` Map
### Benefits
- **Single Source of Truth**: All validation errors are now stored in one place
- **Reduced Memory Usage**: No duplicate storage of error information
- **Simplified State Management**: Only one state to update when errors change
- **Cleaner Data Structure**: Row data no longer contains validation metadata
- **More Maintainable Code**: Clearer separation of concerns between data and validation
### Future Improvements
While this refactoring addresses the core issue of inefficient error storage, there are still opportunities for further optimization:
1. ✅ **Redundant Error Processing**: ~~The validation process still performs some redundant calculations that could be optimized.~~ This has been largely addressed by the re-rendering optimizations.
2. **Race Conditions**: Async validation can lead to race conditions when multiple validations are triggered in quick succession.
3. **Memory Leaks**: The timeout management for validation could be improved to prevent potential memory leaks.
4. **Tight Coupling**: Components are still tightly coupled to the validation state structure.
5. **Error Prioritization**: The system doesn't prioritize errors well, showing all errors at once rather than focusing on the most critical ones first.
### Validation Flow
The validation process now works as follows:
1. **Error Generation**:
- Field-level validations generate errors based on validation rules
- Row-level hooks add custom validation errors
- Table-level validations (like uniqueness checks) add errors across rows
2. **Error Storage**:
- All errors are stored in the `validationErrors` Map in `useValidationState`
- The Map uses row indices as keys and objects of field errors as values
3. **Error Display**:
- The `ValidationTable` component checks the `validationErrors` Map to highlight rows with errors
- The `ValidationCell` component receives errors for specific fields from the `validationErrors` Map
- Errors are filtered in the UI to avoid showing "required" errors for fields with values
This focused refactoring approach has successfully addressed a critical issue while keeping changes manageable and targeted.
## 6. ✅ Excessive Re-rendering (RESOLVED)
**Status: RESOLVED**
### Problem
The validation system was suffering from excessive re-renders due to several key issues:
- **Inefficient Error Filtering**: The ValidationCell component was filtering errors on every render
- **Redundant Error Processing**: The same validation work was repeated in multiple components
- **Poor Memoization**: Components were inadequately memoized, causing unnecessary re-renders
- **Inefficient Batch Updates**: The state update system wasn't optimally batching changes
These issues led to performance problems, especially with large datasets, and affected the user experience.
### Solution
We've implemented a comprehensive optimization approach:
- **Optimized Error Processing**: Created an efficient `processErrors` function in ValidationCell that calculates all derived state in one pass
- **Enhanced Memoization**: Improved memo comparison functions to avoid unnecessary rerenders
- **Improved Batch Updates**: Redesigned the batching system to aggregate multiple changes before state updates
- **Single Update Pattern**: Implemented a queue-based update mechanism that applies multiple state changes at once
### Key Changes
1. Added a more efficient error processing function in ValidationCell
2. Created an enhanced error comparison function to properly compare error arrays
3. Improved the memo comparison function in ValidationCell
4. Added a batch update system in useValidationState
5. Implemented a queue-based update mechanism for row modifications
### Benefits
- **Improved Performance**: Reduced render cycles = faster UI response
- **Better User Experience**: Less lag when editing large datasets
- **Reduced Memory Usage**: Fewer component instantiations and temporary objects
- **Increased Scalability**: The application can now handle larger datasets without slowdown
- **Maintainable Code**: More predictable update flow that's easier to debug and extend
### Guidelines for future development
- Use the `processErrors` function for error filtering and processing
- Ensure React.memo components have proper comparison functions
- Use the batched update system for state changes
- Maintain stable references to objects and functions
- Use appropriate React hooks (useMemo, useCallback) with correct dependencies
- Avoid unnecessary recreations of arrays, objects, and functions
## 7. Complex Error Merging Logic
When merging errors from different sources, the logic is complex and potentially error-prone:
```typescript
// Merge field errors and row hook errors
const mergedErrors: Record<string, InfoWithSource> = {}
// Convert field errors to InfoWithSource
Object.entries(fieldErrors).forEach(([key, errors]) => {
if (errors.length > 0) {
mergedErrors[key] = {
message: errors[0].message,
level: errors[0].level,
source: ErrorSources.Row,
type: errors[0].type || ErrorType.Custom
}
}
})
```
This only takes the first error for each field, potentially hiding important validation issues.
## 8. ✅ Inconsistent Error Handling for Empty Values (PARTIALLY RESOLVED)
> **Note: This issue has been partially resolved by standardizing the isEmpty function and error type system.**
The system previously had different approaches to handling empty values:
- Some validations skipped empty values unless they're required
- Others processed empty values differently
- The `isEmpty` function was defined multiple times with slight variations
The solution implemented:
- Standardized the `isEmpty` function implementation
- Ensured consistent error type usage for required field validations
- Made error filtering consistent across the application
**Guidelines for future development:**
- Always use the shared `isEmpty` function for checking empty values
- Ensure consistent handling of empty values across all validation rules
- Use the `ErrorType.Required` type for all required field validations
## 9. Tight Coupling Between Components
The validation system is tightly coupled across components:
- `ValidationCell` needs to understand the structure of errors
- `ValidationTable` needs to extract and pass the right errors
- `ValidationContainer` directly manipulates the error structure
This makes it harder to refactor or reuse components independently.
## 10. Limited Error Prioritization
There's no clear prioritization of errors:
- When multiple errors exist for a field, which one should be shown first?
- Are some errors more important than others?
- The current system mostly shows the first error it finds
A more robust approach would be to have a consistent error source identification system and a clear prioritization strategy for displaying errors.
------------
Let me explain how these hooks fit together to create the validation errors that eventually get filtered in the `ValidationCell` component:
## The Validation Flow
1. **useValidationState Hook**:
This is the main state management hook used by the `ValidationContainer` component. It:
- Manages the core data state (`data`)
- Tracks validation errors in a Map (`validationErrors`)
- Provides functions to update and validate rows
2. **useValidation Hook**:
This is a utility hook that provides the core validation logic:
- `validateField`: Validates a single field against its validation rules
- `validateRow`: Validates an entire row, field by field
- `validateTable`: Runs table-level validations
- `validateUnique`: Checks for uniqueness constraints
- `validateData`: Orchestrates the complete validation process
## How Errors Are Generated
Validation errors come from multiple sources:
1. **Field-Level Validations**:
In `useValidation.tsx`, the `validateField` function checks individual fields against rules like:
- `required`: Field must have a value
- `regex`: Value must match a pattern
- `min`/`max`: Numeric constraints
2. **Row-Level Validations**:
The `validateRow` function in `useValidation.tsx` runs:
- Field validations for each field in the row
- Special validations for required fields like supplier and company
- Custom row hooks provided by the application
3. **Table-Level Validations**:
- `validateUnique` checks for duplicate values in fields marked as unique
- `validateTable` runs custom table hooks for cross-row validations
4. **API-Based Validations**:
In `useValidationState.tsx` and `ValidationContainer.tsx`:
- UPC validation via API calls
- Item number uniqueness checks
## The Error Flow
1. Errors are collected in the `validationErrors` Map in `useValidationState`
2. This Map is passed to `ValidationTable` as a prop
3. `ValidationTable` extracts the relevant errors for each cell and passes them to `ValidationCell`
4. In `ValidationCell`, the errors are filtered based on whether the cell has a value:
```typescript
// Updated implementation using type-based filtering
const filteredErrors = React.useMemo(() => {
return !isEmpty(value)
? errors.filter(error => error.type !== ErrorType.Required)
: errors;
}, [value, errors]);
```
## Key Insights
1. **Error Structure**:
Errors now have a consistent structure with type information:
```typescript
type ErrorObject = {
message: string;
level: string; // 'error', 'warning', etc.
source?: ErrorSources; // Where the error came from
type: ErrorType; // The type of error (Required, Regex, Unique, etc.)
}
```
2. **Error Sources**:
Errors can come from:
- Field validations (required, regex, etc.)
- Row validations (custom business logic)
- Table validations (uniqueness checks)
- API validations (UPC checks)
3. **Error Types**:
Errors are now categorized by type:
- `ErrorType.Required`: Field is required but empty
- `ErrorType.Regex`: Value doesn't match the regex pattern
- `ErrorType.Unique`: Value must be unique across rows
- `ErrorType.Custom`: Custom validation errors
- `ErrorType.Api`: Errors from API calls
4. **Error Filtering**:
The filtering in `ValidationCell` is now more robust:
- When a field has a value, errors of type `ErrorType.Required` are filtered out
- When a field is empty, all errors are shown
5. **Performance Optimizations**:
- Batch processing of validations
- Debounced updates to avoid excessive re-renders
- Memoization of computed values

538
docs/validation-table-scroll-issue.md
View File

@@ -1,538 +0,0 @@
# ValidationTable Scroll Position Issue
## Problem Description
The `ValidationTable` component in the inventory application suffers from a persistent scroll position issue. When the table content updates or re-renders, the scroll position resets to the top left corner. This creates a poor user experience, especially when users are working with large datasets and need to maintain their position while making edits or filtering data.
Specific behaviors:
- Scroll position resets to the top left corner during re-renders
- User loses their place in the table when data is updated
- The table does not preserve vertical or horizontal scroll position
## Relevant Files
- **`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationTable.tsx`**
- Main component that renders the validation table
- Handles scroll position management
- **`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationContainer.tsx`**
- Parent component that wraps ValidationTable
- Creates an EnhancedValidationTable wrapper component
- Manages data and state for the validation table
- **`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/hooks/useValidationState.tsx`**
- Provides state management and data manipulation functions
- Contains scroll-related code in the `updateRow` function
- **`inventory/src/lib/react-spreadsheet-import/src/steps/ValidationStepNew/components/ValidationCell.tsx`**
- Renders individual cells in the table
- May influence re-renders that affect scroll position
## Failed Attempts
We've tried multiple approaches to fix the scroll position issue, none of which have been successful:
### 1. Using Refs for Scroll Position
```typescript
const scrollPosition = useRef({
left: 0,
top: 0
});
// Capture position on scroll
const handleScroll = useCallback(() => {
if (tableContainerRef.current) {
scrollPosition.current = {
left: tableContainerRef.current.scrollLeft,
top: tableContainerRef.current.scrollTop
};
}
}, []);
// Restore in useLayoutEffect
useLayoutEffect(() => {
const container = tableContainerRef.current;
if (container) {
const { left, top } = scrollPosition.current;
if (left || top) {
container.scrollLeft = left;
container.scrollTop = top;
}
}
});
```
Result: Scroll position was still lost during updates.
### 2. Multiple Restoration Attempts with Timeouts
```typescript
// Multiple timeouts at different intervals
setTimeout(() => {
if (tableContainerRef.current) {
tableContainerRef.current.scrollTop = savedPosition.top;
tableContainerRef.current.scrollLeft = savedPosition.left;
}
}, 0);
setTimeout(() => {
if (tableContainerRef.current) {
tableContainerRef.current.scrollTop = savedPosition.top;
tableContainerRef.current.scrollLeft = savedPosition.left;
}
}, 50);
// Additional timeouts at 100ms, 300ms
```
Result: Still not reliable, scroll position would reset between timeouts or after all timeouts completed.
### 3. Using MutationObserver and ResizeObserver
```typescript
// Create a mutation observer to detect DOM changes
const mutationObserver = new MutationObserver(() => {
if (shouldPreserveScroll) {
restoreScrollPosition();
}
});
// Start observing the table for DOM changes
mutationObserver.observe(scrollableContainer, {
childList: true,
subtree: true,
attributes: true,
attributeFilter: ['style', 'class']
});
// Create a resize observer
const resizeObserver = new ResizeObserver(() => {
if (shouldPreserveScroll) {
restoreScrollPosition();
}
});
// Observe the table container
resizeObserver.observe(scrollableContainer);
```
Result: Did not reliably maintain scroll position, and sometimes caused other rendering issues.
### 4. Recursive Restoration Approach
```typescript
let attempts = 0;
const maxAttempts = 5;
const restore = () => {
if (tableContainerRef.current) {
tableContainerRef.current.scrollTop = y;
tableContainerRef.current.scrollLeft = x;
attempts++;
if (attempts < maxAttempts) {
setTimeout(restore, 50 * attempts);
}
}
};
restore();
```
Result: No improvement, scroll position still reset.
### 5. Using React State for Scroll Position
```typescript
const [scrollPos, setScrollPos] = useState<{top: number; left: number}>({top: 0, left: 0});
// Track the scroll event
useEffect(() => {
const handleScroll = () => {
if (scrollContainerRef.current) {
setScrollPos({
top: scrollContainerRef.current.scrollTop,
left: scrollContainerRef.current.scrollLeft
});
}
};
// Add scroll listener...
}, []);
// Restore scroll position
useLayoutEffect(() => {
const container = scrollContainerRef.current;
const { top, left } = scrollPos;
if (top > 0 || left > 0) {
requestAnimationFrame(() => {
if (container) {
container.scrollTop = top;
container.scrollLeft = left;
}
});
}
}, [scrollPos, data]);
```
Result: Caused the screen to shake violently when scrolling and did not preserve position.
### 6. Using Key Attribute for Stability
```typescript
return (
<div
key="validation-table-container"
ref={scrollContainerRef}
className="overflow-auto max-h-[calc(100vh-300px)]"
>
{/* Table content */}
</div>
);
```
Result: Did not resolve the issue and may have contributed to rendering instability.
### 7. Removing Scroll Management from Other Components
We removed scroll position management code from:
- `useValidationState.tsx` (in the updateRow function)
- `ValidationContainer.tsx` (in the enhancedUpdateRow function)
Result: This did not fix the issue either.
### 8. Simple Scroll Position Management with Event Listeners
```typescript
// Create a ref to store scroll position
const scrollPosition = useRef({ left: 0, top: 0 });
const tableContainerRef = useRef<HTMLDivElement>(null);
// Save scroll position when scrolling
const handleScroll = useCallback(() => {
if (tableContainerRef.current) {
scrollPosition.current = {
left: tableContainerRef.current.scrollLeft,
top: tableContainerRef.current.scrollTop
};
}
}, []);
// Add scroll listener
useEffect(() => {
const container = tableContainerRef.current;
if (container) {
container.addEventListener('scroll', handleScroll);
return () => container.removeEventListener('scroll', handleScroll);
}
}, [handleScroll]);
// Restore scroll position after data changes
useLayoutEffect(() => {
const container = tableContainerRef.current;
if (container) {
const { left, top } = scrollPosition.current;
if (left > 0 || top > 0) {
container.scrollLeft = left;
container.scrollTop = top;
}
}
}, [data]);
```
Result: Still did not maintain scroll position during updates.
### 9. Memoized Scroll Container Component
```typescript
// Create a stable scroll container that won't re-render with the table
const ScrollContainer = React.memo(({ children }: { children: React.ReactNode }) => {
const containerRef = useRef<HTMLDivElement>(null);
const scrollPosition = useRef({ left: 0, top: 0 });
const handleScroll = useCallback(() => {
if (containerRef.current) {
scrollPosition.current = {
left: containerRef.current.scrollLeft,
top: containerRef.current.scrollTop
};
}
}, []);
useEffect(() => {
const container = containerRef.current;
if (container) {
// Set initial scroll position if it exists
if (scrollPosition.current.left > 0 || scrollPosition.current.top > 0) {
container.scrollLeft = scrollPosition.current.left;
container.scrollTop = scrollPosition.current.top;
}
container.addEventListener('scroll', handleScroll);
return () => container.removeEventListener('scroll', handleScroll);
}
}, [handleScroll]);
return (
<div ref={containerRef} className="overflow-auto max-h-[calc(100vh-300px)]">
{children}
</div>
);
});
```
Result: Still did not maintain scroll position during updates, even with a memoized container.
### 10. Using TanStack Table State Management
```typescript
// Track scroll state in the table instance
const [scrollState, setScrollState] = useState({ scrollLeft: 0, scrollTop: 0 });
const table = useReactTable({
data,
columns,
getCoreRowModel: getCoreRowModel(),
state: {
rowSelection,
// Include scroll position in table state
scrollLeft: scrollState.scrollLeft,
scrollTop: scrollState.scrollTop
},
onStateChange: (updater) => {
if (typeof updater === 'function') {
const newState = updater({
rowSelection,
scrollLeft: scrollState.scrollLeft,
scrollTop: scrollState.scrollTop
});
if ('scrollLeft' in newState || 'scrollTop' in newState) {
setScrollState({
scrollLeft: newState.scrollLeft ?? scrollState.scrollLeft,
scrollTop: newState.scrollTop ?? scrollState.scrollTop
});
}
}
}
});
// Handle scroll events
const handleScroll = useCallback((event: React.UIEvent<HTMLDivElement>) => {
const target = event.target as HTMLDivElement;
setScrollState({
scrollLeft: target.scrollLeft,
scrollTop: target.scrollTop
});
}, []);
// Restore scroll position after updates
useLayoutEffect(() => {
if (tableContainerRef.current) {
tableContainerRef.current.scrollLeft = scrollState.scrollLeft;
tableContainerRef.current.scrollTop = scrollState.scrollTop;
}
}, [data, scrollState]);
```
Result: Still did not maintain scroll position during updates, even with table state management.
### 11. Using CSS Sticky Positioning
```typescript
return (
<div className="relative max-h-[calc(100vh-300px)] overflow-auto">
<Table>
<TableHeader className="sticky top-0 z-10 bg-background">
<TableRow>
{table.getFlatHeaders().map((header) => (
<TableHead
key={header.id}
style={{
width: `${header.getSize()}px`,
minWidth: `${header.getSize()}px`,
position: 'sticky',
top: 0,
backgroundColor: 'inherit'
}}
>
{/* Header content */}
</TableHead>
))}
</TableRow>
</TableHeader>
<TableBody>
{/* Table body content */}
</TableBody>
</Table>
</div>
);
```
Result: Still did not maintain scroll position during updates, even with native CSS scrolling.
### 12. Optimized Memoization with Object.is
```typescript
// Memoize data structures to prevent unnecessary re-renders
const memoizedData = useMemo(() => data, [data]);
const memoizedValidationErrors = useMemo(() => validationErrors, [validationErrors]);
const memoizedValidatingCells = useMemo(() => validatingCells, [validatingCells]);
const memoizedItemNumbers = useMemo(() => itemNumbers, [itemNumbers]);
// Use Object.is for more efficient comparisons
export default React.memo(ValidationTable, (prev, next) => {
if (!Object.is(prev.data.length, next.data.length)) return false;
if (prev.validationErrors.size !== next.validationErrors.size) return false;
for (const [key, value] of prev.validationErrors) {
if (!next.validationErrors.has(key)) return false;
if (!Object.is(value, next.validationErrors.get(key))) return false;
}
// ... more optimized comparisons ...
});
```
Result: Caused the page to crash with "TypeError: undefined has no properties" in the MemoizedCell component.
### 13. Simplified Component Structure
```typescript
const ValidationTable = <T extends string>({
data,
fields,
rowSelection,
setRowSelection,
updateRow,
validationErrors,
// ... other props
}) => {
const tableContainerRef = useRef<HTMLDivElement>(null);
const lastScrollPosition = useRef({ left: 0, top: 0 });
// Simple scroll position management
const handleScroll = useCallback(() => {
if (tableContainerRef.current) {
lastScrollPosition.current = {
left: tableContainerRef.current.scrollLeft,
top: tableContainerRef.current.scrollTop
};
}
}, []);
useEffect(() => {
const container = tableContainerRef.current;
if (container) {
container.addEventListener('scroll', handleScroll);
return () => container.removeEventListener('scroll', handleScroll);
}
}, [handleScroll]);
useLayoutEffect(() => {
const container = tableContainerRef.current;
if (container) {
const { left, top } = lastScrollPosition.current;
if (left > 0 || top > 0) {
requestAnimationFrame(() => {
if (container) {
container.scrollLeft = left;
container.scrollTop = top;
}
});
}
}
}, [data]);
return (
<div ref={tableContainerRef} className="overflow-auto max-h-[calc(100vh-300px)]">
<Table>
{/* ... table content ... */}
<TableBody>
{table.getRowModel().rows.map((row) => (
<TableRow
key={row.id}
className={cn(
"hover:bg-muted/50",
row.getIsSelected() ? "bg-muted/50" : "",
validationErrors.get(data.indexOf(row.original)) ? "bg-red-50/40" : ""
)}
>
{/* ... row content ... */}
</TableRow>
))}
</TableBody>
</Table>
</div>
);
};
```
Result: Still did not maintain scroll position during updates. However, this implementation restored the subtle red highlight on rows with validation errors, which is a useful visual indicator that should be preserved in future attempts.
### 14. Portal-Based Scroll Container
```typescript
// Create a stable container outside of React's control
const createStableContainer = () => {
const containerId = 'validation-table-container';
let container = document.getElementById(containerId);
if (!container) {
container = document.createElement('div');
container.id = containerId;
container.className = 'overflow-auto';
container.style.maxHeight = 'calc(100vh - 300px)';
document.body.appendChild(container);
}
return container;
};
const ValidationTable = <T extends string>({...props}) => {
const [container] = useState(createStableContainer);
const [mounted, setMounted] = useState(false);
useEffect(() => {
setMounted(true);
return () => {
if (container && container.parentNode) {
container.parentNode.removeChild(container);
}
};
}, [container]);
// ... table configuration ...
return createPortal(content, container);
};
```
Result: The table contents failed to render at all. The portal-based approach to maintain scroll position by moving the scroll container outside of React's control was unsuccessful.
## Current Understanding
The scroll position issue appears to be complex and likely stems from multiple factors:
1. React's virtual DOM reconciliation may be replacing the scroll container element during updates
2. The table uses complex memo patterns with custom equality checks that may not be working as expected
3. The data structure may be changing in ways that cause complete re-renders
4. The component hierarchy (with EnhancedValidationTable wrapper) may be affecting DOM stability
## Next Steps to Consider
At this point, we have tried multiple approaches without success:
1. Various scroll position management techniques
2. Memoization and optimization strategies
3. Different component structures
4. Portal-based rendering
Given that none of these approaches have fully resolved the issue, it may be worth:
1. Investigating if there are any parent component updates forcing re-renders
2. Profiling the application to identify the exact timing of scroll position resets
3. Considering if the current table implementation could be simplified
4. Exploring if the data update patterns could be optimized to reduce re-renders
## Conclusion
The scroll position issue has proven resistant to multiple solution attempts. Each approach has either failed to maintain scroll position, introduced new issues, or in some cases (like the portal-based approach) prevented the table from rendering entirely. A deeper investigation into the component lifecycle and data flow may be necessary to identify the root cause.

214
ecosystem.config.js
View File

@@ -1,209 +1,5 @@
// ecosystem.config.js
const path = require('path');
const dotenv = require('dotenv');
// Load environment variables safely with error handling
const loadEnvFile = (envPath) => {
try {
console.log('Loading env from:', envPath);
const result = dotenv.config({ path: envPath });
if (result.error) {
console.warn(`Warning: .env file not found or invalid at ${envPath}:`, result.error.message);
return {};
}
console.log('Env variables loaded from', envPath, ':', Object.keys(result.parsed || {}));
return result.parsed || {};
} catch (error) {
console.warn(`Warning: Error loading .env file at ${envPath}:`, error.message);
return {};
}
}
// Load environment variables for each server
const authEnv = loadEnvFile(path.resolve(__dirname, 'dashboard/auth-server/.env'));
const aircallEnv = loadEnvFile(path.resolve(__dirname, 'dashboard/aircall-server/.env'));
const klaviyoEnv = loadEnvFile(path.resolve(__dirname, 'dashboard/klaviyo-server/.env'));
const metaEnv = loadEnvFile(path.resolve(__dirname, 'dashboard/meta-server/.env'));
const googleAnalyticsEnv = require('dotenv').config({
path: path.resolve(__dirname, 'dashboard/google-server/.env')
}).parsed || {};
const typeformEnv = loadEnvFile(path.resolve(__dirname, 'dashboard/typeform-server/.env'));
const inventoryEnv = loadEnvFile(path.resolve(__dirname, 'inventory/.env'));
// Common log settings for all apps
const logSettings = {
log_rotate: true,
max_size: '10M',
retain: '10',
log_date_format: 'YYYY-MM-DD HH:mm:ss'
};
// Common app settings
const commonSettings = {
instances: 1,
exec_mode: 'fork',
autorestart: true,
watch: false,
max_memory_restart: '1G',
time: true,
...logSettings,
ignore_watch: [
'node_modules',
'logs',
'.git',
'*.log'
],
min_uptime: 5000,
max_restarts: 5,
restart_delay: 4000,
listen_timeout: 50000,
kill_timeout: 5000,
node_args: '--max-old-space-size=1536'
};
module.exports = {
apps: [
{
...commonSettings,
name: 'auth-server',
script: './dashboard/auth-server/index.js',
env: {
NODE_ENV: 'production',
PORT: 3003,
...authEnv
},
error_file: 'dashboard/auth-server/logs/pm2/err.log',
out_file: 'dashboard/auth-server/logs/pm2/out.log',
log_file: 'dashboard/auth-server/logs/pm2/combined.log',
env_production: {
NODE_ENV: 'production',
PORT: 3003
},
env_development: {
NODE_ENV: 'development',
PORT: 3003
}
},
{
...commonSettings,
name: 'aircall-server',
script: './dashboard/aircall-server/server.js',
env: {
NODE_ENV: 'production',
AIRCALL_PORT: 3002,
...aircallEnv
},
error_file: 'dashboard/aircall-server/logs/pm2/err.log',
out_file: 'dashboard/aircall-server/logs/pm2/out.log',
log_file: 'dashboard/aircall-server/logs/pm2/combined.log',
env_production: {
NODE_ENV: 'production',
AIRCALL_PORT: 3002
}
},
{
...commonSettings,
name: 'klaviyo-server',
script: './dashboard/klaviyo-server/server.js',
env: {
NODE_ENV: 'production',
KLAVIYO_PORT: 3004,
...klaviyoEnv
},
error_file: 'dashboard/klaviyo-server/logs/pm2/err.log',
out_file: 'dashboard/klaviyo-server/logs/pm2/out.log',
log_file: 'dashboard/klaviyo-server/logs/pm2/combined.log',
env_production: {
NODE_ENV: 'production',
KLAVIYO_PORT: 3004
}
},
{
...commonSettings,
name: 'meta-server',
script: './dashboard/meta-server/server.js',
env: {
NODE_ENV: 'production',
PORT: 3005,
...metaEnv
},
error_file: 'dashboard/meta-server/logs/pm2/err.log',
out_file: 'dashboard/meta-server/logs/pm2/out.log',
log_file: 'dashboard/meta-server/logs/pm2/combined.log',
env_production: {
NODE_ENV: 'production',
PORT: 3005
}
},
{
name: "gorgias-server",
script: "./dashboard/gorgias-server/server.js",
env: {
NODE_ENV: "development",
PORT: 3006
},
env_production: {
NODE_ENV: "production",
PORT: 3006
},
error_file: "dashboard/logs/gorgias-server-error.log",
out_file: "dashboard/logs/gorgias-server-out.log",
log_file: "dashboard/logs/gorgias-server-combined.log",
time: true
},
{
...commonSettings,
name: 'google-server',
script: path.resolve(__dirname, 'dashboard/google-server/server.js'),
watch: false,
env: {
NODE_ENV: 'production',
GOOGLE_ANALYTICS_PORT: 3007,
...googleAnalyticsEnv
},
error_file: path.resolve(__dirname, 'dashboard/google-server/logs/pm2/err.log'),
out_file: path.resolve(__dirname, 'dashboard/google-server/logs/pm2/out.log'),
log_file: path.resolve(__dirname, 'dashboard/google-server/logs/pm2/combined.log'),
env_production: {
NODE_ENV: 'production',
GOOGLE_ANALYTICS_PORT: 3007
}
},
{
...commonSettings,
name: 'typeform-server',
script: './dashboard/typeform-server/server.js',
env: {
NODE_ENV: 'production',
TYPEFORM_PORT: 3008,
...typeformEnv
},
error_file: 'dashboard/typeform-server/logs/pm2/err.log',
out_file: 'dashboard/typeform-server/logs/pm2/out.log',
log_file: 'dashboard/typeform-server/logs/pm2/combined.log',
env_production: {
NODE_ENV: 'production',
TYPEFORM_PORT: 3008
}
},
{
...commonSettings,
name: 'inventory-server',
script: './inventory/src/server.js',
env: {
NODE_ENV: 'production',
PORT: 3010,
...inventoryEnv
},
error_file: 'inventory/logs/pm2/err.log',
out_file: 'inventory/logs/pm2/out.log',
log_file: 'inventory/logs/pm2/combined.log',
env_production: {
NODE_ENV: 'production',
PORT: 3010,
...inventoryEnv
}
},
{
...commonSettings,
name: 'new-auth-server',
@@ -211,12 +7,16 @@ module.exports = {
env: {
NODE_ENV: 'production',
AUTH_PORT: 3011,
...inventoryEnv,
JWT_SECRET: process.env.JWT_SECRET
},
error_file: 'inventory-server/auth/logs/pm2/err.log',
out_file: 'inventory-server/auth/logs/pm2/out.log',
log_file: 'inventory-server/auth/logs/pm2/combined.log'
log_file: 'inventory-server/auth/logs/pm2/combined.log',
env_production: {
NODE_ENV: 'production',
AUTH_PORT: 3011,
JWT_SECRET: process.env.JWT_SECRET
}
}
]
};
};

103
inventory-server/auth/add-user.js
View File

@@ -1,103 +0,0 @@
require('dotenv').config({ path: '../.env' });
const bcrypt = require('bcrypt');
const { Pool } = require('pg');
const inquirer = require('inquirer');
// Log connection details for debugging (remove in production)
console.log('Attempting to connect with:', {
host: process.env.DB_HOST,
user: process.env.DB_USER,
database: process.env.DB_NAME,
port: process.env.DB_PORT
});
const pool = new Pool({
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
port: process.env.DB_PORT,
});
async function promptUser() {
const questions = [
{
type: 'input',
name: 'username',
message: 'Enter username:',
validate: (input) => {
if (input.length < 3) {
return 'Username must be at least 3 characters long';
}
return true;
}
},
{
type: 'password',
name: 'password',
message: 'Enter password:',
mask: '*',
validate: (input) => {
if (input.length < 8) {
return 'Password must be at least 8 characters long';
}
return true;
}
},
{
type: 'password',
name: 'confirmPassword',
message: 'Confirm password:',
mask: '*',
validate: (input, answers) => {
if (input !== answers.password) {
return 'Passwords do not match';
}
return true;
}
}
];
return inquirer.prompt(questions);
}
async function addUser() {
try {
// Get user input
const answers = await promptUser();
const { username, password } = answers;
// Hash password
const saltRounds = 10;
const hashedPassword = await bcrypt.hash(password, saltRounds);
// Check if user already exists
const checkResult = await pool.query(
'SELECT id FROM users WHERE username = $1',
[username]
);
if (checkResult.rows.length > 0) {
console.error('Error: Username already exists');
process.exit(1);
}
// Insert new user
const result = await pool.query(
'INSERT INTO users (username, password) VALUES ($1, $2) RETURNING id',
[username, hashedPassword]
);
console.log(`User ${username} created successfully with id ${result.rows[0].id}`);
} catch (error) {
console.error('Error creating user:', error);
console.error('Error details:', error.message);
if (error.code) {
console.error('Error code:', error.code);
}
} finally {
await pool.end();
}
}
addUser();

41
inventory-server/auth/add_user.js Normal file
View File

@@ -0,0 +1,41 @@
const bcrypt = require('bcrypt');
const mysql = require('mysql2/promise');
const readline = require('readline').createInterface({
input: process.stdin,
output: process.stdout,
});
require('dotenv').config({ path: '../.env' });
const dbConfig = {
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
};
async function addUser() {
const username = await askQuestion('Enter username: ');
const password = await askQuestion('Enter password: ');
const hashedPassword = await bcrypt.hash(password, 10);
const connection = await mysql.createConnection(dbConfig);
try {
await connection.query('INSERT INTO users (username, password) VALUES (?, ?)', [username, hashedPassword]);
console.log(`User ${username} added successfully.`);
} catch (error) {
console.error('Error adding user:', error);
} finally {
connection.end();
readline.close();
}
}
function askQuestion(query) {
return new Promise(resolve => readline.question(query, ans => {
resolve(ans);
}));
}
addUser();

882
inventory-server/auth/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

20
inventory-server/auth/package.json
View File

@@ -1,19 +1,21 @@
{
"name": "inventory-auth-server",
"name": "auth-server",
"version": "1.0.0",
"description": "Authentication server for inventory management system",
"description": "Authentication server for inventory management",
"main": "server.js",
"scripts": {
"start": "node server.js"
"start": "node server.js",
"dev": "nodemon server.js",
"add_user": "node add_user.js"
},
"dependencies": {
"bcrypt": "^5.1.1",
"cors": "^2.8.5",
"dotenv": "^16.4.7",
"dotenv": "^16.4.5",
"express": "^4.18.2",
"inquirer": "^8.2.6",
"jsonwebtoken": "^9.0.2",
"morgan": "^1.10.0",
"pg": "^8.11.3"
"jsonwebtoken": "^9.0.2"
},
"devDependencies": {
"nodemon": "^3.1.0"
}
}
}

128
inventory-server/auth/permissions.js
View File

@@ -1,128 +0,0 @@
// Get pool from global or create a new one if not available
let pool;
if (typeof global.pool !== 'undefined') {
pool = global.pool;
} else {
// If global pool is not available, create a new connection
const { Pool } = require('pg');
pool = new Pool({
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
port: process.env.DB_PORT,
});
console.log('Created new database pool in permissions.js');
}
/**
* Check if a user has a specific permission
* @param {number} userId - The user ID to check
* @param {string} permissionCode - The permission code to check
* @returns {Promise<boolean>} - Whether the user has the permission
*/
async function checkPermission(userId, permissionCode) {
try {
// First check if the user is an admin
const adminResult = await pool.query(
'SELECT is_admin FROM users WHERE id = $1',
[userId]
);
// If user is admin, automatically grant permission
if (adminResult.rows.length > 0 && adminResult.rows[0].is_admin) {
return true;
}
// Otherwise check for specific permission
const result = await pool.query(
`SELECT COUNT(*) AS has_permission
FROM user_permissions up
JOIN permissions p ON up.permission_id = p.id
WHERE up.user_id = $1 AND p.code = $2`,
[userId, permissionCode]
);
return result.rows[0].has_permission > 0;
} catch (error) {
console.error('Error checking permission:', error);
return false;
}
}
/**
* Middleware to require a specific permission
* @param {string} permissionCode - The permission code required
* @returns {Function} - Express middleware function
*/
function requirePermission(permissionCode) {
return async (req, res, next) => {
try {
// Check if user is authenticated
if (!req.user || !req.user.id) {
return res.status(401).json({ error: 'Authentication required' });
}
const hasPermission = await checkPermission(req.user.id, permissionCode);
if (!hasPermission) {
return res.status(403).json({
error: 'Insufficient permissions',
requiredPermission: permissionCode
});
}
next();
} catch (error) {
console.error('Permission middleware error:', error);
res.status(500).json({ error: 'Server error checking permissions' });
}
};
}
/**
* Get all permissions for a user
* @param {number} userId - The user ID
* @returns {Promise<string[]>} - Array of permission codes
*/
async function getUserPermissions(userId) {
try {
// Check if user is admin
const adminResult = await pool.query(
'SELECT is_admin FROM users WHERE id = $1',
[userId]
);
if (adminResult.rows.length === 0) {
return [];
}
const isAdmin = adminResult.rows[0].is_admin;
if (isAdmin) {
// Admin gets all permissions
const allPermissions = await pool.query('SELECT code FROM permissions');
return allPermissions.rows.map(p => p.code);
} else {
// Get assigned permissions
const permissions = await pool.query(
`SELECT p.code
FROM permissions p
JOIN user_permissions up ON p.id = up.permission_id
WHERE up.user_id = $1`,
[userId]
);
return permissions.rows.map(p => p.code);
}
} catch (error) {
console.error('Error getting user permissions:', error);
return [];
}
}
module.exports = {
checkPermission,
requirePermission,
getUserPermissions
};

513
inventory-server/auth/routes.js
View File

@@ -1,513 +0,0 @@
const express = require('express');
const router = express.Router();
const bcrypt = require('bcrypt');
const jwt = require('jsonwebtoken');
const { requirePermission, getUserPermissions } = require('./permissions');
// Get pool from global or create a new one if not available
let pool;
if (typeof global.pool !== 'undefined') {
pool = global.pool;
} else {
// If global pool is not available, create a new connection
const { Pool } = require('pg');
pool = new Pool({
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
port: process.env.DB_PORT,
});
console.log('Created new database pool in routes.js');
}
// Authentication middleware
const authenticate = async (req, res, next) => {
try {
const authHeader = req.headers.authorization;
if (!authHeader || !authHeader.startsWith('Bearer ')) {
return res.status(401).json({ error: 'Authentication required' });
}
const token = authHeader.split(' ')[1];
const decoded = jwt.verify(token, process.env.JWT_SECRET);
// Get user from database
const result = await pool.query(
'SELECT id, username, is_admin FROM users WHERE id = $1',
[decoded.userId]
);
if (result.rows.length === 0) {
return res.status(401).json({ error: 'User not found' });
}
// Attach user to request
req.user = result.rows[0];
next();
} catch (error) {
console.error('Authentication error:', error);
res.status(401).json({ error: 'Invalid token' });
}
};
// Login route
router.post('/login', async (req, res) => {
try {
const { username, password } = req.body;
// Get user from database
const result = await pool.query(
'SELECT id, username, password, is_admin, is_active FROM users WHERE username = $1',
[username]
);
if (result.rows.length === 0) {
return res.status(401).json({ error: 'Invalid username or password' });
}
const user = result.rows[0];
// Check if user is active
if (!user.is_active) {
return res.status(403).json({ error: 'Account is inactive' });
}
// Verify password
const validPassword = await bcrypt.compare(password, user.password);
if (!validPassword) {
return res.status(401).json({ error: 'Invalid username or password' });
}
// Update last login
await pool.query(
'UPDATE users SET last_login = CURRENT_TIMESTAMP WHERE id = $1',
[user.id]
);
// Generate JWT
const token = jwt.sign(
{ userId: user.id, username: user.username },
process.env.JWT_SECRET,
{ expiresIn: '8h' }
);
// Get user permissions
const permissions = await getUserPermissions(user.id);
res.json({
token,
user: {
id: user.id,
username: user.username,
is_admin: user.is_admin,
permissions
}
});
} catch (error) {
console.error('Login error:', error);
res.status(500).json({ error: 'Server error' });
}
});
// Get current user
router.get('/me', authenticate, async (req, res) => {
try {
// Get user permissions
const permissions = await getUserPermissions(req.user.id);
res.json({
id: req.user.id,
username: req.user.username,
is_admin: req.user.is_admin,
permissions
});
} catch (error) {
console.error('Error getting current user:', error);
res.status(500).json({ error: 'Server error' });
}
});
// Get all users
router.get('/users', authenticate, requirePermission('view:users'), async (req, res) => {
try {
const result = await pool.query(`
SELECT id, username, email, is_admin, is_active, created_at, last_login
FROM users
ORDER BY username
`);
res.json(result.rows);
} catch (error) {
console.error('Error getting users:', error);
res.status(500).json({ error: 'Server error' });
}
});
// Get user with permissions
router.get('/users/:id', authenticate, requirePermission('view:users'), async (req, res) => {
try {
const userId = req.params.id;
// Get user details
const userResult = await pool.query(`
SELECT id, username, email, is_admin, is_active, created_at, last_login
FROM users
WHERE id = $1
`, [userId]);
if (userResult.rows.length === 0) {
return res.status(404).json({ error: 'User not found' });
}
// Get user permissions
const permissionsResult = await pool.query(`
SELECT p.id, p.name, p.code, p.category, p.description
FROM permissions p
JOIN user_permissions up ON p.id = up.permission_id
WHERE up.user_id = $1
ORDER BY p.category, p.name
`, [userId]);
// Combine user and permissions
const user = {
...userResult.rows[0],
permissions: permissionsResult.rows
};
res.json(user);
} catch (error) {
console.error('Error getting user:', error);
res.status(500).json({ error: 'Server error' });
}
});
// Create new user
router.post('/users', authenticate, requirePermission('create:users'), async (req, res) => {
const client = await pool.connect();
try {
const { username, email, password, is_admin, is_active, permissions } = req.body;
console.log("Create user request:", {
username,
email,
is_admin,
is_active,
permissions: permissions || []
});
// Validate required fields
if (!username || !password) {
return res.status(400).json({ error: 'Username and password are required' });
}
// Check if username is taken
const existingUser = await client.query(
'SELECT id FROM users WHERE username = $1',
[username]
);
if (existingUser.rows.length > 0) {
return res.status(400).json({ error: 'Username already exists' });
}
// Start transaction
await client.query('BEGIN');
// Hash password
const saltRounds = 10;
const hashedPassword = await bcrypt.hash(password, saltRounds);
// Insert new user
const userResult = await client.query(`
INSERT INTO users (username, email, password, is_admin, is_active, created_at)
VALUES ($1, $2, $3, $4, $5, CURRENT_TIMESTAMP)
RETURNING id
`, [username, email || null, hashedPassword, !!is_admin, is_active !== false]);
const userId = userResult.rows[0].id;
// Assign permissions if provided and not admin
if (!is_admin && Array.isArray(permissions) && permissions.length > 0) {
console.log("Adding permissions for new user:", userId);
console.log("Permissions received:", permissions);
// Check permission format
const permissionIds = permissions.map(p => {
if (typeof p === 'object' && p.id) {
console.log("Permission is an object with ID:", p.id);
return parseInt(p.id, 10);
} else if (typeof p === 'number') {
console.log("Permission is a number:", p);
return p;
} else if (typeof p === 'string' && !isNaN(parseInt(p, 10))) {
console.log("Permission is a string that can be parsed as a number:", p);
return parseInt(p, 10);
} else {
console.log("Unknown permission format:", typeof p, p);
// If it's a permission code, we need to look up the ID
return null;
}
}).filter(id => id !== null);
console.log("Filtered permission IDs:", permissionIds);
if (permissionIds.length > 0) {
const permissionValues = permissionIds
.map(permId => `(${userId}, ${permId})`)
.join(',');
console.log("Inserting permission values:", permissionValues);
try {
await client.query(`
INSERT INTO user_permissions (user_id, permission_id)
VALUES ${permissionValues}
ON CONFLICT DO NOTHING
`);
console.log("Successfully inserted permissions for new user:", userId);
} catch (err) {
console.error("Error inserting permissions for new user:", err);
throw err;
}
} else {
console.log("No valid permission IDs found to insert for new user");
}
} else {
console.log("Not adding permissions: is_admin =", is_admin, "permissions array:", Array.isArray(permissions), "length:", permissions ? permissions.length : 0);
}
await client.query('COMMIT');
res.status(201).json({
id: userId,
message: 'User created successfully'
});
} catch (error) {
await client.query('ROLLBACK');
console.error('Error creating user:', error);
res.status(500).json({ error: 'Server error' });
} finally {
client.release();
}
});
// Update user
router.put('/users/:id', authenticate, requirePermission('edit:users'), async (req, res) => {
const client = await pool.connect();
try {
const userId = req.params.id;
const { username, email, password, is_admin, is_active, permissions } = req.body;
console.log("Update user request:", {
userId,
username,
email,
is_admin,
is_active,
permissions: permissions || []
});
// Check if user exists
const userExists = await client.query(
'SELECT id FROM users WHERE id = $1',
[userId]
);
if (userExists.rows.length === 0) {
return res.status(404).json({ error: 'User not found' });
}
// Start transaction
await client.query('BEGIN');
// Build update fields
const updateFields = [];
const updateValues = [userId]; // First parameter is the user ID
let paramIndex = 2;
if (username !== undefined) {
updateFields.push(`username = $${paramIndex++}`);
updateValues.push(username);
}
if (email !== undefined) {
updateFields.push(`email = $${paramIndex++}`);
updateValues.push(email || null);
}
if (is_admin !== undefined) {
updateFields.push(`is_admin = $${paramIndex++}`);
updateValues.push(!!is_admin);
}
if (is_active !== undefined) {
updateFields.push(`is_active = $${paramIndex++}`);
updateValues.push(!!is_active);
}
// Update password if provided
if (password) {
const saltRounds = 10;
const hashedPassword = await bcrypt.hash(password, saltRounds);
updateFields.push(`password = $${paramIndex++}`);
updateValues.push(hashedPassword);
}
// Update user if there are fields to update
if (updateFields.length > 0) {
updateFields.push(`updated_at = CURRENT_TIMESTAMP`);
await client.query(`
UPDATE users
SET ${updateFields.join(', ')}
WHERE id = $1
`, updateValues);
}
// Update permissions if provided
if (Array.isArray(permissions)) {
console.log("Updating permissions for user:", userId);
console.log("Permissions received:", permissions);
// First remove existing permissions
await client.query(
'DELETE FROM user_permissions WHERE user_id = $1',
[userId]
);
console.log("Deleted existing permissions for user:", userId);
// Add new permissions if any and not admin
const newIsAdmin = is_admin !== undefined ? is_admin : (await client.query('SELECT is_admin FROM users WHERE id = $1', [userId])).rows[0].is_admin;
console.log("User is admin:", newIsAdmin);
if (!newIsAdmin && permissions.length > 0) {
console.log("Adding permissions:", permissions);
// Check permission format
const permissionIds = permissions.map(p => {
if (typeof p === 'object' && p.id) {
console.log("Permission is an object with ID:", p.id);
return parseInt(p.id, 10);
} else if (typeof p === 'number') {
console.log("Permission is a number:", p);
return p;
} else if (typeof p === 'string' && !isNaN(parseInt(p, 10))) {
console.log("Permission is a string that can be parsed as a number:", p);
return parseInt(p, 10);
} else {
console.log("Unknown permission format:", typeof p, p);
// If it's a permission code, we need to look up the ID
return null;
}
}).filter(id => id !== null);
console.log("Filtered permission IDs:", permissionIds);
if (permissionIds.length > 0) {
const permissionValues = permissionIds
.map(permId => `(${userId}, ${permId})`)
.join(',');
console.log("Inserting permission values:", permissionValues);
try {
await client.query(`
INSERT INTO user_permissions (user_id, permission_id)
VALUES ${permissionValues}
ON CONFLICT DO NOTHING
`);
console.log("Successfully inserted permissions for user:", userId);
} catch (err) {
console.error("Error inserting permissions:", err);
throw err;
}
} else {
console.log("No valid permission IDs found to insert");
}
}
}
await client.query('COMMIT');
res.json({ message: 'User updated successfully' });
} catch (error) {
await client.query('ROLLBACK');
console.error('Error updating user:', error);
res.status(500).json({ error: 'Server error' });
} finally {
client.release();
}
});
// Delete user
router.delete('/users/:id', authenticate, requirePermission('delete:users'), async (req, res) => {
try {
const userId = req.params.id;
// Check that user is not deleting themselves
if (req.user.id === parseInt(userId, 10)) {
return res.status(400).json({ error: 'Cannot delete your own account' });
}
// Delete user (this will cascade to user_permissions due to FK constraints)
const result = await pool.query(
'DELETE FROM users WHERE id = $1 RETURNING id',
[userId]
);
if (result.rows.length === 0) {
return res.status(404).json({ error: 'User not found' });
}
res.json({ message: 'User deleted successfully' });
} catch (error) {
console.error('Error deleting user:', error);
res.status(500).json({ error: 'Server error' });
}
});
// Get all permissions grouped by category
router.get('/permissions/categories', authenticate, requirePermission('view:users'), async (req, res) => {
try {
const result = await pool.query(`
SELECT category, json_agg(
json_build_object(
'id', id,
'name', name,
'code', code,
'description', description
) ORDER BY name
) as permissions
FROM permissions
GROUP BY category
ORDER BY category
`);
res.json(result.rows);
} catch (error) {
console.error('Error getting permissions:', error);
res.status(500).json({ error: 'Server error' });
}
});
// Get all permissions
router.get('/permissions', authenticate, requirePermission('view:users'), async (req, res) => {
try {
const result = await pool.query(`
SELECT *
FROM permissions
ORDER BY category, name
`);
res.json(result.rows);
} catch (error) {
console.error('Error getting permissions:', error);
res.status(500).json({ error: 'Server error' });
}
});
module.exports = router;

95
inventory-server/auth/schema.sql
View File

@@ -1,89 +1,6 @@
CREATE TABLE users (
id SERIAL PRIMARY KEY,
username VARCHAR(255) NOT NULL UNIQUE,
password VARCHAR(255) NOT NULL,
email VARCHAR UNIQUE,
is_admin BOOLEAN DEFAULT FALSE,
is_active BOOLEAN DEFAULT TRUE,
last_login TIMESTAMP WITH TIME ZONE,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
-- Function to update the updated_at timestamp
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- Sequence and defined type for users table if not exists
CREATE SEQUENCE IF NOT EXISTS users_id_seq;
-- Create permissions table
CREATE TABLE IF NOT EXISTS "public"."permissions" (
"id" SERIAL PRIMARY KEY,
"name" varchar NOT NULL UNIQUE,
"code" varchar NOT NULL UNIQUE,
"description" text,
"category" varchar NOT NULL,
"created_at" timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
"updated_at" timestamp with time zone DEFAULT CURRENT_TIMESTAMP
);
-- Create user_permissions junction table
CREATE TABLE IF NOT EXISTS "public"."user_permissions" (
"user_id" int4 NOT NULL REFERENCES "public"."users"("id") ON DELETE CASCADE,
"permission_id" int4 NOT NULL REFERENCES "public"."permissions"("id") ON DELETE CASCADE,
"created_at" timestamp with time zone DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY ("user_id", "permission_id")
);
-- Add triggers for updated_at on users and permissions
DROP TRIGGER IF EXISTS update_users_updated_at ON users;
CREATE TRIGGER update_users_updated_at
BEFORE UPDATE ON users
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
DROP TRIGGER IF EXISTS update_permissions_updated_at ON permissions;
CREATE TRIGGER update_permissions_updated_at
BEFORE UPDATE ON permissions
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- Insert default permissions by page - only the ones used in application
INSERT INTO permissions (name, code, description, category) VALUES
('Dashboard Access', 'access:dashboard', 'Can access the Dashboard page', 'Pages'),
('Products Access', 'access:products', 'Can access the Products page', 'Pages'),
('Categories Access', 'access:categories', 'Can access the Categories page', 'Pages'),
('Vendors Access', 'access:vendors', 'Can access the Vendors page', 'Pages'),
('Analytics Access', 'access:analytics', 'Can access the Analytics page', 'Pages'),
('Forecasting Access', 'access:forecasting', 'Can access the Forecasting page', 'Pages'),
('Purchase Orders Access', 'access:purchase_orders', 'Can access the Purchase Orders page', 'Pages'),
('Import Access', 'access:import', 'Can access the Import page', 'Pages'),
('Settings Access', 'access:settings', 'Can access the Settings page', 'Pages'),
('AI Validation Debug Access', 'access:ai_validation_debug', 'Can access the AI Validation Debug page', 'Pages')
ON CONFLICT (code) DO NOTHING;
-- Settings section permissions
INSERT INTO permissions (name, code, description, category) VALUES
('Data Management', 'settings:data_management', 'Access to the Data Management settings section', 'Settings'),
('Stock Management', 'settings:stock_management', 'Access to the Stock Management settings section', 'Settings'),
('Performance Metrics', 'settings:performance_metrics', 'Access to the Performance Metrics settings section', 'Settings'),
('Calculation Settings', 'settings:calculation_settings', 'Access to the Calculation Settings section', 'Settings'),
('Template Management', 'settings:templates', 'Access to the Template Management settings section', 'Settings'),
('User Management', 'settings:user_management', 'Access to the User Management settings section', 'Settings')
ON CONFLICT (code) DO NOTHING;
-- Set any existing users as admin
UPDATE users SET is_admin = TRUE WHERE is_admin IS NULL;
-- Grant all permissions to admin users
INSERT INTO user_permissions (user_id, permission_id)
SELECT u.id, p.id
FROM users u, permissions p
WHERE u.is_admin = TRUE
ON CONFLICT DO NOTHING;
CREATE TABLE `users` (
`id` INT AUTO_INCREMENT PRIMARY KEY,
`username` VARCHAR(255) NOT NULL UNIQUE,
`password` VARCHAR(255) NOT NULL,
`created_at` TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);

225
inventory-server/auth/server.js
View File

@@ -1,164 +1,135 @@
require('dotenv').config({ path: '../.env' });
const express = require('express');
const cors = require('cors');
const bcrypt = require('bcrypt');
const jwt = require('jsonwebtoken');
const { Pool } = require('pg');
const morgan = require('morgan');
const authRoutes = require('./routes');
// Log startup configuration
console.log('Starting auth server with config:', {
host: process.env.DB_HOST,
user: process.env.DB_USER,
database: process.env.DB_NAME,
port: process.env.DB_PORT,
auth_port: process.env.AUTH_PORT
});
const cors = require('cors');
const mysql = require('mysql2/promise');
require('dotenv').config({ path: '../.env' });
const app = express();
const port = process.env.AUTH_PORT || 3011;
const PORT = process.env.AUTH_PORT || 3011;
// Database configuration
const pool = new Pool({
const dbConfig = {
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
port: process.env.DB_PORT,
};
// Create a connection pool
const pool = mysql.createPool(dbConfig);
app.use(cors({
origin: [
'https://inventory.kent.pw',
'http://localhost:5173',
'http://127.0.0.1:5173',
/^http:\/\/192\.168\.\d+\.\d+(:\d+)?$/,
/^http:\/\/10\.\d+\.\d+\.\d+(:\d+)?$/
],
methods: ['GET', 'POST', 'OPTIONS'],
allowedHeaders: ['Content-Type', 'Authorization', 'X-Requested-With'],
credentials: true,
exposedHeaders: ['set-cookie']
}));
app.use(express.json());
// Debug middleware to log request details
app.use((req, res, next) => {
console.log('Request details:', {
method: req.method,
url: req.url,
origin: req.get('Origin'),
headers: req.headers,
body: req.body,
});
next();
});
// Make pool available globally
global.pool = pool;
// Registration endpoint
app.post('/register', async (req, res) => {
try {
const { username, password } = req.body;
const hashedPassword = await bcrypt.hash(password, 10);
// Middleware
app.use(express.json());
app.use(morgan('combined'));
app.use(cors({
origin: ['http://localhost:5173', 'http://localhost:5174', 'https://inventory.kent.pw'],
credentials: true
}));
const connection = await pool.getConnection();
await connection.query('INSERT INTO users (username, password) VALUES (?, ?)', [username, hashedPassword]);
connection.release();
res.status(201).json({ message: 'User registered successfully' });
} catch (error) {
console.error('Registration error:', error);
res.status(500).json({ error: 'Registration failed' });
}
});
// Login endpoint
app.post('/login', async (req, res) => {
const { username, password } = req.body;
try {
// Get user from database
const result = await pool.query(
'SELECT id, username, password, is_admin, is_active FROM users WHERE username = $1',
[username]
const { username, password } = req.body;
console.log(`Login attempt for user: ${username}`);
const connection = await pool.getConnection();
const [rows] = await connection.query(
'SELECT * FROM users WHERE username = ?',
[username],
);
connection.release();
const user = result.rows[0];
if (rows.length === 1) {
const user = rows[0];
const passwordMatch = await bcrypt.compare(password, user.password);
// Check if user exists and password is correct
if (!user || !(await bcrypt.compare(password, user.password))) {
return res.status(401).json({ error: 'Invalid username or password' });
}
// Check if user is active
if (!user.is_active) {
return res.status(403).json({ error: 'Account is inactive' });
}
// Generate JWT token
const token = jwt.sign(
{ userId: user.id, username: user.username },
process.env.JWT_SECRET,
{ expiresIn: '24h' }
);
// Get user permissions for the response
const permissionsResult = await pool.query(`
SELECT code
FROM permissions p
JOIN user_permissions up ON p.id = up.permission_id
WHERE up.user_id = $1
`, [user.id]);
const permissions = permissionsResult.rows.map(row => row.code);
res.json({
token,
user: {
id: user.id,
username: user.username,
is_admin: user.is_admin,
permissions: user.is_admin ? [] : permissions
if (passwordMatch) {
console.log(`User ${username} authenticated successfully`);
const token = jwt.sign(
{ username: user.username },
process.env.JWT_SECRET,
{ expiresIn: '1h' },
);
res.json({ token });
} else {
console.error(`Invalid password for user: ${username}`);
res.status(401).json({ error: 'Invalid credentials' });
}
});
} else {
console.error(`User not found: ${username}`);
res.status(401).json({ error: 'Invalid credentials' });
}
} catch (error) {
console.error('Login error:', error);
res.status(500).json({ error: 'Internal server error' });
res.status(500).json({ error: 'Login failed' });
}
});
// User info endpoint
app.get('/me', async (req, res) => {
// Protected endpoint example
app.get('/protected', async (req, res) => {
const authHeader = req.headers.authorization;
if (!authHeader || !authHeader.startsWith('Bearer ')) {
return res.status(401).json({ error: 'No token provided' });
if (!authHeader) {
return res.status(401).json({ error: 'Unauthorized' });
}
const token = authHeader.split(' ')[1];
try {
const token = authHeader.split(' ')[1];
const decoded = jwt.verify(token, process.env.JWT_SECRET);
// Get user details from database
const userResult = await pool.query(
'SELECT id, username, email, is_admin, is_active FROM users WHERE id = $1',
[decoded.userId]
);
if (userResult.rows.length === 0) {
return res.status(404).json({ error: 'User not found' });
// Optionally, you can fetch the user from the database here
// to verify that the user still exists or to get more user information
const connection = await pool.getConnection();
const [rows] = await connection.query('SELECT * FROM users WHERE username = ?', [decoded.username]);
connection.release();
if (rows.length === 0) {
return res.status(401).json({ error: 'User not found' });
}
const user = userResult.rows[0];
// Get user permissions
let permissions = [];
if (!user.is_admin) {
const permissionsResult = await pool.query(`
SELECT code
FROM permissions p
JOIN user_permissions up ON p.id = up.permission_id
WHERE up.user_id = $1
`, [user.id]);
permissions = permissionsResult.rows.map(row => row.code);
}
res.json({
id: user.id,
username: user.username,
email: user.email,
is_admin: user.is_admin,
permissions: permissions
});
res.json({ message: 'Protected resource accessed', user: decoded });
} catch (error) {
console.error('Token verification error:', error);
res.status(401).json({ error: 'Invalid token' });
console.error('Protected endpoint error:', error);
res.status(403).json({ error: 'Invalid token' });
}
});
// Mount all routes from routes.js
app.use('/', authRoutes);
// Health check endpoint
app.get('/health', (req, res) => {
res.json({ status: 'healthy' });
});
// Error handling middleware
app.use((err, req, res, next) => {
console.error(err.stack);
res.status(500).json({ error: 'Something broke!' });
});
// Start server
app.listen(port, () => {
console.log(`Auth server running on port ${port}`);
});
app.listen(PORT, "0.0.0.0", () => {
console.log(`Auth server running on port ${PORT}`);
});

287
inventory-server/db/config-schema.sql
View File

@@ -1,207 +1,150 @@
-- Configuration tables schema
-- Create function for updating timestamps if it doesn't exist
CREATE OR REPLACE FUNCTION update_updated_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- Create function for updating updated_at timestamps
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- Stock threshold configurations
CREATE TABLE stock_thresholds (
id INTEGER NOT NULL,
CREATE TABLE IF NOT EXISTS stock_thresholds (
id INT NOT NULL,
category_id BIGINT, -- NULL means default/global threshold
vendor VARCHAR(100), -- NULL means applies to all vendors
critical_days INTEGER NOT NULL DEFAULT 7,
reorder_days INTEGER NOT NULL DEFAULT 14,
overstock_days INTEGER NOT NULL DEFAULT 90,
low_stock_threshold INTEGER NOT NULL DEFAULT 5,
min_reorder_quantity INTEGER NOT NULL DEFAULT 1,
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
critical_days INT NOT NULL DEFAULT 7,
reorder_days INT NOT NULL DEFAULT 14,
overstock_days INT NOT NULL DEFAULT 90,
low_stock_threshold INT NOT NULL DEFAULT 5,
min_reorder_quantity INT NOT NULL DEFAULT 1,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id),
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
UNIQUE (category_id, vendor)
UNIQUE KEY unique_category_vendor (category_id, vendor),
INDEX idx_st_metrics (category_id, vendor)
);
CREATE TRIGGER update_stock_thresholds_updated
BEFORE UPDATE ON stock_thresholds
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
CREATE INDEX idx_st_metrics ON stock_thresholds(category_id, vendor);
-- Lead time threshold configurations
CREATE TABLE lead_time_thresholds (
id INTEGER NOT NULL,
CREATE TABLE IF NOT EXISTS lead_time_thresholds (
id INT NOT NULL,
category_id BIGINT, -- NULL means default/global threshold
vendor VARCHAR(100), -- NULL means applies to all vendors
target_days INTEGER NOT NULL DEFAULT 14,
warning_days INTEGER NOT NULL DEFAULT 21,
critical_days INTEGER NOT NULL DEFAULT 30,
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
target_days INT NOT NULL DEFAULT 14,
warning_days INT NOT NULL DEFAULT 21,
critical_days INT NOT NULL DEFAULT 30,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id),
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
UNIQUE (category_id, vendor)
UNIQUE KEY unique_category_vendor (category_id, vendor)
);
CREATE TRIGGER update_lead_time_thresholds_updated
BEFORE UPDATE ON lead_time_thresholds
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- Sales velocity window configurations
CREATE TABLE sales_velocity_config (
id INTEGER NOT NULL,
CREATE TABLE IF NOT EXISTS sales_velocity_config (
id INT NOT NULL,
category_id BIGINT, -- NULL means default/global threshold
vendor VARCHAR(100), -- NULL means applies to all vendors
daily_window_days INTEGER NOT NULL DEFAULT 30,
weekly_window_days INTEGER NOT NULL DEFAULT 7,
monthly_window_days INTEGER NOT NULL DEFAULT 90,
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
daily_window_days INT NOT NULL DEFAULT 30,
weekly_window_days INT NOT NULL DEFAULT 7,
monthly_window_days INT NOT NULL DEFAULT 90,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id),
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
UNIQUE (category_id, vendor)
UNIQUE KEY unique_category_vendor (category_id, vendor),
INDEX idx_sv_metrics (category_id, vendor)
);
CREATE TRIGGER update_sales_velocity_config_updated
BEFORE UPDATE ON sales_velocity_config
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
CREATE INDEX idx_sv_metrics ON sales_velocity_config(category_id, vendor);
-- ABC Classification configurations
CREATE TABLE abc_classification_config (
id INTEGER NOT NULL PRIMARY KEY,
CREATE TABLE IF NOT EXISTS abc_classification_config (
id INT NOT NULL PRIMARY KEY,
a_threshold DECIMAL(5,2) NOT NULL DEFAULT 20.0,
b_threshold DECIMAL(5,2) NOT NULL DEFAULT 50.0,
classification_period_days INTEGER NOT NULL DEFAULT 90,
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
classification_period_days INT NOT NULL DEFAULT 90,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
);
CREATE TRIGGER update_abc_classification_config_updated
BEFORE UPDATE ON abc_classification_config
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- Safety stock configurations
CREATE TABLE safety_stock_config (
id INTEGER NOT NULL,
CREATE TABLE IF NOT EXISTS safety_stock_config (
id INT NOT NULL,
category_id BIGINT, -- NULL means default/global threshold
vendor VARCHAR(100), -- NULL means applies to all vendors
coverage_days INTEGER NOT NULL DEFAULT 14,
coverage_days INT NOT NULL DEFAULT 14,
service_level DECIMAL(5,2) NOT NULL DEFAULT 95.0,
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id),
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
UNIQUE (category_id, vendor)
UNIQUE KEY unique_category_vendor (category_id, vendor),
INDEX idx_ss_metrics (category_id, vendor)
);
CREATE TRIGGER update_safety_stock_config_updated
BEFORE UPDATE ON safety_stock_config
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
CREATE INDEX idx_ss_metrics ON safety_stock_config(category_id, vendor);
-- Turnover rate configurations
CREATE TABLE turnover_config (
id INTEGER NOT NULL,
CREATE TABLE IF NOT EXISTS turnover_config (
id INT NOT NULL,
category_id BIGINT, -- NULL means default/global threshold
vendor VARCHAR(100), -- NULL means applies to all vendors
calculation_period_days INTEGER NOT NULL DEFAULT 30,
calculation_period_days INT NOT NULL DEFAULT 30,
target_rate DECIMAL(10,2) NOT NULL DEFAULT 1.0,
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id),
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
UNIQUE (category_id, vendor)
UNIQUE KEY unique_category_vendor (category_id, vendor)
);
CREATE TRIGGER update_turnover_config_updated
BEFORE UPDATE ON turnover_config
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- Create table for sales seasonality factors
CREATE TABLE sales_seasonality (
month INTEGER NOT NULL,
CREATE TABLE IF NOT EXISTS sales_seasonality (
month INT NOT NULL,
seasonality_factor DECIMAL(5,3) DEFAULT 0,
last_updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
last_updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (month),
CONSTRAINT month_range CHECK (month BETWEEN 1 AND 12),
CONSTRAINT seasonality_range CHECK (seasonality_factor BETWEEN -1.0 AND 1.0)
CHECK (month BETWEEN 1 AND 12),
CHECK (seasonality_factor BETWEEN -1.0 AND 1.0)
);
CREATE TRIGGER update_sales_seasonality_updated
BEFORE UPDATE ON sales_seasonality
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- Insert default global thresholds
-- Insert default global thresholds if not exists
INSERT INTO stock_thresholds (id, category_id, vendor, critical_days, reorder_days, overstock_days)
VALUES (1, NULL, NULL, 7, 14, 90)
ON CONFLICT (id) DO UPDATE SET
critical_days = EXCLUDED.critical_days,
reorder_days = EXCLUDED.reorder_days,
overstock_days = EXCLUDED.overstock_days;
ON DUPLICATE KEY UPDATE
critical_days = VALUES(critical_days),
reorder_days = VALUES(reorder_days),
overstock_days = VALUES(overstock_days);
INSERT INTO lead_time_thresholds (id, category_id, vendor, target_days, warning_days, critical_days)
VALUES (1, NULL, NULL, 14, 21, 30)
ON CONFLICT (id) DO UPDATE SET
target_days = EXCLUDED.target_days,
warning_days = EXCLUDED.warning_days,
critical_days = EXCLUDED.critical_days;
ON DUPLICATE KEY UPDATE
target_days = VALUES(target_days),
warning_days = VALUES(warning_days),
critical_days = VALUES(critical_days);
INSERT INTO sales_velocity_config (id, category_id, vendor, daily_window_days, weekly_window_days, monthly_window_days)
VALUES (1, NULL, NULL, 30, 7, 90)
ON CONFLICT (id) DO UPDATE SET
daily_window_days = EXCLUDED.daily_window_days,
weekly_window_days = EXCLUDED.weekly_window_days,
monthly_window_days = EXCLUDED.monthly_window_days;
ON DUPLICATE KEY UPDATE
daily_window_days = VALUES(daily_window_days),
weekly_window_days = VALUES(weekly_window_days),
monthly_window_days = VALUES(monthly_window_days);
INSERT INTO abc_classification_config (id, a_threshold, b_threshold, classification_period_days)
VALUES (1, 20.0, 50.0, 90)
ON CONFLICT (id) DO UPDATE SET
a_threshold = EXCLUDED.a_threshold,
b_threshold = EXCLUDED.b_threshold,
classification_period_days = EXCLUDED.classification_period_days;
ON DUPLICATE KEY UPDATE
a_threshold = VALUES(a_threshold),
b_threshold = VALUES(b_threshold),
classification_period_days = VALUES(classification_period_days);
INSERT INTO safety_stock_config (id, category_id, vendor, coverage_days, service_level)
VALUES (1, NULL, NULL, 14, 95.0)
ON CONFLICT (id) DO UPDATE SET
coverage_days = EXCLUDED.coverage_days,
service_level = EXCLUDED.service_level;
ON DUPLICATE KEY UPDATE
coverage_days = VALUES(coverage_days),
service_level = VALUES(service_level);
INSERT INTO turnover_config (id, category_id, vendor, calculation_period_days, target_rate)
VALUES (1, NULL, NULL, 30, 1.0)
ON CONFLICT (id) DO UPDATE SET
calculation_period_days = EXCLUDED.calculation_period_days,
target_rate = EXCLUDED.target_rate;
ON DUPLICATE KEY UPDATE
calculation_period_days = VALUES(calculation_period_days),
target_rate = VALUES(target_rate);
-- Insert default seasonality factors (neutral)
INSERT INTO sales_seasonality (month, seasonality_factor)
VALUES
(1, 0), (2, 0), (3, 0), (4, 0), (5, 0), (6, 0),
(7, 0), (8, 0), (9, 0), (10, 0), (11, 0), (12, 0)
ON CONFLICT (month) DO UPDATE SET
last_updated = CURRENT_TIMESTAMP;
ON DUPLICATE KEY UPDATE last_updated = CURRENT_TIMESTAMP;
-- View to show thresholds with category names
CREATE OR REPLACE VIEW stock_thresholds_view AS
@@ -210,9 +153,9 @@ SELECT
c.name as category_name,
CASE
WHEN st.category_id IS NULL AND st.vendor IS NULL THEN 'Global Default'
WHEN st.category_id IS NULL THEN 'Vendor: ' || st.vendor
WHEN st.vendor IS NULL THEN 'Category: ' || c.name
ELSE 'Category: ' || c.name || ' / Vendor: ' || st.vendor
WHEN st.category_id IS NULL THEN CONCAT('Vendor: ', st.vendor)
WHEN st.vendor IS NULL THEN CONCAT('Category: ', c.name)
ELSE CONCAT('Category: ', c.name, ' / Vendor: ', st.vendor)
END as threshold_scope
FROM
stock_thresholds st
@@ -228,51 +171,59 @@ ORDER BY
c.name,
st.vendor;
-- History and status tables
CREATE TABLE IF NOT EXISTS calculate_history (
id BIGSERIAL PRIMARY KEY,
start_time TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
end_time TIMESTAMP WITH TIME ZONE NULL,
duration_seconds INTEGER,
duration_minutes DECIMAL(10,2) GENERATED ALWAYS AS (duration_seconds::decimal / 60.0) STORED,
total_products INTEGER DEFAULT 0,
total_orders INTEGER DEFAULT 0,
total_purchase_orders INTEGER DEFAULT 0,
processed_products INTEGER DEFAULT 0,
processed_orders INTEGER DEFAULT 0,
processed_purchase_orders INTEGER DEFAULT 0,
status calculation_status DEFAULT 'running',
id BIGINT AUTO_INCREMENT PRIMARY KEY,
start_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
end_time TIMESTAMP NULL,
duration_seconds INT,
duration_minutes DECIMAL(10,2) GENERATED ALWAYS AS (duration_seconds / 60.0) STORED,
total_products INT DEFAULT 0,
total_orders INT DEFAULT 0,
total_purchase_orders INT DEFAULT 0,
processed_products INT DEFAULT 0,
processed_orders INT DEFAULT 0,
processed_purchase_orders INT DEFAULT 0,
status ENUM('running', 'completed', 'failed', 'cancelled') DEFAULT 'running',
error_message TEXT,
additional_info JSONB
additional_info JSON,
INDEX idx_status_time (status, start_time)
);
CREATE TABLE IF NOT EXISTS calculate_status (
module_name module_name PRIMARY KEY,
last_calculation_timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
module_name ENUM(
'product_metrics',
'time_aggregates',
'financial_metrics',
'vendor_metrics',
'category_metrics',
'brand_metrics',
'sales_forecasts',
'abc_classification'
) PRIMARY KEY,
last_calculation_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
INDEX idx_last_calc (last_calculation_timestamp)
);
CREATE TABLE IF NOT EXISTS sync_status (
table_name VARCHAR(50) PRIMARY KEY,
last_sync_timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
last_sync_id BIGINT
last_sync_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
last_sync_id BIGINT,
INDEX idx_last_sync (last_sync_timestamp)
);
CREATE TABLE IF NOT EXISTS import_history (
id BIGSERIAL PRIMARY KEY,
id BIGINT AUTO_INCREMENT PRIMARY KEY,
table_name VARCHAR(50) NOT NULL,
start_time TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
end_time TIMESTAMP WITH TIME ZONE NULL,
duration_seconds INTEGER,
duration_minutes DECIMAL(10,2) GENERATED ALWAYS AS (duration_seconds::decimal / 60.0) STORED,
records_added INTEGER DEFAULT 0,
records_updated INTEGER DEFAULT 0,
start_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
end_time TIMESTAMP NULL,
duration_seconds INT,
duration_minutes DECIMAL(10,2) GENERATED ALWAYS AS (duration_seconds / 60.0) STORED,
records_added INT DEFAULT 0,
records_updated INT DEFAULT 0,
is_incremental BOOLEAN DEFAULT FALSE,
status calculation_status DEFAULT 'running',
status ENUM('running', 'completed', 'failed', 'cancelled') DEFAULT 'running',
error_message TEXT,
additional_info JSONB
);
-- Create all indexes after tables are fully created
CREATE INDEX IF NOT EXISTS idx_last_calc ON calculate_status(last_calculation_timestamp);
CREATE INDEX IF NOT EXISTS idx_last_sync ON sync_status(last_sync_timestamp);
CREATE INDEX IF NOT EXISTS idx_table_time ON import_history(table_name, start_time);
additional_info JSON,
INDEX idx_table_time (table_name, start_time),
INDEX idx_status (status)
);

431
inventory-server/db/metrics-schema.sql
View File

@@ -1,8 +1,8 @@
-- Disable foreign key checks
SET session_replication_role = 'replica';
SET FOREIGN_KEY_CHECKS = 0;
-- Temporary tables for batch metrics processing
CREATE TABLE temp_sales_metrics (
CREATE TABLE IF NOT EXISTS temp_sales_metrics (
pid BIGINT NOT NULL,
daily_sales_avg DECIMAL(10,3),
weekly_sales_avg DECIMAL(10,3),
@@ -14,9 +14,9 @@ CREATE TABLE temp_sales_metrics (
PRIMARY KEY (pid)
);
CREATE TABLE temp_purchase_metrics (
CREATE TABLE IF NOT EXISTS temp_purchase_metrics (
pid BIGINT NOT NULL,
avg_lead_time_days INTEGER,
avg_lead_time_days INT,
last_purchase_date DATE,
first_received_date DATE,
last_received_date DATE,
@@ -24,7 +24,7 @@ CREATE TABLE temp_purchase_metrics (
);
-- New table for product metrics
CREATE TABLE product_metrics (
CREATE TABLE IF NOT EXISTS product_metrics (
pid BIGINT NOT NULL,
last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-- Sales velocity metrics
@@ -32,16 +32,16 @@ CREATE TABLE product_metrics (
weekly_sales_avg DECIMAL(10,3),
monthly_sales_avg DECIMAL(10,3),
avg_quantity_per_order DECIMAL(10,3),
number_of_orders INTEGER,
number_of_orders INT,
first_sale_date DATE,
last_sale_date DATE,
-- Stock metrics
days_of_inventory INTEGER,
weeks_of_inventory INTEGER,
reorder_point INTEGER,
safety_stock INTEGER,
reorder_qty INTEGER DEFAULT 0,
overstocked_amt INTEGER DEFAULT 0,
days_of_inventory INT,
weeks_of_inventory INT,
reorder_point INT,
safety_stock INT,
reorder_qty INT DEFAULT 0,
overstocked_amt INT DEFAULT 0,
-- Financial metrics
avg_margin_percent DECIMAL(10,3),
total_revenue DECIMAL(10,3),
@@ -50,7 +50,7 @@ CREATE TABLE product_metrics (
gross_profit DECIMAL(10,3),
gmroi DECIMAL(10,3),
-- Purchase metrics
avg_lead_time_days INTEGER,
avg_lead_time_days INT,
last_purchase_date DATE,
first_received_date DATE,
last_received_date DATE,
@@ -60,99 +60,97 @@ CREATE TABLE product_metrics (
-- Turnover metrics
turnover_rate DECIMAL(12,3),
-- Lead time metrics
current_lead_time INTEGER,
target_lead_time INTEGER,
current_lead_time INT,
target_lead_time INT,
lead_time_status VARCHAR(20),
-- Forecast metrics
forecast_accuracy DECIMAL(5,2) DEFAULT NULL,
forecast_bias DECIMAL(5,2) DEFAULT NULL,
last_forecast_date DATE DEFAULT NULL,
PRIMARY KEY (pid),
FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE
FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE,
INDEX idx_metrics_revenue (total_revenue),
INDEX idx_metrics_stock_status (stock_status),
INDEX idx_metrics_lead_time (lead_time_status),
INDEX idx_metrics_turnover (turnover_rate),
INDEX idx_metrics_last_calculated (last_calculated_at),
INDEX idx_metrics_abc (abc_class),
INDEX idx_metrics_sales (daily_sales_avg, weekly_sales_avg, monthly_sales_avg),
INDEX idx_metrics_forecast (forecast_accuracy, forecast_bias)
);
CREATE INDEX idx_metrics_revenue ON product_metrics(total_revenue);
CREATE INDEX idx_metrics_stock_status ON product_metrics(stock_status);
CREATE INDEX idx_metrics_lead_time ON product_metrics(lead_time_status);
CREATE INDEX idx_metrics_turnover ON product_metrics(turnover_rate);
CREATE INDEX idx_metrics_last_calculated ON product_metrics(last_calculated_at);
CREATE INDEX idx_metrics_abc ON product_metrics(abc_class);
CREATE INDEX idx_metrics_sales ON product_metrics(daily_sales_avg, weekly_sales_avg, monthly_sales_avg);
CREATE INDEX idx_metrics_forecast ON product_metrics(forecast_accuracy, forecast_bias);
-- New table for time-based aggregates
CREATE TABLE product_time_aggregates (
CREATE TABLE IF NOT EXISTS product_time_aggregates (
pid BIGINT NOT NULL,
year INTEGER NOT NULL,
month INTEGER NOT NULL,
year INT NOT NULL,
month INT NOT NULL,
-- Sales metrics
total_quantity_sold INTEGER DEFAULT 0,
total_quantity_sold INT DEFAULT 0,
total_revenue DECIMAL(10,3) DEFAULT 0,
total_cost DECIMAL(10,3) DEFAULT 0,
order_count INTEGER DEFAULT 0,
order_count INT DEFAULT 0,
-- Stock changes
stock_received INTEGER DEFAULT 0,
stock_ordered INTEGER DEFAULT 0,
stock_received INT DEFAULT 0,
stock_ordered INT DEFAULT 0,
-- Calculated fields
avg_price DECIMAL(10,3),
profit_margin DECIMAL(10,3),
inventory_value DECIMAL(10,3),
gmroi DECIMAL(10,3),
PRIMARY KEY (pid, year, month),
FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE
FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE,
INDEX idx_date (year, month)
);
CREATE INDEX idx_date ON product_time_aggregates(year, month);
-- Create vendor_details table
CREATE TABLE vendor_details (
vendor VARCHAR(100) PRIMARY KEY,
-- Create vendor details table
CREATE TABLE IF NOT EXISTS vendor_details (
vendor VARCHAR(100) NOT NULL,
contact_name VARCHAR(100),
email VARCHAR(255),
phone VARCHAR(50),
email VARCHAR(100),
phone VARCHAR(20),
status VARCHAR(20) DEFAULT 'active',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
notes TEXT,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (vendor),
INDEX idx_vendor_status (status)
);
CREATE INDEX idx_vendor_details_status ON vendor_details(status);
-- New table for vendor metrics
CREATE TABLE vendor_metrics (
CREATE TABLE IF NOT EXISTS vendor_metrics (
vendor VARCHAR(100) NOT NULL,
last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-- Performance metrics
avg_lead_time_days DECIMAL(10,3),
on_time_delivery_rate DECIMAL(5,2),
order_fill_rate DECIMAL(5,2),
total_orders INTEGER DEFAULT 0,
total_late_orders INTEGER DEFAULT 0,
total_purchase_value DECIMAL(10,3) DEFAULT 0,
avg_order_value DECIMAL(10,3),
total_orders INT DEFAULT 0,
total_late_orders INT DEFAULT 0,
total_purchase_value DECIMAL(15,3) DEFAULT 0,
avg_order_value DECIMAL(15,3),
-- Product metrics
active_products INTEGER DEFAULT 0,
total_products INTEGER DEFAULT 0,
active_products INT DEFAULT 0,
total_products INT DEFAULT 0,
-- Financial metrics
total_revenue DECIMAL(10,3) DEFAULT 0,
total_revenue DECIMAL(15,3) DEFAULT 0,
avg_margin_percent DECIMAL(5,2),
-- Status
status VARCHAR(20) DEFAULT 'active',
PRIMARY KEY (vendor),
FOREIGN KEY (vendor) REFERENCES vendor_details(vendor) ON DELETE CASCADE
FOREIGN KEY (vendor) REFERENCES vendor_details(vendor) ON DELETE CASCADE,
INDEX idx_vendor_performance (on_time_delivery_rate),
INDEX idx_vendor_status (status),
INDEX idx_metrics_last_calculated (last_calculated_at),
INDEX idx_vendor_metrics_orders (total_orders, total_late_orders)
);
CREATE INDEX idx_vendor_performance ON vendor_metrics(on_time_delivery_rate);
CREATE INDEX idx_vendor_status ON vendor_metrics(status);
CREATE INDEX idx_vendor_metrics_last_calculated ON vendor_metrics(last_calculated_at);
CREATE INDEX idx_vendor_metrics_orders ON vendor_metrics(total_orders, total_late_orders);
-- New table for category metrics
CREATE TABLE category_metrics (
CREATE TABLE IF NOT EXISTS category_metrics (
category_id BIGINT NOT NULL,
last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-- Product metrics
product_count INTEGER DEFAULT 0,
active_products INTEGER DEFAULT 0,
product_count INT DEFAULT 0,
active_products INT DEFAULT 0,
-- Financial metrics
total_value DECIMAL(15,3) DEFAULT 0,
avg_margin DECIMAL(5,2),
@@ -161,215 +159,272 @@ CREATE TABLE category_metrics (
-- Status
status VARCHAR(20) DEFAULT 'active',
PRIMARY KEY (category_id),
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
INDEX idx_category_status (status),
INDEX idx_category_growth (growth_rate),
INDEX idx_metrics_last_calculated (last_calculated_at),
INDEX idx_category_metrics_products (product_count, active_products)
);
CREATE INDEX idx_category_status ON category_metrics(status);
CREATE INDEX idx_category_growth ON category_metrics(growth_rate);
CREATE INDEX idx_metrics_last_calculated_cat ON category_metrics(last_calculated_at);
CREATE INDEX idx_category_metrics_products ON category_metrics(product_count, active_products);
-- New table for vendor time-based metrics
CREATE TABLE vendor_time_metrics (
CREATE TABLE IF NOT EXISTS vendor_time_metrics (
vendor VARCHAR(100) NOT NULL,
year INTEGER NOT NULL,
month INTEGER NOT NULL,
year INT NOT NULL,
month INT NOT NULL,
-- Order metrics
total_orders INTEGER DEFAULT 0,
late_orders INTEGER DEFAULT 0,
total_orders INT DEFAULT 0,
late_orders INT DEFAULT 0,
avg_lead_time_days DECIMAL(10,3),
-- Financial metrics
total_purchase_value DECIMAL(10,3) DEFAULT 0,
total_revenue DECIMAL(10,3) DEFAULT 0,
avg_margin_percent DECIMAL(5,2),
PRIMARY KEY (vendor, year, month),
FOREIGN KEY (vendor) REFERENCES vendor_details(vendor) ON DELETE CASCADE
FOREIGN KEY (vendor) REFERENCES vendor_details(vendor) ON DELETE CASCADE,
INDEX idx_vendor_date (year, month)
);
CREATE INDEX idx_vendor_date ON vendor_time_metrics(year, month);
-- New table for category time-based metrics
CREATE TABLE category_time_metrics (
CREATE TABLE IF NOT EXISTS category_time_metrics (
category_id BIGINT NOT NULL,
year INTEGER NOT NULL,
month INTEGER NOT NULL,
year INT NOT NULL,
month INT NOT NULL,
-- Product metrics
product_count INTEGER DEFAULT 0,
active_products INTEGER DEFAULT 0,
product_count INT DEFAULT 0,
active_products INT DEFAULT 0,
-- Financial metrics
total_value DECIMAL(15,3) DEFAULT 0,
total_revenue DECIMAL(15,3) DEFAULT 0,
avg_margin DECIMAL(5,2),
turnover_rate DECIMAL(12,3),
PRIMARY KEY (category_id, year, month),
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
INDEX idx_category_date (year, month)
);
CREATE INDEX idx_category_date ON category_time_metrics(year, month);
-- New table for category-based sales metrics
CREATE TABLE category_sales_metrics (
CREATE TABLE IF NOT EXISTS category_sales_metrics (
category_id BIGINT NOT NULL,
brand VARCHAR(100) NOT NULL,
period_start DATE NOT NULL,
period_end DATE NOT NULL,
avg_daily_sales DECIMAL(10,3) DEFAULT 0,
total_sold INTEGER DEFAULT 0,
num_products INTEGER DEFAULT 0,
total_sold INT DEFAULT 0,
num_products INT DEFAULT 0,
avg_price DECIMAL(10,3) DEFAULT 0,
last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (category_id, brand, period_start, period_end),
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
INDEX idx_category_brand (category_id, brand),
INDEX idx_period (period_start, period_end)
);
CREATE INDEX idx_category_brand ON category_sales_metrics(category_id, brand);
CREATE INDEX idx_period ON category_sales_metrics(period_start, period_end);
-- New table for brand metrics
CREATE TABLE brand_metrics (
CREATE TABLE IF NOT EXISTS brand_metrics (
brand VARCHAR(100) NOT NULL,
last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-- Product metrics
product_count INTEGER DEFAULT 0,
active_products INTEGER DEFAULT 0,
product_count INT DEFAULT 0,
active_products INT DEFAULT 0,
-- Stock metrics
total_stock_units INTEGER DEFAULT 0,
total_stock_units INT DEFAULT 0,
total_stock_cost DECIMAL(15,2) DEFAULT 0,
total_stock_retail DECIMAL(15,2) DEFAULT 0,
-- Sales metrics
total_revenue DECIMAL(15,2) DEFAULT 0,
avg_margin DECIMAL(5,2) DEFAULT 0,
growth_rate DECIMAL(5,2) DEFAULT 0,
PRIMARY KEY (brand)
PRIMARY KEY (brand),
INDEX idx_brand_metrics_last_calculated (last_calculated_at),
INDEX idx_brand_metrics_revenue (total_revenue),
INDEX idx_brand_metrics_growth (growth_rate)
);
CREATE INDEX idx_brand_metrics_last_calculated ON brand_metrics(last_calculated_at);
CREATE INDEX idx_brand_metrics_revenue ON brand_metrics(total_revenue);
CREATE INDEX idx_brand_metrics_growth ON brand_metrics(growth_rate);
-- New table for brand time-based metrics
CREATE TABLE brand_time_metrics (
CREATE TABLE IF NOT EXISTS brand_time_metrics (
brand VARCHAR(100) NOT NULL,
year INTEGER NOT NULL,
month INTEGER NOT NULL,
year INT NOT NULL,
month INT NOT NULL,
-- Product metrics
product_count INTEGER DEFAULT 0,
active_products INTEGER DEFAULT 0,
product_count INT DEFAULT 0,
active_products INT DEFAULT 0,
-- Stock metrics
total_stock_units INTEGER DEFAULT 0,
total_stock_units INT DEFAULT 0,
total_stock_cost DECIMAL(15,2) DEFAULT 0,
total_stock_retail DECIMAL(15,2) DEFAULT 0,
-- Sales metrics
total_revenue DECIMAL(15,2) DEFAULT 0,
avg_margin DECIMAL(5,2) DEFAULT 0,
growth_rate DECIMAL(5,2) DEFAULT 0,
PRIMARY KEY (brand, year, month)
PRIMARY KEY (brand, year, month),
INDEX idx_brand_date (year, month)
);
CREATE INDEX idx_brand_time_date ON brand_time_metrics(year, month);
-- New table for sales forecasts
CREATE TABLE sales_forecasts (
CREATE TABLE IF NOT EXISTS sales_forecasts (
pid BIGINT NOT NULL,
forecast_date DATE NOT NULL,
forecast_quantity INTEGER,
confidence_level DECIMAL(5,2),
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
forecast_units DECIMAL(10,2) DEFAULT 0,
forecast_revenue DECIMAL(10,2) DEFAULT 0,
confidence_level DECIMAL(5,2) DEFAULT 0,
last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (pid, forecast_date),
FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE
FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE,
INDEX idx_forecast_date (forecast_date),
INDEX idx_forecast_last_calculated (last_calculated_at)
);
CREATE INDEX idx_forecast_date ON sales_forecasts(forecast_date);
-- New table for category forecasts
CREATE TABLE category_forecasts (
CREATE TABLE IF NOT EXISTS category_forecasts (
category_id BIGINT NOT NULL,
forecast_date DATE NOT NULL,
forecast_revenue DECIMAL(15,2),
forecast_units INTEGER,
confidence_level DECIMAL(5,2),
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
forecast_units DECIMAL(10,2) DEFAULT 0,
forecast_revenue DECIMAL(10,2) DEFAULT 0,
confidence_level DECIMAL(5,2) DEFAULT 0,
last_calculated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (category_id, forecast_date),
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE
FOREIGN KEY (category_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
INDEX idx_category_forecast_date (forecast_date),
INDEX idx_category_forecast_last_calculated (last_calculated_at)
);
CREATE INDEX idx_cat_forecast_date ON category_forecasts(forecast_date);
-- Create views for common calculations
-- Create view for inventory health
CREATE OR REPLACE VIEW inventory_health AS
WITH stock_levels AS (
WITH product_thresholds AS (
SELECT
p.pid,
p.title,
p.SKU,
p.stock_quantity,
p.preorder_count,
pm.daily_sales_avg,
pm.weekly_sales_avg,
pm.monthly_sales_avg,
pm.reorder_point,
pm.safety_stock,
pm.days_of_inventory,
pm.weeks_of_inventory,
pm.stock_status,
pm.abc_class,
pm.turnover_rate,
pm.avg_lead_time_days,
pm.current_lead_time,
pm.target_lead_time,
pm.lead_time_status,
p.cost_price,
p.price,
pm.inventory_value,
pm.gmroi
COALESCE(
-- Try category+vendor specific
(SELECT critical_days FROM stock_thresholds st
JOIN product_categories pc ON st.category_id = pc.cat_id
WHERE pc.pid = p.pid
AND st.vendor = p.vendor LIMIT 1),
-- Try category specific
(SELECT critical_days FROM stock_thresholds st
JOIN product_categories pc ON st.category_id = pc.cat_id
WHERE pc.pid = p.pid
AND st.vendor IS NULL LIMIT 1),
-- Try vendor specific
(SELECT critical_days FROM stock_thresholds st
WHERE st.category_id IS NULL
AND st.vendor = p.vendor LIMIT 1),
-- Fall back to default
(SELECT critical_days FROM stock_thresholds st
WHERE st.category_id IS NULL
AND st.vendor IS NULL LIMIT 1),
7
) as critical_days,
COALESCE(
-- Try category+vendor specific
(SELECT reorder_days FROM stock_thresholds st
JOIN product_categories pc ON st.category_id = pc.cat_id
WHERE pc.pid = p.pid
AND st.vendor = p.vendor LIMIT 1),
-- Try category specific
(SELECT reorder_days FROM stock_thresholds st
JOIN product_categories pc ON st.category_id = pc.cat_id
WHERE pc.pid = p.pid
AND st.vendor IS NULL LIMIT 1),
-- Try vendor specific
(SELECT reorder_days FROM stock_thresholds st
WHERE st.category_id IS NULL
AND st.vendor = p.vendor LIMIT 1),
-- Fall back to default
(SELECT reorder_days FROM stock_thresholds st
WHERE st.category_id IS NULL
AND st.vendor IS NULL LIMIT 1),
14
) as reorder_days,
COALESCE(
-- Try category+vendor specific
(SELECT overstock_days FROM stock_thresholds st
JOIN product_categories pc ON st.category_id = pc.cat_id
WHERE pc.pid = p.pid
AND st.vendor = p.vendor LIMIT 1),
-- Try category specific
(SELECT overstock_days FROM stock_thresholds st
JOIN product_categories pc ON st.category_id = pc.cat_id
WHERE pc.pid = p.pid
AND st.vendor IS NULL LIMIT 1),
-- Try vendor specific
(SELECT overstock_days FROM stock_thresholds st
WHERE st.category_id IS NULL
AND st.vendor = p.vendor LIMIT 1),
-- Fall back to default
(SELECT overstock_days FROM stock_thresholds st
WHERE st.category_id IS NULL
AND st.vendor IS NULL LIMIT 1),
90
) as overstock_days
FROM products p
LEFT JOIN product_metrics pm ON p.pid = pm.pid
WHERE p.managing_stock = true AND p.visible = true
)
SELECT
*,
p.pid,
p.SKU,
p.title,
p.stock_quantity,
COALESCE(pm.daily_sales_avg, 0) as daily_sales_avg,
COALESCE(pm.days_of_inventory, 0) as days_of_inventory,
COALESCE(pm.reorder_point, 0) as reorder_point,
COALESCE(pm.safety_stock, 0) as safety_stock,
CASE
WHEN stock_quantity <= safety_stock THEN 'Critical'
WHEN stock_quantity <= reorder_point THEN 'Low'
WHEN stock_quantity > (reorder_point * 3) THEN 'Excess'
WHEN pm.daily_sales_avg = 0 THEN 'New'
WHEN p.stock_quantity <= CEIL(pm.daily_sales_avg * pt.critical_days) THEN 'Critical'
WHEN p.stock_quantity <= CEIL(pm.daily_sales_avg * pt.reorder_days) THEN 'Reorder'
WHEN p.stock_quantity > (pm.daily_sales_avg * pt.overstock_days) THEN 'Overstocked'
ELSE 'Healthy'
END as inventory_status,
CASE
WHEN lead_time_status = 'delayed' AND stock_status = 'low' THEN 'High'
WHEN lead_time_status = 'delayed' OR stock_status = 'low' THEN 'Medium'
ELSE 'Low'
END as risk_level
FROM stock_levels;
END as stock_status
FROM
products p
LEFT JOIN
product_metrics pm ON p.pid = pm.pid
LEFT JOIN
product_thresholds pt ON p.pid = pt.pid
WHERE
p.managing_stock = true;
-- Create view for category performance trends
CREATE OR REPLACE VIEW category_performance_trends AS
WITH monthly_trends AS (
SELECT
c.cat_id,
c.name as category_name,
ctm.year,
ctm.month,
ctm.product_count,
ctm.active_products,
ctm.total_value,
ctm.total_revenue,
ctm.avg_margin,
ctm.turnover_rate,
LAG(ctm.total_revenue) OVER (PARTITION BY c.cat_id ORDER BY ctm.year, ctm.month) as prev_month_revenue,
LAG(ctm.turnover_rate) OVER (PARTITION BY c.cat_id ORDER BY ctm.year, ctm.month) as prev_month_turnover
FROM categories c
JOIN category_time_metrics ctm ON c.cat_id = ctm.category_id
)
SELECT
*,
c.cat_id as category_id,
c.name,
c.description,
p.name as parent_name,
c.status,
cm.product_count,
cm.active_products,
cm.total_value,
cm.avg_margin,
cm.turnover_rate,
cm.growth_rate,
CASE
WHEN prev_month_revenue IS NULL THEN 0
ELSE ((total_revenue - prev_month_revenue) / prev_month_revenue) * 100
END as revenue_growth_percent,
CASE
WHEN prev_month_turnover IS NULL THEN 0
ELSE ((turnover_rate - prev_month_turnover) / prev_month_turnover) * 100
END as turnover_growth_percent
FROM monthly_trends;
WHEN cm.growth_rate >= 20 THEN 'High Growth'
WHEN cm.growth_rate >= 5 THEN 'Growing'
WHEN cm.growth_rate >= -5 THEN 'Stable'
ELSE 'Declining'
END as performance_rating
FROM
categories c
LEFT JOIN
categories p ON c.parent_id = p.cat_id
LEFT JOIN
category_metrics cm ON c.cat_id = cm.category_id;
SET session_replication_role = 'origin';
-- Re-enable foreign key checks
SET FOREIGN_KEY_CHECKS = 1;
-- Create table for sales seasonality factors
CREATE TABLE IF NOT EXISTS sales_seasonality (
month INT NOT NULL,
seasonality_factor DECIMAL(5,3) DEFAULT 0,
last_updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (month),
CHECK (month BETWEEN 1 AND 12),
CHECK (seasonality_factor BETWEEN -1.0 AND 1.0)
);
-- Insert default seasonality factors (neutral)
INSERT INTO sales_seasonality (month, seasonality_factor)
VALUES
(1, 0), (2, 0), (3, 0), (4, 0), (5, 0), (6, 0),
(7, 0), (8, 0), (9, 0), (10, 0), (11, 0), (12, 0)
ON DUPLICATE KEY UPDATE last_updated = CURRENT_TIMESTAMP;

208
inventory-server/db/schema.sql
View File

@@ -1,13 +1,6 @@
-- Enable strict error reporting
SET session_replication_role = 'replica'; -- Disable foreign key checks temporarily
-- Create function for updating timestamps
CREATE OR REPLACE FUNCTION update_updated_column() RETURNS TRIGGER AS $func$
BEGIN
NEW.updated = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$func$ language plpgsql;
SET sql_mode = 'STRICT_ALL_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_ZERO_DATE,NO_ZERO_IN_DATE,NO_ENGINE_SUBSTITUTION';
SET FOREIGN_KEY_CHECKS = 0;
-- Create tables
CREATE TABLE products (
@@ -15,18 +8,18 @@ CREATE TABLE products (
title VARCHAR(255) NOT NULL,
description TEXT,
SKU VARCHAR(50) NOT NULL,
created_at TIMESTAMP WITH TIME ZONE,
first_received TIMESTAMP WITH TIME ZONE,
stock_quantity INTEGER DEFAULT 0,
preorder_count INTEGER DEFAULT 0,
notions_inv_count INTEGER DEFAULT 0,
created_at TIMESTAMP NULL,
first_received TIMESTAMP NULL,
stock_quantity INT DEFAULT 0,
preorder_count INT DEFAULT 0,
notions_inv_count INT DEFAULT 0,
price DECIMAL(10, 3) NOT NULL,
regular_price DECIMAL(10, 3) NOT NULL,
cost_price DECIMAL(10, 3),
landing_cost_price DECIMAL(10, 3),
barcode VARCHAR(50),
harmonized_tariff_code VARCHAR(20),
updated_at TIMESTAMP WITH TIME ZONE,
updated_at TIMESTAMP,
visible BOOLEAN DEFAULT true,
managing_stock BOOLEAN DEFAULT true,
replenishable BOOLEAN DEFAULT true,
@@ -44,64 +37,59 @@ CREATE TABLE products (
artist VARCHAR(100),
options TEXT,
tags TEXT,
moq INTEGER DEFAULT 1,
uom INTEGER DEFAULT 1,
moq INT DEFAULT 1,
uom INT DEFAULT 1,
rating DECIMAL(10,2) DEFAULT 0.00,
reviews INTEGER DEFAULT 0,
reviews INT UNSIGNED DEFAULT 0,
weight DECIMAL(10,3),
length DECIMAL(10,3),
width DECIMAL(10,3),
height DECIMAL(10,3),
country_of_origin VARCHAR(5),
location VARCHAR(50),
total_sold INTEGER DEFAULT 0,
baskets INTEGER DEFAULT 0,
notifies INTEGER DEFAULT 0,
total_sold INT UNSIGNED DEFAULT 0,
baskets INT UNSIGNED DEFAULT 0,
notifies INT UNSIGNED DEFAULT 0,
date_last_sold DATE,
updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (pid)
);
-- Create trigger for products
CREATE TRIGGER update_products_updated
BEFORE UPDATE ON products
FOR EACH ROW
EXECUTE FUNCTION update_updated_column();
-- Create indexes for products table
CREATE INDEX idx_products_sku ON products(SKU);
CREATE INDEX idx_products_vendor ON products(vendor);
CREATE INDEX idx_products_brand ON products(brand);
CREATE INDEX idx_products_location ON products(location);
CREATE INDEX idx_products_total_sold ON products(total_sold);
CREATE INDEX idx_products_date_last_sold ON products(date_last_sold);
CREATE INDEX idx_products_updated ON products(updated);
updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (pid),
INDEX idx_sku (SKU),
INDEX idx_vendor (vendor),
INDEX idx_brand (brand),
INDEX idx_location (location),
INDEX idx_total_sold (total_sold),
INDEX idx_date_last_sold (date_last_sold),
INDEX idx_updated (updated)
) ENGINE=InnoDB;
-- Create categories table with hierarchy support
CREATE TABLE categories (
cat_id BIGINT PRIMARY KEY,
name VARCHAR(100) NOT NULL,
type SMALLINT NOT NULL,
type SMALLINT NOT NULL COMMENT '10=section, 11=category, 12=subcategory, 13=subsubcategory, 1=company, 2=line, 3=subline, 40=artist',
parent_id BIGINT,
description TEXT,
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
status VARCHAR(20) DEFAULT 'active',
FOREIGN KEY (parent_id) REFERENCES categories(cat_id)
);
FOREIGN KEY (parent_id) REFERENCES categories(cat_id),
INDEX idx_parent (parent_id),
INDEX idx_type (type),
INDEX idx_status (status),
INDEX idx_name_type (name, type)
) ENGINE=InnoDB;
-- Create trigger for categories
CREATE TRIGGER update_categories_updated_at
BEFORE UPDATE ON categories
FOR EACH ROW
EXECUTE FUNCTION update_updated_column();
COMMENT ON COLUMN categories.type IS '10=section, 11=category, 12=subcategory, 13=subsubcategory, 1=company, 2=line, 3=subline, 40=artist';
CREATE INDEX idx_categories_parent ON categories(parent_id);
CREATE INDEX idx_categories_type ON categories(type);
CREATE INDEX idx_categories_status ON categories(status);
CREATE INDEX idx_categories_name_type ON categories(name, type);
-- Create vendor_details table
CREATE TABLE vendor_details (
vendor VARCHAR(100) PRIMARY KEY,
contact_name VARCHAR(100),
email VARCHAR(255),
phone VARCHAR(50),
status VARCHAR(20) DEFAULT 'active',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
INDEX idx_status (status)
) ENGINE=InnoDB;
-- Create product_categories junction table
CREATE TABLE product_categories (
@@ -109,98 +97,78 @@ CREATE TABLE product_categories (
pid BIGINT NOT NULL,
PRIMARY KEY (pid, cat_id),
FOREIGN KEY (pid) REFERENCES products(pid) ON DELETE CASCADE,
FOREIGN KEY (cat_id) REFERENCES categories(cat_id) ON DELETE CASCADE
);
CREATE INDEX idx_product_categories_category ON product_categories(cat_id);
CREATE INDEX idx_product_categories_product ON product_categories(pid);
FOREIGN KEY (cat_id) REFERENCES categories(cat_id) ON DELETE CASCADE,
INDEX idx_category (cat_id),
INDEX idx_product (pid)
) ENGINE=InnoDB;
-- Create orders table with its indexes
CREATE TABLE orders (
id BIGSERIAL PRIMARY KEY,
CREATE TABLE IF NOT EXISTS orders (
id BIGINT NOT NULL AUTO_INCREMENT,
order_number VARCHAR(50) NOT NULL,
pid BIGINT NOT NULL,
SKU VARCHAR(50) NOT NULL,
date DATE NOT NULL,
price DECIMAL(10,3) NOT NULL,
quantity INTEGER NOT NULL,
quantity INT NOT NULL,
discount DECIMAL(10,3) DEFAULT 0.000,
tax DECIMAL(10,3) DEFAULT 0.000,
tax_included BOOLEAN DEFAULT false,
tax_included TINYINT(1) DEFAULT 0,
shipping DECIMAL(10,3) DEFAULT 0.000,
costeach DECIMAL(10,3) DEFAULT 0.000,
customer VARCHAR(50) NOT NULL,
customer_name VARCHAR(100),
status VARCHAR(20) DEFAULT 'pending',
canceled BOOLEAN DEFAULT false,
updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
UNIQUE (order_number, pid)
);
-- Create trigger for orders
CREATE TRIGGER update_orders_updated
BEFORE UPDATE ON orders
FOR EACH ROW
EXECUTE FUNCTION update_updated_column();
CREATE INDEX idx_orders_number ON orders(order_number);
CREATE INDEX idx_orders_pid ON orders(pid);
CREATE INDEX idx_orders_customer ON orders(customer);
CREATE INDEX idx_orders_date ON orders(date);
CREATE INDEX idx_orders_status ON orders(status);
CREATE INDEX idx_orders_metrics ON orders(pid, date, canceled);
CREATE INDEX idx_orders_updated ON orders(updated);
canceled TINYINT(1) DEFAULT 0,
updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (id),
UNIQUE KEY unique_order_line (order_number, pid),
KEY order_number (order_number),
KEY pid (pid),
KEY customer (customer),
KEY date (date),
KEY status (status),
INDEX idx_orders_metrics (pid, date, canceled),
INDEX idx_updated (updated)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
-- Create purchase_orders table with its indexes
CREATE TABLE purchase_orders (
id BIGSERIAL PRIMARY KEY,
id BIGINT AUTO_INCREMENT PRIMARY KEY,
po_id VARCHAR(50) NOT NULL,
vendor VARCHAR(100) NOT NULL,
date DATE NOT NULL,
expected_date DATE,
pid BIGINT NOT NULL,
sku VARCHAR(50) NOT NULL,
name VARCHAR(100) NOT NULL,
name VARCHAR(100) NOT NULL COMMENT 'Product name from products.description',
cost_price DECIMAL(10, 3) NOT NULL,
po_cost_price DECIMAL(10, 3) NOT NULL,
status SMALLINT DEFAULT 1,
receiving_status SMALLINT DEFAULT 1,
po_cost_price DECIMAL(10, 3) NOT NULL COMMENT 'Original cost from PO, before receiving adjustments',
status TINYINT UNSIGNED DEFAULT 1 COMMENT '0=canceled,1=created,10=electronically_ready_send,11=ordered,12=preordered,13=electronically_sent,15=receiving_started,50=done',
receiving_status TINYINT UNSIGNED DEFAULT 1 COMMENT '0=canceled,1=created,30=partial_received,40=full_received,50=paid',
notes TEXT,
long_note TEXT,
ordered INTEGER NOT NULL,
received INTEGER DEFAULT 0,
received_date DATE,
last_received_date DATE,
received_by VARCHAR(100),
receiving_history JSONB,
updated TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
ordered INT NOT NULL,
received INT DEFAULT 0,
received_date DATE COMMENT 'Date of first receiving',
last_received_date DATE COMMENT 'Date of most recent receiving',
received_by VARCHAR(100) COMMENT 'Name of person who first received this PO line',
receiving_history JSON COMMENT 'Array of receiving records with qty, date, cost, receiving_id, and alt_po flag',
updated TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
FOREIGN KEY (pid) REFERENCES products(pid),
UNIQUE (po_id, pid)
);
INDEX idx_po_id (po_id),
INDEX idx_vendor (vendor),
INDEX idx_status (status),
INDEX idx_receiving_status (receiving_status),
INDEX idx_purchase_orders_metrics (pid, date, status, ordered, received),
INDEX idx_po_metrics (pid, date, receiving_status, received_date),
INDEX idx_po_product_date (pid, date),
INDEX idx_po_product_status (pid, status),
INDEX idx_updated (updated),
UNIQUE KEY unique_po_product (po_id, pid)
) ENGINE=InnoDB;
-- Create trigger for purchase_orders
CREATE TRIGGER update_purchase_orders_updated
BEFORE UPDATE ON purchase_orders
FOR EACH ROW
EXECUTE FUNCTION update_updated_column();
COMMENT ON COLUMN purchase_orders.name IS 'Product name from products.description';
COMMENT ON COLUMN purchase_orders.po_cost_price IS 'Original cost from PO, before receiving adjustments';
COMMENT ON COLUMN purchase_orders.status IS '0=canceled,1=created,10=electronically_ready_send,11=ordered,12=preordered,13=electronically_sent,15=receiving_started,50=done';
COMMENT ON COLUMN purchase_orders.receiving_status IS '0=canceled,1=created,30=partial_received,40=full_received,50=paid';
COMMENT ON COLUMN purchase_orders.receiving_history IS 'Array of receiving records with qty, date, cost, receiving_id, and alt_po flag';
CREATE INDEX idx_po_id ON purchase_orders(po_id);
CREATE INDEX idx_po_vendor ON purchase_orders(vendor);
CREATE INDEX idx_po_status ON purchase_orders(status);
CREATE INDEX idx_po_receiving_status ON purchase_orders(receiving_status);
CREATE INDEX idx_po_metrics ON purchase_orders(pid, date, status, ordered, received);
CREATE INDEX idx_po_metrics_receiving ON purchase_orders(pid, date, receiving_status, received_date);
CREATE INDEX idx_po_product_date ON purchase_orders(pid, date);
CREATE INDEX idx_po_product_status ON purchase_orders(pid, status);
CREATE INDEX idx_po_updated ON purchase_orders(updated);
SET session_replication_role = 'origin'; -- Re-enable foreign key checks
SET FOREIGN_KEY_CHECKS = 1;
-- Create views for common calculations
-- product_sales_trends view moved to metrics-schema.sql

85
inventory-server/db/setup-schema.sql
View File

@@ -1,85 +0,0 @@
-- Templates table for storing import templates
CREATE TABLE IF NOT EXISTS templates (
id SERIAL PRIMARY KEY,
company TEXT NOT NULL,
product_type TEXT NOT NULL,
supplier TEXT,
msrp DECIMAL(10,2),
cost_each DECIMAL(10,2),
qty_per_unit INTEGER,
case_qty INTEGER,
hts_code TEXT,
description TEXT,
weight DECIMAL(10,2),
length DECIMAL(10,2),
width DECIMAL(10,2),
height DECIMAL(10,2),
tax_cat TEXT,
size_cat TEXT,
categories TEXT[],
ship_restrictions TEXT[],
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
UNIQUE(company, product_type)
);
-- AI Prompts table for storing validation prompts
CREATE TABLE IF NOT EXISTS ai_prompts (
id SERIAL PRIMARY KEY,
prompt_text TEXT NOT NULL,
prompt_type TEXT NOT NULL CHECK (prompt_type IN ('general', 'company_specific', 'system')),
company TEXT,
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT unique_company_prompt UNIQUE (company),
CONSTRAINT company_required_for_specific CHECK (
(prompt_type = 'general' AND company IS NULL) OR
(prompt_type = 'system' AND company IS NULL) OR
(prompt_type = 'company_specific' AND company IS NOT NULL)
)
);
-- Create a unique partial index to ensure only one general prompt
CREATE UNIQUE INDEX IF NOT EXISTS idx_unique_general_prompt
ON ai_prompts (prompt_type)
WHERE prompt_type = 'general';
-- Create a unique partial index to ensure only one system prompt
CREATE UNIQUE INDEX IF NOT EXISTS idx_unique_system_prompt
ON ai_prompts (prompt_type)
WHERE prompt_type = 'system';
-- AI Validation Performance Tracking
CREATE TABLE IF NOT EXISTS ai_validation_performance (
id SERIAL PRIMARY KEY,
prompt_length INTEGER NOT NULL,
product_count INTEGER NOT NULL,
start_time TIMESTAMP WITH TIME ZONE NOT NULL,
end_time TIMESTAMP WITH TIME ZONE NOT NULL,
duration_seconds DECIMAL(10,2) GENERATED ALWAYS AS (EXTRACT(EPOCH FROM (end_time - start_time))) STORED,
created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP
);
-- Create index on prompt_length for efficient querying
CREATE INDEX IF NOT EXISTS idx_ai_validation_prompt_length ON ai_validation_performance(prompt_length);
-- Function to update the updated_at timestamp
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = CURRENT_TIMESTAMP;
RETURN NEW;
END;
$$ language 'plpgsql';
-- Trigger to automatically update the updated_at column
CREATE TRIGGER update_templates_updated_at
BEFORE UPDATE ON templates
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- Trigger to automatically update the updated_at column for ai_prompts
CREATE TRIGGER update_ai_prompts_updated_at
BEFORE UPDATE ON ai_prompts
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();

957
inventory-server/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

6
inventory-server/package.json
View File

@@ -18,18 +18,12 @@
"author": "",
"license": "ISC",
"dependencies": {
"@types/diff": "^7.0.1",
"axios": "^1.8.1",
"bcrypt": "^5.1.1",
"cors": "^2.8.5",
"csv-parse": "^5.6.0",
"diff": "^7.0.0",
"dotenv": "^16.4.7",
"express": "^4.18.2",
"multer": "^1.4.5-lts.1",
"mysql2": "^3.12.0",
"openai": "^4.85.3",
"pg": "^8.13.3",
"pm2": "^5.3.0",
"ssh2": "^1.16.0",
"uuid": "^9.0.1"

96
inventory-server/scripts/calculate-metrics.js
View File

@@ -44,34 +44,6 @@ global.clearProgress = progress.clearProgress;
global.getProgress = progress.getProgress;
global.logError = progress.logError;
// List of temporary tables used in the calculation process
const TEMP_TABLES = [
'temp_revenue_ranks',
'temp_sales_metrics',
'temp_purchase_metrics',
'temp_product_metrics',
'temp_vendor_metrics',
'temp_category_metrics',
'temp_brand_metrics',
'temp_forecast_dates',
'temp_daily_sales',
'temp_product_stats',
'temp_category_sales',
'temp_category_stats'
];
// Add cleanup function for temporary tables
async function cleanupTemporaryTables(connection) {
try {
for (const table of TEMP_TABLES) {
await connection.query(`DROP TEMPORARY TABLE IF EXISTS ${table}`);
}
} catch (error) {
logError(error, 'Error cleaning up temporary tables');
throw error; // Re-throw to be handled by the caller
}
}
const { getConnection, closePool } = require('./metrics/utils/db');
const calculateProductMetrics = require('./metrics/product-metrics');
const calculateTimeAggregates = require('./metrics/time-aggregates');
@@ -132,17 +104,54 @@ async function calculateMetrics() {
WHERE status = 'running'
`);
// Get counts from all relevant tables
// Get counts of records that need updating based on last calculation time
const [[productCount], [orderCount], [poCount]] = await Promise.all([
connection.query('SELECT COUNT(*) as total FROM products'),
connection.query('SELECT COUNT(*) as total FROM orders'),
connection.query('SELECT COUNT(*) as total FROM purchase_orders')
connection.query(`
SELECT COUNT(DISTINCT p.pid) as total
FROM products p
FORCE INDEX (PRIMARY)
LEFT JOIN calculate_status cs ON cs.module_name = 'product_metrics'
LEFT JOIN orders o FORCE INDEX (idx_orders_metrics) ON p.pid = o.pid
AND o.updated > COALESCE(cs.last_calculation_timestamp, '1970-01-01')
AND o.canceled = false
LEFT JOIN purchase_orders po FORCE INDEX (idx_purchase_orders_metrics) ON p.pid = po.pid
AND po.updated > COALESCE(cs.last_calculation_timestamp, '1970-01-01')
WHERE p.updated > COALESCE(cs.last_calculation_timestamp, '1970-01-01')
OR o.pid IS NOT NULL
OR po.pid IS NOT NULL
`),
connection.query(`
SELECT COUNT(DISTINCT o.id) as total
FROM orders o
FORCE INDEX (idx_orders_metrics)
LEFT JOIN calculate_status cs ON cs.module_name = 'product_metrics'
WHERE o.updated > COALESCE(cs.last_calculation_timestamp, '1970-01-01')
AND o.canceled = false
`),
connection.query(`
SELECT COUNT(DISTINCT po.id) as total
FROM purchase_orders po
FORCE INDEX (idx_purchase_orders_metrics)
LEFT JOIN calculate_status cs ON cs.module_name = 'product_metrics'
WHERE po.updated > COALESCE(cs.last_calculation_timestamp, '1970-01-01')
`)
]);
totalProducts = productCount.total;
totalOrders = orderCount.total;
totalPurchaseOrders = poCount.total;
// If nothing needs updating, we can exit early
if (totalProducts === 0 && totalOrders === 0 && totalPurchaseOrders === 0) {
console.log('No records need updating');
return {
processedProducts: 0,
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
};
}
// Create history record for this calculation
const [historyResult] = await connection.query(`
INSERT INTO calculate_history (
@@ -267,7 +276,7 @@ async function calculateMetrics() {
});
if (!SKIP_PRODUCT_METRICS) {
const result = await calculateProductMetrics(startTime, totalProducts);
const result = await calculateProductMetrics(startTime, totalProducts, processedProducts, isCancelled);
await updateProgress(result.processedProducts, result.processedOrders, result.processedPurchaseOrders);
if (!result.success) {
throw new Error('Product metrics calculation failed');
@@ -678,26 +687,19 @@ async function calculateMetrics() {
throw error;
} finally {
if (connection) {
// Ensure temporary tables are cleaned up
await cleanupTemporaryTables(connection);
connection.release();
}
// Close the connection pool when we're done
await closePool();
}
} catch (error) {
success = false;
logError(error, 'Error in metrics calculation');
throw error;
} finally {
// Close the connection pool when we're done
await closePool();
}
}
// Export as a module with all necessary functions
module.exports = {
calculateMetrics,
cancelCalculation,
getProgress: global.getProgress
};
// Export both functions and progress checker
module.exports = calculateMetrics;
module.exports.cancelCalculation = cancelCalculation;
module.exports.getProgress = global.getProgress;
// Run directly if called from command line
if (require.main === module) {

115
inventory-server/scripts/full-reset.js
View File

@@ -1,115 +0,0 @@
const path = require('path');
const { spawn } = require('child_process');
function outputProgress(data) {
if (!data.status) {
data = {
status: 'running',
...data
};
}
console.log(JSON.stringify(data));
}
function runScript(scriptPath) {
return new Promise((resolve, reject) => {
const child = spawn('node', [scriptPath], {
stdio: ['inherit', 'pipe', 'pipe'],
env: {
...process.env,
PGHOST: process.env.DB_HOST,
PGUSER: process.env.DB_USER,
PGPASSWORD: process.env.DB_PASSWORD,
PGDATABASE: process.env.DB_NAME,
PGPORT: process.env.DB_PORT || '5432'
}
});
let output = '';
child.stdout.on('data', (data) => {
const lines = data.toString().split('\n');
lines.filter(line => line.trim()).forEach(line => {
try {
console.log(line); // Pass through the JSON output
output += line + '\n';
} catch (e) {
console.log(line); // If not JSON, just log it directly
}
});
});
child.stderr.on('data', (data) => {
console.error(data.toString());
});
child.on('close', (code) => {
if (code !== 0) {
reject(new Error(`Script ${scriptPath} exited with code ${code}`));
} else {
resolve(output);
}
});
child.on('error', (err) => {
reject(err);
});
});
}
async function fullReset() {
try {
// Step 1: Reset Database
outputProgress({
operation: 'Starting full reset',
message: 'Step 1/3: Resetting database...'
});
await runScript(path.join(__dirname, 'reset-db.js'));
outputProgress({
status: 'complete',
operation: 'Database reset step complete',
message: 'Database reset finished, moving to import...'
});
// Step 2: Import from Production
outputProgress({
operation: 'Starting import',
message: 'Step 2/3: Importing from production...'
});
await runScript(path.join(__dirname, 'import-from-prod.js'));
outputProgress({
status: 'complete',
operation: 'Import step complete',
message: 'Import finished, moving to metrics calculation...'
});
// Step 3: Calculate Metrics
outputProgress({
operation: 'Starting metrics calculation',
message: 'Step 3/3: Calculating metrics...'
});
await runScript(path.join(__dirname, 'calculate-metrics.js'));
// Final completion message
outputProgress({
status: 'complete',
operation: 'Full reset complete',
message: 'Successfully completed all steps: database reset, import, and metrics calculation'
});
} catch (error) {
outputProgress({
status: 'error',
operation: 'Full reset failed',
error: error.message,
stack: error.stack
});
process.exit(1);
}
}
// Run if called directly
if (require.main === module) {
fullReset();
}
module.exports = fullReset;

100
inventory-server/scripts/full-update.js
View File

@@ -1,100 +0,0 @@
const path = require('path');
const { spawn } = require('child_process');
function outputProgress(data) {
if (!data.status) {
data = {
status: 'running',
...data
};
}
console.log(JSON.stringify(data));
}
function runScript(scriptPath) {
return new Promise((resolve, reject) => {
const child = spawn('node', [scriptPath], {
stdio: ['inherit', 'pipe', 'pipe']
});
let output = '';
child.stdout.on('data', (data) => {
const lines = data.toString().split('\n');
lines.filter(line => line.trim()).forEach(line => {
try {
console.log(line); // Pass through the JSON output
output += line + '\n';
} catch (e) {
console.log(line); // If not JSON, just log it directly
}
});
});
child.stderr.on('data', (data) => {
console.error(data.toString());
});
child.on('close', (code) => {
if (code !== 0) {
reject(new Error(`Script ${scriptPath} exited with code ${code}`));
} else {
resolve(output);
}
});
child.on('error', (err) => {
reject(err);
});
});
}
async function fullUpdate() {
try {
// Step 1: Import from Production
outputProgress({
operation: 'Starting full update',
message: 'Step 1/2: Importing from production...'
});
await runScript(path.join(__dirname, 'import-from-prod.js'));
outputProgress({
status: 'complete',
operation: 'Import step complete',
message: 'Import finished, moving to metrics calculation...'
});
// Step 2: Calculate Metrics
outputProgress({
operation: 'Starting metrics calculation',
message: 'Step 2/2: Calculating metrics...'
});
await runScript(path.join(__dirname, 'calculate-metrics.js'));
outputProgress({
status: 'complete',
operation: 'Metrics step complete',
message: 'Metrics calculation finished'
});
// Final completion message
outputProgress({
status: 'complete',
operation: 'Full update complete',
message: 'Successfully completed all steps: import and metrics calculation'
});
} catch (error) {
outputProgress({
status: 'error',
operation: 'Full update failed',
error: error.message,
stack: error.stack
});
process.exit(1);
}
}
// Run if called directly
if (require.main === module) {
fullUpdate();
}
module.exports = fullUpdate;

105
inventory-server/scripts/import-from-prod.js
View File

@@ -19,6 +19,7 @@ const IMPORT_PURCHASE_ORDERS = true;
const INCREMENTAL_UPDATE = process.env.INCREMENTAL_UPDATE !== 'false'; // Default to true unless explicitly set to false
// SSH configuration
// In import-from-prod.js
const sshConfig = {
ssh: {
host: process.env.PROD_SSH_HOST,
@@ -30,7 +31,6 @@ const sshConfig = {
compress: true, // Enable SSH compression
},
prodDbConfig: {
// MySQL config for production
host: process.env.PROD_DB_HOST || "localhost",
user: process.env.PROD_DB_USER,
password: process.env.PROD_DB_PASSWORD,
@@ -39,16 +39,21 @@ const sshConfig = {
timezone: 'Z',
},
localDbConfig: {
// PostgreSQL config for local
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
port: process.env.DB_PORT || 5432,
ssl: process.env.DB_SSL === 'true',
connectionTimeoutMillis: 60000,
idleTimeoutMillis: 30000,
max: 10 // connection pool max size
multipleStatements: true,
waitForConnections: true,
connectionLimit: 10,
queueLimit: 0,
namedPlaceholders: true,
connectTimeout: 60000,
enableKeepAlive: true,
keepAliveInitialDelay: 10000,
compress: true,
timezone: 'Z',
stringifyObjects: false,
}
};
@@ -103,7 +108,7 @@ async function main() {
SET
status = 'cancelled',
end_time = NOW(),
duration_seconds = EXTRACT(EPOCH FROM (NOW() - start_time))::INTEGER,
duration_seconds = TIMESTAMPDIFF(SECOND, start_time, NOW()),
error_message = 'Previous import was not completed properly'
WHERE status = 'running'
`);
@@ -113,10 +118,9 @@ async function main() {
CREATE TABLE IF NOT EXISTS sync_status (
table_name VARCHAR(50) PRIMARY KEY,
last_sync_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
last_sync_id BIGINT
last_sync_id BIGINT,
INDEX idx_last_sync (last_sync_timestamp)
);
CREATE INDEX IF NOT EXISTS idx_last_sync ON sync_status (last_sync_timestamp);
`);
// Create import history record for the overall session
@@ -130,17 +134,17 @@ async function main() {
) VALUES (
'all_tables',
NOW(),
$1::boolean,
?,
'running',
jsonb_build_object(
'categories_enabled', $2::boolean,
'products_enabled', $3::boolean,
'orders_enabled', $4::boolean,
'purchase_orders_enabled', $5::boolean
JSON_OBJECT(
'categories_enabled', ?,
'products_enabled', ?,
'orders_enabled', ?,
'purchase_orders_enabled', ?
)
) RETURNING id
)
`, [INCREMENTAL_UPDATE, IMPORT_CATEGORIES, IMPORT_PRODUCTS, IMPORT_ORDERS, IMPORT_PURCHASE_ORDERS]);
importHistoryId = historyResult.rows[0].id;
importHistoryId = historyResult.insertId;
const results = {
categories: null,
@@ -158,8 +162,8 @@ async function main() {
if (isImportCancelled) throw new Error("Import cancelled");
completedSteps++;
console.log('Categories import result:', results.categories);
totalRecordsAdded += parseInt(results.categories?.recordsAdded || 0);
totalRecordsUpdated += parseInt(results.categories?.recordsUpdated || 0);
totalRecordsAdded += results.categories?.recordsAdded || 0;
totalRecordsUpdated += results.categories?.recordsUpdated || 0;
}
if (IMPORT_PRODUCTS) {
@@ -167,8 +171,8 @@ async function main() {
if (isImportCancelled) throw new Error("Import cancelled");
completedSteps++;
console.log('Products import result:', results.products);
totalRecordsAdded += parseInt(results.products?.recordsAdded || 0);
totalRecordsUpdated += parseInt(results.products?.recordsUpdated || 0);
totalRecordsAdded += results.products?.recordsAdded || 0;
totalRecordsUpdated += results.products?.recordsUpdated || 0;
}
if (IMPORT_ORDERS) {
@@ -176,8 +180,8 @@ async function main() {
if (isImportCancelled) throw new Error("Import cancelled");
completedSteps++;
console.log('Orders import result:', results.orders);
totalRecordsAdded += parseInt(results.orders?.recordsAdded || 0);
totalRecordsUpdated += parseInt(results.orders?.recordsUpdated || 0);
totalRecordsAdded += results.orders?.recordsAdded || 0;
totalRecordsUpdated += results.orders?.recordsUpdated || 0;
}
if (IMPORT_PURCHASE_ORDERS) {
@@ -185,8 +189,8 @@ async function main() {
if (isImportCancelled) throw new Error("Import cancelled");
completedSteps++;
console.log('Purchase orders import result:', results.purchaseOrders);
totalRecordsAdded += parseInt(results.purchaseOrders?.recordsAdded || 0);
totalRecordsUpdated += parseInt(results.purchaseOrders?.recordsUpdated || 0);
totalRecordsAdded += results.purchaseOrders?.recordsAdded || 0;
totalRecordsUpdated += results.purchaseOrders?.recordsUpdated || 0;
}
const endTime = Date.now();
@@ -197,21 +201,21 @@ async function main() {
UPDATE import_history
SET
end_time = NOW(),
duration_seconds = $1,
records_added = $2,
records_updated = $3,
duration_seconds = ?,
records_added = ?,
records_updated = ?,
status = 'completed',
additional_info = jsonb_build_object(
'categories_enabled', $4::boolean,
'products_enabled', $5::boolean,
'orders_enabled', $6::boolean,
'purchase_orders_enabled', $7::boolean,
'categories_result', COALESCE($8::jsonb, 'null'::jsonb),
'products_result', COALESCE($9::jsonb, 'null'::jsonb),
'orders_result', COALESCE($10::jsonb, 'null'::jsonb),
'purchase_orders_result', COALESCE($11::jsonb, 'null'::jsonb)
additional_info = JSON_OBJECT(
'categories_enabled', ?,
'products_enabled', ?,
'orders_enabled', ?,
'purchase_orders_enabled', ?,
'categories_result', CAST(? AS JSON),
'products_result', CAST(? AS JSON),
'orders_result', CAST(? AS JSON),
'purchase_orders_result', CAST(? AS JSON)
)
WHERE id = $12
WHERE id = ?
`, [
totalElapsedSeconds,
totalRecordsAdded,
@@ -255,10 +259,10 @@ async function main() {
UPDATE import_history
SET
end_time = NOW(),
duration_seconds = $1,
status = $2,
error_message = $3
WHERE id = $4
duration_seconds = ?,
status = ?,
error_message = ?
WHERE id = ?
`, [totalElapsedSeconds, error.message === "Import cancelled" ? 'cancelled' : 'failed', error.message, importHistoryId]);
}
@@ -284,23 +288,16 @@ async function main() {
throw error;
} finally {
if (connections) {
await closeConnections(connections).catch(err => {
console.error("Error closing connections:", err);
});
await closeConnections(connections);
}
}
}
// Run the import only if this is the main module
if (require.main === module) {
main().then((results) => {
console.log('Import completed successfully:', results);
// Force exit after a small delay to ensure all logs are written
setTimeout(() => process.exit(0), 500);
}).catch((error) => {
main().catch((error) => {
console.error("Unhandled error in main process:", error);
// Force exit with error code after a small delay
setTimeout(() => process.exit(1), 500);
process.exit(1);
});
}

294
inventory-server/scripts/import/categories.js
View File

@@ -9,206 +9,170 @@ async function importCategories(prodConnection, localConnection) {
const startTime = Date.now();
const typeOrder = [10, 20, 11, 21, 12, 13];
let totalInserted = 0;
let totalUpdated = 0;
let skippedCategories = [];
try {
// Start a single transaction for the entire import
await localConnection.query('BEGIN');
// Process each type in order with its own savepoint
// Process each type in order with its own query
for (const type of typeOrder) {
try {
// Create a savepoint for this type
await localConnection.query(`SAVEPOINT category_type_${type}`);
const [categories] = await prodConnection.query(
`
SELECT
pc.cat_id,
pc.name,
pc.type,
CASE
WHEN pc.type IN (10, 20) THEN NULL -- Top level categories should have no parent
WHEN pc.master_cat_id IS NULL THEN NULL
ELSE pc.master_cat_id
END as parent_id,
pc.combined_name as description
FROM product_categories pc
WHERE pc.type = ?
ORDER BY pc.cat_id
`,
[type]
);
// Production query remains MySQL compatible
const [categories] = await prodConnection.query(
`
SELECT
pc.cat_id,
pc.name,
pc.type,
CASE
WHEN pc.type IN (10, 20) THEN NULL -- Top level categories should have no parent
WHEN pc.master_cat_id IS NULL THEN NULL
ELSE pc.master_cat_id
END as parent_id,
pc.combined_name as description
FROM product_categories pc
WHERE pc.type = ?
ORDER BY pc.cat_id
`,
[type]
if (categories.length === 0) continue;
console.log(`\nProcessing ${categories.length} type ${type} categories`);
if (type === 10) {
console.log("Type 10 categories:", JSON.stringify(categories, null, 2));
}
// For types that can have parents (11, 21, 12, 13), verify parent existence
let categoriesToInsert = categories;
if (![10, 20].includes(type)) {
// Get all parent IDs
const parentIds = [
...new Set(
categories.map((c) => c.parent_id).filter((id) => id !== null)
),
];
// Check which parents exist
const [existingParents] = await localConnection.query(
"SELECT cat_id FROM categories WHERE cat_id IN (?)",
[parentIds]
);
const existingParentIds = new Set(existingParents.map((p) => p.cat_id));
// Filter categories and track skipped ones
categoriesToInsert = categories.filter(
(cat) =>
cat.parent_id === null || existingParentIds.has(cat.parent_id)
);
const invalidCategories = categories.filter(
(cat) =>
cat.parent_id !== null && !existingParentIds.has(cat.parent_id)
);
if (categories.length === 0) {
await localConnection.query(`RELEASE SAVEPOINT category_type_${type}`);
continue;
}
if (invalidCategories.length > 0) {
const skippedInfo = invalidCategories.map((c) => ({
id: c.cat_id,
name: c.name,
type: c.type,
missing_parent: c.parent_id,
}));
skippedCategories.push(...skippedInfo);
console.log(`\nProcessing ${categories.length} type ${type} categories`);
if (type === 10) {
console.log("Type 10 categories:", JSON.stringify(categories, null, 2));
}
// For types that can have parents (11, 21, 12, 13), verify parent existence
let categoriesToInsert = categories;
if (![10, 20].includes(type)) {
// Get all parent IDs
const parentIds = [
...new Set(
categories
.filter(c => c && c.parent_id !== null)
.map(c => c.parent_id)
),
];
console.log(`Processing ${categories.length} type ${type} categories with ${parentIds.length} unique parent IDs`);
console.log('Parent IDs:', parentIds);
// No need to check for parent existence - we trust they exist since they were just inserted
categoriesToInsert = categories;
console.log(
"\nSkipping categories with missing parents:",
invalidCategories
.map(
(c) =>
`${c.cat_id} - ${c.name} (missing parent: ${c.parent_id})`
)
.join("\n")
);
}
if (categoriesToInsert.length === 0) {
console.log(
`No valid categories of type ${type} to insert`
`No valid categories of type ${type} to insert - all had missing parents`
);
await localConnection.query(`RELEASE SAVEPOINT category_type_${type}`);
continue;
}
console.log(
`Inserting ${categoriesToInsert.length} type ${type} categories`
);
// PostgreSQL upsert query with parameterized values
const values = categoriesToInsert.flatMap((cat) => [
cat.cat_id,
cat.name,
cat.type,
cat.parent_id,
cat.description,
'active',
new Date(),
new Date()
]);
console.log('Attempting to insert/update with values:', JSON.stringify(values, null, 2));
const placeholders = categoriesToInsert
.map((_, i) => `($${i * 8 + 1}, $${i * 8 + 2}, $${i * 8 + 3}, $${i * 8 + 4}, $${i * 8 + 5}, $${i * 8 + 6}, $${i * 8 + 7}, $${i * 8 + 8})`)
.join(',');
console.log('Using placeholders:', placeholders);
// Insert categories with ON CONFLICT clause for PostgreSQL
const query = `
WITH inserted_categories AS (
INSERT INTO categories (
cat_id, name, type, parent_id, description, status, created_at, updated_at
)
VALUES ${placeholders}
ON CONFLICT (cat_id) DO UPDATE SET
name = EXCLUDED.name,
type = EXCLUDED.type,
parent_id = EXCLUDED.parent_id,
description = EXCLUDED.description,
status = EXCLUDED.status,
updated_at = EXCLUDED.updated_at
RETURNING
cat_id,
CASE
WHEN xmax = 0 THEN true
ELSE false
END as is_insert
)
SELECT
COUNT(*) as total,
COUNT(*) FILTER (WHERE is_insert) as inserted,
COUNT(*) FILTER (WHERE NOT is_insert) as updated
FROM inserted_categories`;
console.log('Executing query:', query);
const result = await localConnection.query(query, values);
console.log('Query result:', result);
// Get the first result since query returns an array
const queryResult = Array.isArray(result) ? result[0] : result;
if (!queryResult || !queryResult.rows || !queryResult.rows[0]) {
console.error('Query failed to return results. Result:', queryResult);
throw new Error('Query did not return expected results');
}
const total = parseInt(queryResult.rows[0].total) || 0;
const inserted = parseInt(queryResult.rows[0].inserted) || 0;
const updated = parseInt(queryResult.rows[0].updated) || 0;
console.log(`Total: ${total}, Inserted: ${inserted}, Updated: ${updated}`);
totalInserted += inserted;
totalUpdated += updated;
// Release the savepoint for this type
await localConnection.query(`RELEASE SAVEPOINT category_type_${type}`);
outputProgress({
status: "running",
operation: "Categories import",
message: `Imported ${inserted} (updated ${updated}) categories of type ${type}`,
current: totalInserted + totalUpdated,
total: categories.length,
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
});
} catch (error) {
// Rollback to the savepoint for this type
await localConnection.query(`ROLLBACK TO SAVEPOINT category_type_${type}`);
throw error;
}
console.log(
`Inserting ${categoriesToInsert.length} type ${type} categories`
);
const placeholders = categoriesToInsert
.map(() => "(?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)")
.join(",");
const values = categoriesToInsert.flatMap((cat) => [
cat.cat_id,
cat.name,
cat.type,
cat.parent_id,
cat.description,
"active",
]);
// Insert categories and create relationships in one query to avoid race conditions
await localConnection.query(
`
INSERT INTO categories (cat_id, name, type, parent_id, description, status, created_at, updated_at)
VALUES ${placeholders}
ON DUPLICATE KEY UPDATE
name = VALUES(name),
type = VALUES(type),
parent_id = VALUES(parent_id),
description = VALUES(description),
status = VALUES(status),
updated_at = CURRENT_TIMESTAMP
`,
values
);
totalInserted += categoriesToInsert.length;
outputProgress({
status: "running",
operation: "Categories import",
current: totalInserted,
total: totalInserted,
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
});
}
// Commit the entire transaction - we'll do this even if we have skipped categories
await localConnection.query('COMMIT');
// After all imports, if we skipped any categories, throw an error
if (skippedCategories.length > 0) {
const error = new Error(
"Categories import completed with errors - some categories were skipped due to missing parents"
);
error.skippedCategories = skippedCategories;
throw error;
}
outputProgress({
status: "complete",
operation: "Categories import completed",
current: totalInserted + totalUpdated,
total: totalInserted + totalUpdated,
current: totalInserted,
total: totalInserted,
duration: formatElapsedTime((Date.now() - startTime) / 1000),
warnings: skippedCategories.length > 0 ? {
message: "Some categories were skipped due to missing parents",
skippedCategories
} : undefined
});
return {
status: "complete",
recordsAdded: totalInserted,
recordsUpdated: totalUpdated,
totalRecords: totalInserted + totalUpdated,
warnings: skippedCategories.length > 0 ? {
message: "Some categories were skipped due to missing parents",
skippedCategories
} : undefined
totalImported: totalInserted
};
} catch (error) {
console.error("Error importing categories:", error);
// Only rollback if we haven't committed yet
try {
await localConnection.query('ROLLBACK');
} catch (rollbackError) {
console.error("Error during rollback:", rollbackError);
if (error.skippedCategories) {
console.error(
"Skipped categories:",
JSON.stringify(error.skippedCategories, null, 2)
);
}
outputProgress({
status: "error",
operation: "Categories import failed",
error: error.message
error: error.message,
skippedCategories: error.skippedCategories
});
throw error;

751
inventory-server/scripts/import/orders.js
View File

@@ -2,7 +2,7 @@ const { outputProgress, formatElapsedTime, estimateRemaining, calculateRate } =
const { importMissingProducts, setupTemporaryTables, cleanupTemporaryTables, materializeCalculations } = require('./products');
/**
* Imports orders from a production MySQL database to a local PostgreSQL database.
* Imports orders from a production MySQL database to a local MySQL database.
* It can run in two modes:
* 1. Incremental update mode (default): Only fetch orders that have changed since the last sync time.
* 2. Full update mode: Fetch all eligible orders within the last 5 years regardless of timestamp.
@@ -23,18 +23,93 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
let importedCount = 0;
let totalOrderItems = 0;
let totalUniqueOrders = 0;
// Add a cumulative counter for processed orders before the loop
let cumulativeProcessedOrders = 0;
try {
// Clean up any existing temp tables first
await localConnection.query(`
DROP TEMPORARY TABLE IF EXISTS temp_order_items;
DROP TEMPORARY TABLE IF EXISTS temp_order_meta;
DROP TEMPORARY TABLE IF EXISTS temp_order_discounts;
DROP TEMPORARY TABLE IF EXISTS temp_order_taxes;
DROP TEMPORARY TABLE IF EXISTS temp_order_costs;
`);
// Create all temp tables with correct schema
await localConnection.query(`
CREATE TEMPORARY TABLE temp_order_items (
order_id INT UNSIGNED NOT NULL,
pid INT UNSIGNED NOT NULL,
SKU VARCHAR(50) NOT NULL,
price DECIMAL(10,2) NOT NULL,
quantity INT NOT NULL,
base_discount DECIMAL(10,2) DEFAULT 0,
PRIMARY KEY (order_id, pid)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
`);
await localConnection.query(`
CREATE TEMPORARY TABLE temp_order_meta (
order_id INT UNSIGNED NOT NULL,
date DATE NOT NULL,
customer VARCHAR(100) NOT NULL,
customer_name VARCHAR(150) NOT NULL,
status INT,
canceled TINYINT(1),
summary_discount DECIMAL(10,2) DEFAULT 0.00,
summary_subtotal DECIMAL(10,2) DEFAULT 0.00,
PRIMARY KEY (order_id)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
`);
await localConnection.query(`
CREATE TEMPORARY TABLE temp_order_discounts (
order_id INT UNSIGNED NOT NULL,
pid INT UNSIGNED NOT NULL,
discount DECIMAL(10,2) NOT NULL,
PRIMARY KEY (order_id, pid)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
`);
await localConnection.query(`
CREATE TEMPORARY TABLE temp_order_taxes (
order_id INT UNSIGNED NOT NULL,
pid INT UNSIGNED NOT NULL,
tax DECIMAL(10,2) NOT NULL,
PRIMARY KEY (order_id, pid)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
`);
await localConnection.query(`
CREATE TEMPORARY TABLE temp_order_costs (
order_id INT UNSIGNED NOT NULL,
pid INT UNSIGNED NOT NULL,
costeach DECIMAL(10,3) DEFAULT 0.000,
PRIMARY KEY (order_id, pid)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
`);
// Get column names from the local table
const [columns] = await localConnection.query(`
SELECT COLUMN_NAME
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME = 'orders'
AND COLUMN_NAME != 'updated' -- Exclude the updated column
ORDER BY ORDINAL_POSITION
`);
const columnNames = columns.map(col => col.COLUMN_NAME);
// Get last sync info
const [syncInfo] = await localConnection.query(
"SELECT last_sync_timestamp FROM sync_status WHERE table_name = 'orders'"
);
const lastSyncTime = syncInfo?.rows?.[0]?.last_sync_timestamp || '1970-01-01';
const lastSyncTime = syncInfo?.[0]?.last_sync_timestamp || '1970-01-01';
console.log('Orders: Using last sync time:', lastSyncTime);
// First get count of order items - Keep MySQL compatible for production
// First get count of order items
const [[{ total }]] = await prodConnection.query(`
SELECT COUNT(*) as total
FROM order_items oi
@@ -66,13 +141,12 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
totalOrderItems = total;
console.log('Orders: Found changes:', totalOrderItems);
// Get order items - Keep MySQL compatible for production
console.log('Orders: Starting MySQL query...');
// Get order items in batches
const [orderItems] = await prodConnection.query(`
SELECT
oi.order_id,
oi.prod_pid,
COALESCE(NULLIF(TRIM(oi.prod_itemnumber), ''), 'NO-SKU') as SKU,
oi.prod_pid as pid,
oi.prod_itemnumber as SKU,
oi.prod_price as price,
oi.qty_ordered as quantity,
COALESCE(oi.prod_price_reg - oi.prod_price, 0) as base_discount,
@@ -103,78 +177,24 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
` : ''}
`, incrementalUpdate ? [lastSyncTime, lastSyncTime, lastSyncTime] : []);
console.log('Orders: Found', orderItems.length, 'order items to process');
// Create tables in PostgreSQL for debugging
await localConnection.query(`
DROP TABLE IF EXISTS debug_order_items;
DROP TABLE IF EXISTS debug_order_meta;
DROP TABLE IF EXISTS debug_order_discounts;
DROP TABLE IF EXISTS debug_order_taxes;
DROP TABLE IF EXISTS debug_order_costs;
CREATE TABLE debug_order_items (
order_id INTEGER NOT NULL,
pid INTEGER NOT NULL,
SKU VARCHAR(50) NOT NULL,
price DECIMAL(10,2) NOT NULL,
quantity INTEGER NOT NULL,
base_discount DECIMAL(10,2) DEFAULT 0,
PRIMARY KEY (order_id, pid)
);
CREATE TABLE debug_order_meta (
order_id INTEGER NOT NULL,
date DATE NOT NULL,
customer VARCHAR(100) NOT NULL,
customer_name VARCHAR(150) NOT NULL,
status INTEGER,
canceled BOOLEAN,
summary_discount DECIMAL(10,2) DEFAULT 0.00,
summary_subtotal DECIMAL(10,2) DEFAULT 0.00,
PRIMARY KEY (order_id)
);
CREATE TABLE debug_order_discounts (
order_id INTEGER NOT NULL,
pid INTEGER NOT NULL,
discount DECIMAL(10,2) NOT NULL,
PRIMARY KEY (order_id, pid)
);
CREATE TABLE debug_order_taxes (
order_id INTEGER NOT NULL,
pid INTEGER NOT NULL,
tax DECIMAL(10,2) NOT NULL,
PRIMARY KEY (order_id, pid)
);
CREATE TABLE debug_order_costs (
order_id INTEGER NOT NULL,
pid INTEGER NOT NULL,
costeach DECIMAL(10,3) DEFAULT 0.000,
PRIMARY KEY (order_id, pid)
);
`);
console.log('Orders: Processing', orderItems.length, 'order items');
// Insert order items in batches
for (let i = 0; i < orderItems.length; i += 5000) {
const batch = orderItems.slice(i, Math.min(i + 5000, orderItems.length));
const placeholders = batch.map((_, idx) =>
`($${idx * 6 + 1}, $${idx * 6 + 2}, $${idx * 6 + 3}, $${idx * 6 + 4}, $${idx * 6 + 5}, $${idx * 6 + 6})`
).join(",");
const placeholders = batch.map(() => "(?, ?, ?, ?, ?, ?)").join(",");
const values = batch.flatMap(item => [
item.order_id, item.prod_pid, item.SKU, item.price, item.quantity, item.base_discount
item.order_id, item.pid, item.SKU, item.price, item.quantity, item.base_discount
]);
await localConnection.query(`
INSERT INTO debug_order_items (order_id, pid, SKU, price, quantity, base_discount)
INSERT INTO temp_order_items (order_id, pid, SKU, price, quantity, base_discount)
VALUES ${placeholders}
ON CONFLICT (order_id, pid) DO UPDATE SET
SKU = EXCLUDED.SKU,
price = EXCLUDED.price,
quantity = EXCLUDED.quantity,
base_discount = EXCLUDED.base_discount
ON DUPLICATE KEY UPDATE
SKU = VALUES(SKU),
price = VALUES(price),
quantity = VALUES(quantity),
base_discount = VALUES(base_discount)
`, values);
processedCount = i + batch.length;
@@ -183,26 +203,24 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
operation: "Orders import",
message: `Loading order items: ${processedCount} of ${totalOrderItems}`,
current: processedCount,
total: totalOrderItems,
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
remaining: estimateRemaining(startTime, processedCount, totalOrderItems),
rate: calculateRate(startTime, processedCount)
total: totalOrderItems
});
}
// Get unique order IDs
const orderIds = [...new Set(orderItems.map(item => item.order_id))];
totalUniqueOrders = orderIds.length;
console.log('Orders: Processing', totalUniqueOrders, 'unique orders');
console.log('Total unique order IDs:', totalUniqueOrders);
// Reset processed count for order processing phase
processedCount = 0;
// Process metadata, discounts, taxes, and costs in parallel
const METADATA_BATCH_SIZE = 2000;
const PG_BATCH_SIZE = 200;
const processMetadataBatch = async (batchIds) => {
// Get order metadata in batches
for (let i = 0; i < orderIds.length; i += 5000) {
const batchIds = orderIds.slice(i, i + 5000);
console.log(`Processing batch ${i/5000 + 1}, size: ${batchIds.length}`);
console.log('Sample of batch IDs:', batchIds.slice(0, 5));
const [orders] = await prodConnection.query(`
SELECT
o.order_id,
@@ -217,46 +235,64 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
LEFT JOIN users u ON o.order_cid = u.cid
WHERE o.order_id IN (?)
`, [batchIds]);
// Process in sub-batches for PostgreSQL
for (let j = 0; j < orders.length; j += PG_BATCH_SIZE) {
const subBatch = orders.slice(j, j + PG_BATCH_SIZE);
if (subBatch.length === 0) continue;
const placeholders = subBatch.map((_, idx) =>
`($${idx * 8 + 1}, $${idx * 8 + 2}, $${idx * 8 + 3}, $${idx * 8 + 4}, $${idx * 8 + 5}, $${idx * 8 + 6}, $${idx * 8 + 7}, $${idx * 8 + 8})`
).join(",");
const values = subBatch.flatMap(order => [
order.order_id,
order.date,
order.customer,
order.customer_name || '',
order.status,
order.canceled,
order.summary_discount || 0,
order.summary_subtotal || 0
]);
await localConnection.query(`
INSERT INTO debug_order_meta (
order_id, date, customer, customer_name, status, canceled,
summary_discount, summary_subtotal
)
VALUES ${placeholders}
ON CONFLICT (order_id) DO UPDATE SET
date = EXCLUDED.date,
customer = EXCLUDED.customer,
customer_name = EXCLUDED.customer_name,
status = EXCLUDED.status,
canceled = EXCLUDED.canceled,
summary_discount = EXCLUDED.summary_discount,
summary_subtotal = EXCLUDED.summary_subtotal
`, values);
console.log(`Retrieved ${orders.length} orders for ${batchIds.length} IDs`);
const duplicates = orders.filter((order, index, self) =>
self.findIndex(o => o.order_id === order.order_id) !== index
);
if (duplicates.length > 0) {
console.log('Found duplicates:', duplicates);
}
};
const processDiscountsBatch = async (batchIds) => {
const placeholders = orders.map(() => "(?, ?, ?, ?, ?, ?, ?, ?)").join(",");
const values = orders.flatMap(order => [
order.order_id,
order.date,
order.customer,
order.customer_name,
order.status,
order.canceled,
order.summary_discount,
order.summary_subtotal
]);
await localConnection.query(`
INSERT INTO temp_order_meta (
order_id,
date,
customer,
customer_name,
status,
canceled,
summary_discount,
summary_subtotal
) VALUES ${placeholders}
ON DUPLICATE KEY UPDATE
date = VALUES(date),
customer = VALUES(customer),
customer_name = VALUES(customer_name),
status = VALUES(status),
canceled = VALUES(canceled),
summary_discount = VALUES(summary_discount),
summary_subtotal = VALUES(summary_subtotal)
`, values);
processedCount = i + orders.length;
outputProgress({
status: "running",
operation: "Orders import",
message: `Loading order metadata: ${processedCount} of ${totalUniqueOrders}`,
current: processedCount,
total: totalUniqueOrders
});
}
// Reset processed count for final phase
processedCount = 0;
// Get promotional discounts in batches
for (let i = 0; i < orderIds.length; i += 5000) {
const batchIds = orderIds.slice(i, i + 5000);
const [discounts] = await prodConnection.query(`
SELECT order_id, pid, SUM(amount) as discount
FROM order_discount_items
@@ -264,296 +300,313 @@ async function importOrders(prodConnection, localConnection, incrementalUpdate =
GROUP BY order_id, pid
`, [batchIds]);
if (discounts.length === 0) return;
for (let j = 0; j < discounts.length; j += PG_BATCH_SIZE) {
const subBatch = discounts.slice(j, j + PG_BATCH_SIZE);
if (subBatch.length === 0) continue;
const placeholders = subBatch.map((_, idx) =>
`($${idx * 3 + 1}, $${idx * 3 + 2}, $${idx * 3 + 3})`
).join(",");
const values = subBatch.flatMap(d => [
d.order_id,
d.pid,
d.discount || 0
]);
if (discounts.length > 0) {
const placeholders = discounts.map(() => "(?, ?, ?)").join(",");
const values = discounts.flatMap(d => [d.order_id, d.pid, d.discount]);
await localConnection.query(`
INSERT INTO debug_order_discounts (order_id, pid, discount)
VALUES ${placeholders}
ON CONFLICT (order_id, pid) DO UPDATE SET
discount = EXCLUDED.discount
INSERT INTO temp_order_discounts VALUES ${placeholders}
ON DUPLICATE KEY UPDATE
discount = VALUES(discount)
`, values);
}
};
}
const processTaxesBatch = async (batchIds) => {
// Optimized tax query to avoid subquery
// Get tax information in batches
for (let i = 0; i < orderIds.length; i += 5000) {
const batchIds = orderIds.slice(i, i + 5000);
const [taxes] = await prodConnection.query(`
SELECT oti.order_id, otip.pid, otip.item_taxes_to_collect as tax
FROM (
SELECT order_id, MAX(taxinfo_id) as latest_taxinfo_id
SELECT DISTINCT
oti.order_id,
otip.pid,
otip.item_taxes_to_collect as tax
FROM order_tax_info oti
JOIN (
SELECT order_id, MAX(stamp) as max_stamp
FROM order_tax_info
WHERE order_id IN (?)
GROUP BY order_id
) latest_info
JOIN order_tax_info oti ON oti.order_id = latest_info.order_id
AND oti.taxinfo_id = latest_info.latest_taxinfo_id
) latest ON oti.order_id = latest.order_id AND oti.stamp = latest.max_stamp
JOIN order_tax_info_products otip ON oti.taxinfo_id = otip.taxinfo_id
`, [batchIds]);
if (taxes.length === 0) return;
if (taxes.length > 0) {
// Remove any duplicates before inserting
const uniqueTaxes = new Map();
taxes.forEach(t => {
const key = `${t.order_id}-${t.pid}`;
uniqueTaxes.set(key, t);
});
for (let j = 0; j < taxes.length; j += PG_BATCH_SIZE) {
const subBatch = taxes.slice(j, j + PG_BATCH_SIZE);
if (subBatch.length === 0) continue;
const placeholders = subBatch.map((_, idx) =>
`($${idx * 3 + 1}, $${idx * 3 + 2}, $${idx * 3 + 3})`
).join(",");
const values = subBatch.flatMap(t => [
t.order_id,
t.pid,
t.tax || 0
]);
await localConnection.query(`
INSERT INTO debug_order_taxes (order_id, pid, tax)
VALUES ${placeholders}
ON CONFLICT (order_id, pid) DO UPDATE SET
tax = EXCLUDED.tax
`, values);
const values = Array.from(uniqueTaxes.values()).flatMap(t => [t.order_id, t.pid, t.tax]);
if (values.length > 0) {
const placeholders = Array(uniqueTaxes.size).fill("(?, ?, ?)").join(",");
await localConnection.query(`
INSERT INTO temp_order_taxes VALUES ${placeholders}
ON DUPLICATE KEY UPDATE tax = VALUES(tax)
`, values);
}
}
};
}
const processCostsBatch = async (batchIds) => {
// Get costeach values in batches
for (let i = 0; i < orderIds.length; i += 5000) {
const batchIds = orderIds.slice(i, i + 5000);
const [costs] = await prodConnection.query(`
SELECT
oc.orderid as order_id,
oc.orderid as order_id,
oc.pid,
oc.costeach
COALESCE(
oc.costeach,
(SELECT pi.costeach
FROM product_inventory pi
WHERE pi.pid = oc.pid
AND pi.daterec <= o.date_placed
ORDER BY pi.daterec DESC LIMIT 1)
) as costeach
FROM order_costs oc
JOIN _order o ON oc.orderid = o.order_id
WHERE oc.orderid IN (?)
AND oc.pending = 0
`, [batchIds]);
if (costs.length === 0) return;
for (let j = 0; j < costs.length; j += PG_BATCH_SIZE) {
const subBatch = costs.slice(j, j + PG_BATCH_SIZE);
if (subBatch.length === 0) continue;
const placeholders = subBatch.map((_, idx) =>
`($${idx * 3 + 1}, $${idx * 3 + 2}, $${idx * 3 + 3})`
).join(",");
const values = subBatch.flatMap(c => [
c.order_id,
c.pid,
c.costeach || 0
]);
if (costs.length > 0) {
const placeholders = costs.map(() => '(?, ?, ?)').join(",");
const values = costs.flatMap(c => [c.order_id, c.pid, c.costeach || 0]);
await localConnection.query(`
INSERT INTO debug_order_costs (order_id, pid, costeach)
INSERT INTO temp_order_costs (order_id, pid, costeach)
VALUES ${placeholders}
ON CONFLICT (order_id, pid) DO UPDATE SET
costeach = EXCLUDED.costeach
ON DUPLICATE KEY UPDATE costeach = VALUES(costeach)
`, values);
}
};
}
// Process all data types in parallel for each batch
for (let i = 0; i < orderIds.length; i += METADATA_BATCH_SIZE) {
const batchIds = orderIds.slice(i, i + METADATA_BATCH_SIZE);
// Now combine all the data and insert into orders table
// Pre-check all products at once instead of per batch
const allOrderPids = [...new Set(orderItems.map(item => item.pid))];
const [existingProducts] = allOrderPids.length > 0 ? await localConnection.query(
"SELECT pid FROM products WHERE pid IN (?)",
[allOrderPids]
) : [[]];
const existingPids = new Set(existingProducts.map(p => p.pid));
// Process in larger batches
for (let i = 0; i < orderIds.length; i += 5000) {
const batchIds = orderIds.slice(i, i + 5000);
// Get combined data for this batch
const [orders] = await localConnection.query(`
SELECT
oi.order_id as order_number,
oi.pid,
oi.SKU,
om.date,
oi.price,
oi.quantity,
oi.base_discount + COALESCE(od.discount, 0) +
CASE
WHEN om.summary_discount > 0 THEN
ROUND((om.summary_discount * (oi.price * oi.quantity)) /
NULLIF(om.summary_subtotal, 0), 2)
ELSE 0
END as discount,
COALESCE(ot.tax, 0) as tax,
0 as tax_included,
0 as shipping,
om.customer,
om.customer_name,
om.status,
om.canceled,
COALESCE(tc.costeach, 0) as costeach
FROM temp_order_items oi
JOIN temp_order_meta om ON oi.order_id = om.order_id
LEFT JOIN temp_order_discounts od ON oi.order_id = od.order_id AND oi.pid = od.pid
LEFT JOIN temp_order_taxes ot ON oi.order_id = ot.order_id AND oi.pid = ot.pid
LEFT JOIN temp_order_costs tc ON oi.order_id = tc.order_id AND oi.pid = tc.pid
WHERE oi.order_id IN (?)
`, [batchIds]);
// Filter orders and track missing products - do this in a single pass
const validOrders = [];
const values = [];
const processedOrderItems = new Set(); // Track unique order items
const processedOrders = new Set(); // Track unique orders
await Promise.all([
processMetadataBatch(batchIds),
processDiscountsBatch(batchIds),
processTaxesBatch(batchIds),
processCostsBatch(batchIds)
]);
for (const order of orders) {
if (!existingPids.has(order.pid)) {
missingProducts.add(order.pid);
skippedOrders.add(order.order_number);
continue;
}
validOrders.push(order);
values.push(...columnNames.map(col => order[col] ?? null));
processedOrderItems.add(`${order.order_number}-${order.pid}`);
processedOrders.add(order.order_number);
}
processedCount = i + batchIds.length;
if (validOrders.length > 0) {
// Pre-compute the placeholders string once
const singlePlaceholder = `(${columnNames.map(() => "?").join(",")})`;
const placeholders = Array(validOrders.length).fill(singlePlaceholder).join(",");
const result = await localConnection.query(`
INSERT INTO orders (${columnNames.join(",")})
VALUES ${placeholders}
ON DUPLICATE KEY UPDATE
SKU = VALUES(SKU),
date = VALUES(date),
price = VALUES(price),
quantity = VALUES(quantity),
discount = VALUES(discount),
tax = VALUES(tax),
tax_included = VALUES(tax_included),
shipping = VALUES(shipping),
customer = VALUES(customer),
customer_name = VALUES(customer_name),
status = VALUES(status),
canceled = VALUES(canceled),
costeach = VALUES(costeach)
`, validOrders.map(o => columnNames.map(col => o[col] ?? null)).flat());
const affectedRows = result[0].affectedRows;
const updates = Math.floor(affectedRows / 2);
const inserts = affectedRows - (updates * 2);
recordsAdded += inserts;
recordsUpdated += updates;
importedCount += processedOrderItems.size; // Count unique order items processed
}
// Update progress based on unique orders processed
cumulativeProcessedOrders += processedOrders.size;
outputProgress({
status: "running",
operation: "Orders import",
message: `Loading order data: ${processedCount} of ${totalUniqueOrders}`,
current: processedCount,
message: `Imported ${importedCount} order items (${cumulativeProcessedOrders} of ${totalUniqueOrders} orders processed)`,
current: cumulativeProcessedOrders,
total: totalUniqueOrders,
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
remaining: estimateRemaining(startTime, processedCount, totalUniqueOrders),
rate: calculateRate(startTime, processedCount)
remaining: estimateRemaining(startTime, cumulativeProcessedOrders, totalUniqueOrders),
rate: calculateRate(startTime, cumulativeProcessedOrders)
});
}
// Pre-check all products at once
const allOrderPids = [...new Set(orderItems.map(item => item.prod_pid))];
console.log('Orders: Checking', allOrderPids.length, 'unique products');
const [existingProducts] = allOrderPids.length > 0 ? await localConnection.query(
"SELECT pid FROM products WHERE pid = ANY($1::bigint[])",
[allOrderPids]
) : [[]];
const existingPids = new Set(existingProducts.rows.map(p => p.pid));
// Process in smaller batches
for (let i = 0; i < orderIds.length; i += 1000) {
const batchIds = orderIds.slice(i, i + 1000);
// Now try to import any orders that were skipped due to missing products
if (skippedOrders.size > 0) {
try {
outputProgress({
status: "running",
operation: "Orders import",
message: `Retrying import of ${skippedOrders.size} orders with previously missing products`,
});
// Get combined data for this batch in sub-batches
const PG_BATCH_SIZE = 100; // Process 100 records at a time
for (let j = 0; j < batchIds.length; j += PG_BATCH_SIZE) {
const subBatchIds = batchIds.slice(j, j + PG_BATCH_SIZE);
const [orders] = await localConnection.query(`
WITH order_totals AS (
SELECT
oi.order_id,
oi.pid,
SUM(COALESCE(od.discount, 0)) as promo_discount,
COALESCE(ot.tax, 0) as total_tax,
COALESCE(oi.price * 0.5, 0) as costeach
FROM debug_order_items oi
LEFT JOIN debug_order_discounts od ON oi.order_id = od.order_id AND oi.pid = od.pid
LEFT JOIN debug_order_taxes ot ON oi.order_id = ot.order_id AND oi.pid = ot.pid
GROUP BY oi.order_id, oi.pid, ot.tax
)
SELECT
// Get the orders that were skipped
const [skippedProdOrders] = await localConnection.query(`
SELECT DISTINCT
oi.order_id as order_number,
oi.pid::bigint as pid,
oi.SKU as sku,
oi.pid,
oi.SKU,
om.date,
oi.price,
oi.quantity,
(oi.base_discount +
COALESCE(ot.promo_discount, 0) +
CASE
WHEN om.summary_discount > 0 AND om.summary_subtotal > 0 THEN
ROUND((om.summary_discount * (oi.price * oi.quantity)) / NULLIF(om.summary_subtotal, 0), 2)
oi.base_discount + COALESCE(od.discount, 0) +
CASE
WHEN o.summary_discount > 0 THEN
ROUND((o.summary_discount * (oi.price * oi.quantity)) /
NULLIF(o.summary_subtotal, 0), 2)
ELSE 0
END)::DECIMAL(10,2) as discount,
COALESCE(ot.total_tax, 0)::DECIMAL(10,2) as tax,
false as tax_included,
END as discount,
COALESCE(ot.tax, 0) as tax,
0 as tax_included,
0 as shipping,
om.customer,
om.customer_name,
om.status,
om.canceled,
COALESCE(ot.costeach, oi.price * 0.5)::DECIMAL(10,3) as costeach
FROM (
SELECT DISTINCT ON (order_id, pid)
order_id, pid, SKU, price, quantity, base_discount
FROM debug_order_items
WHERE order_id = ANY($1)
ORDER BY order_id, pid
) oi
JOIN debug_order_meta om ON oi.order_id = om.order_id
LEFT JOIN order_totals ot ON oi.order_id = ot.order_id AND oi.pid = ot.pid
ORDER BY oi.order_id, oi.pid
`, [subBatchIds]);
COALESCE(tc.costeach, 0) as costeach
FROM temp_order_items oi
JOIN temp_order_meta om ON oi.order_id = om.order_id
LEFT JOIN _order o ON oi.order_id = o.order_id
LEFT JOIN temp_order_discounts od ON oi.order_id = od.order_id AND oi.pid = od.pid
LEFT JOIN temp_order_taxes ot ON oi.order_id = ot.order_id AND oi.pid = ot.pid
LEFT JOIN temp_order_costs tc ON oi.order_id = tc.order_id AND oi.pid = tc.pid
WHERE oi.order_id IN (?)
`, [Array.from(skippedOrders)]);
// Filter orders and track missing products
const validOrders = [];
const processedOrderItems = new Set();
const processedOrders = new Set();
for (const order of orders.rows) {
if (!existingPids.has(order.pid)) {
missingProducts.add(order.pid);
skippedOrders.add(order.order_number);
continue;
}
validOrders.push(order);
processedOrderItems.add(`${order.order_number}-${order.pid}`);
processedOrders.add(order.order_number);
}
// Check which products exist now
const skippedPids = [...new Set(skippedProdOrders.map(o => o.pid))];
const [existingProducts] = skippedPids.length > 0 ? await localConnection.query(
"SELECT pid FROM products WHERE pid IN (?)",
[skippedPids]
) : [[]];
const existingPids = new Set(existingProducts.map(p => p.pid));
// Process valid orders in smaller sub-batches
const FINAL_BATCH_SIZE = 50;
for (let k = 0; k < validOrders.length; k += FINAL_BATCH_SIZE) {
const subBatch = validOrders.slice(k, k + FINAL_BATCH_SIZE);
// Filter orders that can now be imported
const validOrders = skippedProdOrders.filter(order => existingPids.has(order.pid));
const retryOrderItems = new Set(); // Track unique order items in retry
if (validOrders.length > 0) {
const placeholders = validOrders.map(() => `(${columnNames.map(() => "?").join(", ")})`).join(",");
const values = validOrders.map(o => columnNames.map(col => o[col] ?? null)).flat();
const result = await localConnection.query(`
INSERT INTO orders (${columnNames.join(", ")})
VALUES ${placeholders}
ON DUPLICATE KEY UPDATE
SKU = VALUES(SKU),
date = VALUES(date),
price = VALUES(price),
quantity = VALUES(quantity),
discount = VALUES(discount),
tax = VALUES(tax),
tax_included = VALUES(tax_included),
shipping = VALUES(shipping),
customer = VALUES(customer),
customer_name = VALUES(customer_name),
status = VALUES(status),
canceled = VALUES(canceled),
costeach = VALUES(costeach)
`, values);
const affectedRows = result[0].affectedRows;
const updates = Math.floor(affectedRows / 2);
const inserts = affectedRows - (updates * 2);
const placeholders = subBatch.map((_, idx) => {
const base = idx * 14; // 14 columns (removed updated)
return `($${base + 1}, $${base + 2}, $${base + 3}, $${base + 4}, $${base + 5}, $${base + 6}, $${base + 7}, $${base + 8}, $${base + 9}, $${base + 10}, $${base + 11}, $${base + 12}, $${base + 13}, $${base + 14})`;
}).join(',');
const batchValues = subBatch.flatMap(o => [
o.order_number,
o.pid,
o.sku || 'NO-SKU',
o.date,
o.price,
o.quantity,
o.discount,
o.tax,
o.tax_included,
o.shipping,
o.customer,
o.customer_name,
o.status,
o.canceled
]);
const [result] = await localConnection.query(`
WITH inserted_orders AS (
INSERT INTO orders (
order_number, pid, sku, date, price, quantity, discount,
tax, tax_included, shipping, customer, customer_name,
status, canceled
)
VALUES ${placeholders}
ON CONFLICT (order_number, pid) DO UPDATE SET
sku = EXCLUDED.sku,
date = EXCLUDED.date,
price = EXCLUDED.price,
quantity = EXCLUDED.quantity,
discount = EXCLUDED.discount,
tax = EXCLUDED.tax,
tax_included = EXCLUDED.tax_included,
shipping = EXCLUDED.shipping,
customer = EXCLUDED.customer,
customer_name = EXCLUDED.customer_name,
status = EXCLUDED.status,
canceled = EXCLUDED.canceled
RETURNING xmax = 0 as inserted
)
SELECT
COUNT(*) FILTER (WHERE inserted) as inserted,
COUNT(*) FILTER (WHERE NOT inserted) as updated
FROM inserted_orders
`, batchValues);
// Track unique order items
validOrders.forEach(order => {
retryOrderItems.add(`${order.order_number}-${order.pid}`);
});
const { inserted, updated } = result.rows[0];
recordsAdded += inserted;
recordsUpdated += updated;
importedCount += subBatch.length;
}
outputProgress({
status: "running",
operation: "Orders import",
message: `Successfully imported ${retryOrderItems.size} previously skipped order items`,
});
cumulativeProcessedOrders += processedOrders.size;
outputProgress({
status: "running",
operation: "Orders import",
message: `Importing orders: ${cumulativeProcessedOrders} of ${totalUniqueOrders}`,
current: cumulativeProcessedOrders,
total: totalUniqueOrders,
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
remaining: estimateRemaining(startTime, cumulativeProcessedOrders, totalUniqueOrders),
rate: calculateRate(startTime, cumulativeProcessedOrders)
});
// Update the main counters
recordsAdded += inserts;
recordsUpdated += updates;
importedCount += retryOrderItems.size;
}
} catch (error) {
console.warn('Warning: Failed to retry skipped orders:', error.message);
console.warn(`Skipped ${skippedOrders.size} orders due to ${missingProducts.size} missing products`);
}
}
// Update sync status
// Clean up temporary tables after ALL processing is complete
await localConnection.query(`
DROP TEMPORARY TABLE IF EXISTS temp_order_items;
DROP TEMPORARY TABLE IF EXISTS temp_order_meta;
DROP TEMPORARY TABLE IF EXISTS temp_order_discounts;
DROP TEMPORARY TABLE IF EXISTS temp_order_taxes;
DROP TEMPORARY TABLE IF EXISTS temp_order_costs;
`);
// Only update sync status if we get here (no errors thrown)
await localConnection.query(`
INSERT INTO sync_status (table_name, last_sync_timestamp)
VALUES ('orders', NOW())
ON CONFLICT (table_name) DO UPDATE SET
last_sync_timestamp = NOW()
ON DUPLICATE KEY UPDATE last_sync_timestamp = NOW()
`);
return {

1319
inventory-server/scripts/import/products.js
View File

File diff suppressed because it is too large Load Diff

713
inventory-server/scripts/import/purchase-orders.js
View File

@@ -10,42 +10,22 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
const [syncInfo] = await localConnection.query(
"SELECT last_sync_timestamp FROM sync_status WHERE table_name = 'purchase_orders'"
);
const lastSyncTime = syncInfo?.rows?.[0]?.last_sync_timestamp || '1970-01-01';
const lastSyncTime = syncInfo?.[0]?.last_sync_timestamp || '1970-01-01';
console.log('Purchase Orders: Using last sync time:', lastSyncTime);
// Create temporary tables with PostgreSQL syntax
// Insert temporary table creation query for purchase orders
await localConnection.query(`
DROP TABLE IF EXISTS temp_purchase_orders;
DROP TABLE IF EXISTS temp_po_receivings;
CREATE TEMP TABLE temp_purchase_orders (
po_id INTEGER NOT NULL,
pid INTEGER NOT NULL,
sku VARCHAR(50),
name VARCHAR(255),
CREATE TABLE IF NOT EXISTS temp_purchase_orders (
po_id INT UNSIGNED NOT NULL,
pid INT UNSIGNED NOT NULL,
vendor VARCHAR(255),
date TIMESTAMP WITH TIME ZONE,
expected_date TIMESTAMP WITH TIME ZONE,
status INTEGER,
date DATE,
expected_date DATE,
status INT,
notes TEXT,
ordered INTEGER,
cost_price DECIMAL(10,3),
PRIMARY KEY (po_id, pid)
);
CREATE TEMP TABLE temp_po_receivings (
po_id INTEGER,
pid INTEGER NOT NULL,
receiving_id INTEGER NOT NULL,
qty_each INTEGER,
cost_each DECIMAL(10,3),
received_date TIMESTAMP WITH TIME ZONE,
received_by INTEGER,
received_by_name VARCHAR(255),
is_alt_po INTEGER,
PRIMARY KEY (receiving_id, pid)
);
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
`);
outputProgress({
@@ -53,8 +33,8 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
status: "running",
});
// Get column names - Keep MySQL compatible for production
const [columns] = await prodConnection.query(`
// Get column names first
const [columns] = await localConnection.query(`
SELECT COLUMN_NAME
FROM INFORMATION_SCHEMA.COLUMNS
WHERE TABLE_NAME = 'purchase_orders'
@@ -80,7 +60,7 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
? [lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime, lastSyncTime]
: [];
// First get all relevant PO IDs with basic info - Keep MySQL compatible for production
// First get all relevant PO IDs with basic info
const [[{ total }]] = await prodConnection.query(`
SELECT COUNT(*) as total
FROM (
@@ -119,7 +99,6 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
console.log('Purchase Orders: Found changes:', total);
// Get PO list - Keep MySQL compatible for production
const [poList] = await prodConnection.query(`
SELECT DISTINCT
COALESCE(p.po_id, r.receiving_id) as po_id,
@@ -130,12 +109,12 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
) as vendor,
CASE
WHEN p.po_id IS NOT NULL THEN
COALESCE(
DATE(COALESCE(
NULLIF(p.date_ordered, '0000-00-00 00:00:00'),
p.date_created
)
))
WHEN r.receiving_id IS NOT NULL THEN
r.date_created
DATE(r.date_created)
END as date,
CASE
WHEN p.date_estin = '0000-00-00' THEN NULL
@@ -206,14 +185,14 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
const batch = poList.slice(i, Math.min(i + BATCH_SIZE, poList.length));
const poIds = batch.map(po => po.po_id);
// Get all products for these POs in one query - Keep MySQL compatible for production
// Get all products for these POs in one query
const [poProducts] = await prodConnection.query(`
SELECT
pop.po_id,
pop.pid,
pr.itemnumber as sku,
pr.description as name,
pop.cost_each as cost_price,
pop.cost_each,
pop.qty_each as ordered
FROM po_products pop
USE INDEX (PRIMARY)
@@ -253,397 +232,317 @@ async function importPurchaseOrders(prodConnection, localConnection, incremental
ORDER BY r.po_id, rp.pid, rp.received_date
`, [batchPoIds, productPids]);
// Insert receivings into temp table
if (receivings.length > 0) {
// Process in smaller chunks to avoid parameter limits
const CHUNK_SIZE = 100; // Reduce chunk size to avoid parameter limits
for (let i = 0; i < receivings.length; i += CHUNK_SIZE) {
const chunk = receivings.slice(i, Math.min(i + CHUNK_SIZE, receivings.length));
// Create maps for this sub-batch
const poProductMap = new Map();
productBatch.forEach(product => {
const key = `${product.po_id}-${product.pid}`;
poProductMap.set(key, product);
});
const receivingMap = new Map();
const altReceivingMap = new Map();
const noPOReceivingMap = new Map();
receivings.forEach(receiving => {
const key = `${receiving.po_id}-${receiving.pid}`;
if (receiving.is_alt_po === 2) {
// No PO
if (!noPOReceivingMap.has(receiving.pid)) {
noPOReceivingMap.set(receiving.pid, []);
}
noPOReceivingMap.get(receiving.pid).push(receiving);
} else if (receiving.is_alt_po === 1) {
// Different PO
if (!altReceivingMap.has(receiving.pid)) {
altReceivingMap.set(receiving.pid, []);
}
altReceivingMap.get(receiving.pid).push(receiving);
} else {
// Original PO
if (!receivingMap.has(key)) {
receivingMap.set(key, []);
}
receivingMap.get(key).push(receiving);
}
});
// Verify PIDs exist
const [existingPids] = await localConnection.query(
'SELECT pid FROM products WHERE pid IN (?)',
[productPids]
);
const validPids = new Set(existingPids.map(p => p.pid));
// First check which PO lines already exist and get their current values
const poLines = Array.from(poProductMap.values())
.filter(p => validPids.has(p.pid))
.map(p => [p.po_id, p.pid]);
const [existingPOs] = await localConnection.query(
`SELECT ${columnNames.join(',')} FROM purchase_orders WHERE (po_id, pid) IN (${poLines.map(() => "(?,?)").join(",")})`,
poLines.flat()
);
const existingPOMap = new Map(
existingPOs.map(po => [`${po.po_id}-${po.pid}`, po])
);
// Split into inserts and updates
const insertsAndUpdates = { inserts: [], updates: [] };
let batchProcessed = 0;
for (const po of batch) {
const poProducts = Array.from(poProductMap.values())
.filter(p => p.po_id === po.po_id && validPids.has(p.pid));
for (const product of poProducts) {
const key = `${po.po_id}-${product.pid}`;
const receivingHistory = receivingMap.get(key) || [];
const altReceivingHistory = altReceivingMap.get(product.pid) || [];
const noPOReceivingHistory = noPOReceivingMap.get(product.pid) || [];
const values = [];
const placeholders = [];
chunk.forEach((r, idx) => {
values.push(
r.po_id,
r.pid,
r.receiving_id,
r.qty_each,
r.cost_each,
r.received_date,
r.received_by,
r.received_by_name || null,
r.is_alt_po
);
const offset = idx * 9;
placeholders.push(`($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}, $${offset + 6}, $${offset + 7}, $${offset + 8}, $${offset + 9})`);
// Combine all receivings and sort by date
const allReceivings = [
...receivingHistory.map(r => ({ ...r, type: 'original' })),
...altReceivingHistory.map(r => ({ ...r, type: 'alternate' })),
...noPOReceivingHistory.map(r => ({ ...r, type: 'no_po' }))
].sort((a, b) => new Date(a.received_date || '9999-12-31') - new Date(b.received_date || '9999-12-31'));
// Split receivings into original PO and others
const originalPOReceivings = allReceivings.filter(r => r.type === 'original');
const otherReceivings = allReceivings.filter(r => r.type !== 'original');
// Track FIFO fulfillment
let remainingToFulfill = product.ordered;
const fulfillmentTracking = [];
let totalReceived = 0;
let actualCost = null; // Will store the cost of the first receiving that fulfills this PO
let firstFulfillmentReceiving = null;
let lastFulfillmentReceiving = null;
for (const receiving of allReceivings) {
// Convert quantities to base units using supplier data
const baseQtyReceived = receiving.qty_each * (
receiving.type === 'original' ? 1 :
Math.max(1, product.supplier_qty_per_unit || 1)
);
const qtyToApply = Math.min(remainingToFulfill, baseQtyReceived);
if (qtyToApply > 0) {
// If this is the first receiving being applied, use its cost
if (actualCost === null && receiving.cost_each > 0) {
actualCost = receiving.cost_each;
firstFulfillmentReceiving = receiving;
}
lastFulfillmentReceiving = receiving;
fulfillmentTracking.push({
receiving_id: receiving.receiving_id,
qty_applied: qtyToApply,
qty_total: baseQtyReceived,
cost: receiving.cost_each || actualCost || product.cost_each,
date: receiving.received_date,
received_by: receiving.received_by,
received_by_name: receiving.received_by_name || 'Unknown',
type: receiving.type,
remaining_qty: baseQtyReceived - qtyToApply
});
remainingToFulfill -= qtyToApply;
} else {
// Track excess receivings
fulfillmentTracking.push({
receiving_id: receiving.receiving_id,
qty_applied: 0,
qty_total: baseQtyReceived,
cost: receiving.cost_each || actualCost || product.cost_each,
date: receiving.received_date,
received_by: receiving.received_by,
received_by_name: receiving.received_by_name || 'Unknown',
type: receiving.type,
is_excess: true
});
}
totalReceived += baseQtyReceived;
}
const receiving_status = !totalReceived ? 1 : // created
remainingToFulfill > 0 ? 30 : // partial
40; // full
function formatDate(dateStr) {
if (!dateStr) return null;
if (dateStr === '0000-00-00' || dateStr === '0000-00-00 00:00:00') return null;
if (typeof dateStr === 'string' && !dateStr.match(/^\d{4}-\d{2}-\d{2}/)) return null;
try {
const date = new Date(dateStr);
if (isNaN(date.getTime())) return null;
if (date.getFullYear() < 1900 || date.getFullYear() > 2100) return null;
return date.toISOString().split('T')[0];
} catch (e) {
return null;
}
}
const rowValues = columnNames.map(col => {
switch (col) {
case 'po_id': return po.po_id;
case 'vendor': return po.vendor;
case 'date': return formatDate(po.date);
case 'expected_date': return formatDate(po.expected_date);
case 'pid': return product.pid;
case 'sku': return product.sku;
case 'name': return product.name;
case 'cost_price': return actualCost || product.cost_each;
case 'po_cost_price': return product.cost_each;
case 'status': return po.status;
case 'notes': return po.notes;
case 'long_note': return po.long_note;
case 'ordered': return product.ordered;
case 'received': return totalReceived;
case 'unfulfilled': return remainingToFulfill;
case 'excess_received': return Math.max(0, totalReceived - product.ordered);
case 'received_date': return formatDate(firstFulfillmentReceiving?.received_date);
case 'last_received_date': return formatDate(lastFulfillmentReceiving?.received_date);
case 'received_by': return firstFulfillmentReceiving?.received_by_name || null;
case 'receiving_status': return receiving_status;
case 'receiving_history': return JSON.stringify({
fulfillment: fulfillmentTracking,
ordered_qty: product.ordered,
total_received: totalReceived,
remaining_unfulfilled: remainingToFulfill,
excess_received: Math.max(0, totalReceived - product.ordered),
po_cost: product.cost_each,
actual_cost: actualCost || product.cost_each
});
default: return null;
}
});
await localConnection.query(`
INSERT INTO temp_po_receivings (
po_id, pid, receiving_id, qty_each, cost_each, received_date,
received_by, received_by_name, is_alt_po
)
VALUES ${placeholders.join(',')}
ON CONFLICT (receiving_id, pid) DO UPDATE SET
po_id = EXCLUDED.po_id,
qty_each = EXCLUDED.qty_each,
cost_each = EXCLUDED.cost_each,
received_date = EXCLUDED.received_date,
received_by = EXCLUDED.received_by,
received_by_name = EXCLUDED.received_by_name,
is_alt_po = EXCLUDED.is_alt_po
`, values);
if (existingPOMap.has(key)) {
const existing = existingPOMap.get(key);
// Check if any values are different
const hasChanges = columnNames.some(col => {
const newVal = rowValues[columnNames.indexOf(col)];
const oldVal = existing[col] ?? null;
// Special handling for numbers to avoid type coercion issues
if (typeof newVal === 'number' && typeof oldVal === 'number') {
return Math.abs(newVal - oldVal) > 0.00001; // Allow for tiny floating point differences
}
// Special handling for receiving_history - parse and compare
if (col === 'receiving_history') {
const newHistory = JSON.parse(newVal || '{}');
const oldHistory = JSON.parse(oldVal || '{}');
return JSON.stringify(newHistory) !== JSON.stringify(oldHistory);
}
return newVal !== oldVal;
});
if (hasChanges) {
insertsAndUpdates.updates.push({
po_id: po.po_id,
pid: product.pid,
values: rowValues
});
}
} else {
insertsAndUpdates.inserts.push({
po_id: po.po_id,
pid: product.pid,
values: rowValues
});
}
batchProcessed++;
}
}
// Process each PO product in chunks
const PRODUCT_CHUNK_SIZE = 100;
for (let i = 0; i < productBatch.length; i += PRODUCT_CHUNK_SIZE) {
const chunk = productBatch.slice(i, Math.min(i + PRODUCT_CHUNK_SIZE, productBatch.length));
const values = [];
const placeholders = [];
// Handle inserts
if (insertsAndUpdates.inserts.length > 0) {
const insertPlaceholders = insertsAndUpdates.inserts
.map(() => `(${Array(columnNames.length).fill("?").join(",")})`)
.join(",");
const insertResult = await localConnection.query(`
INSERT INTO purchase_orders (${columnNames.join(",")})
VALUES ${insertPlaceholders}
`, insertsAndUpdates.inserts.map(i => i.values).flat());
const affectedRows = insertResult[0].affectedRows;
// For an upsert, MySQL counts rows twice for updates
// So if affectedRows is odd, we have (updates * 2 + inserts)
const updates = Math.floor(affectedRows / 2);
const inserts = affectedRows - (updates * 2);
chunk.forEach((product, idx) => {
const po = batch.find(p => p.po_id === product.po_id);
if (!po) return;
values.push(
product.po_id,
product.pid,
product.sku,
product.name,
po.vendor,
po.date,
po.expected_date,
po.status,
po.notes || po.long_note,
product.ordered,
product.cost_price
);
const offset = idx * 11; // Updated to match 11 fields
placeholders.push(`($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}, $${offset + 6}, $${offset + 7}, $${offset + 8}, $${offset + 9}, $${offset + 10}, $${offset + 11})`);
recordsAdded += inserts;
recordsUpdated += Math.floor(updates); // Ensure we never have fractional updates
processed += batchProcessed;
}
// Handle updates - now we know these actually have changes
if (insertsAndUpdates.updates.length > 0) {
const updatePlaceholders = insertsAndUpdates.updates
.map(() => `(${Array(columnNames.length).fill("?").join(",")})`)
.join(",");
const updateResult = await localConnection.query(`
INSERT INTO purchase_orders (${columnNames.join(",")})
VALUES ${updatePlaceholders}
ON DUPLICATE KEY UPDATE ${columnNames
.filter((col) => col !== "po_id" && col !== "pid")
.map((col) => `${col} = VALUES(${col})`)
.join(",")};
`, insertsAndUpdates.updates.map(u => u.values).flat());
const affectedRows = updateResult[0].affectedRows;
// For an upsert, MySQL counts rows twice for updates
// So if affectedRows is odd, we have (updates * 2 + inserts)
const updates = Math.floor(affectedRows / 2);
const inserts = affectedRows - (updates * 2);
recordsUpdated += Math.floor(updates); // Ensure we never have fractional updates
processed += batchProcessed;
}
// Update progress based on time interval
const now = Date.now();
if (now - lastProgressUpdate >= PROGRESS_INTERVAL || processed === totalItems) {
outputProgress({
status: "running",
operation: "Purchase orders import",
current: processed,
total: totalItems,
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
remaining: estimateRemaining(startTime, processed, totalItems),
rate: calculateRate(startTime, processed)
});
if (placeholders.length > 0) {
await localConnection.query(`
INSERT INTO temp_purchase_orders (
po_id, pid, sku, name, vendor, date, expected_date,
status, notes, ordered, cost_price
)
VALUES ${placeholders.join(',')}
ON CONFLICT (po_id, pid) DO UPDATE SET
sku = EXCLUDED.sku,
name = EXCLUDED.name,
vendor = EXCLUDED.vendor,
date = EXCLUDED.date,
expected_date = EXCLUDED.expected_date,
status = EXCLUDED.status,
notes = EXCLUDED.notes,
ordered = EXCLUDED.ordered,
cost_price = EXCLUDED.cost_price
`, values);
}
processed += chunk.length;
// Update progress based on time interval
const now = Date.now();
if (now - lastProgressUpdate >= PROGRESS_INTERVAL || processed === totalItems) {
outputProgress({
status: "running",
operation: "Purchase orders import",
current: processed,
total: totalItems,
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
remaining: estimateRemaining(startTime, processed, totalItems),
rate: calculateRate(startTime, processed)
});
lastProgressUpdate = now;
}
lastProgressUpdate = now;
}
}
}
// Insert final data into purchase_orders table in chunks
const FINAL_CHUNK_SIZE = 1000;
let totalProcessed = 0;
const totalPosResult = await localConnection.query('SELECT COUNT(*) as total_pos FROM temp_purchase_orders');
const total_pos = parseInt(totalPosResult.rows?.[0]?.total_pos || '0', 10);
outputProgress({
status: "running",
operation: "Purchase orders final import",
message: `Processing ${total_pos} purchase orders for final import`,
current: 0,
total: total_pos
});
// Process in chunks using cursor-based pagination
let lastPoId = 0;
let lastPid = 0;
let recordsAdded = 0;
let recordsUpdated = 0;
while (true) {
console.log('Fetching next chunk with lastPoId:', lastPoId, 'lastPid:', lastPid);
const chunkResult = await localConnection.query(`
SELECT po_id, pid FROM temp_purchase_orders
WHERE (po_id, pid) > ($1, $2)
ORDER BY po_id, pid
LIMIT $3
`, [lastPoId, lastPid, FINAL_CHUNK_SIZE]);
if (!chunkResult?.rows) {
console.error('No rows returned from chunk query:', chunkResult);
break;
}
const chunk = chunkResult.rows;
console.log('Got chunk of size:', chunk.length);
if (chunk.length === 0) break;
const result = await localConnection.query(`
WITH inserted_pos AS (
INSERT INTO purchase_orders (
po_id, pid, sku, name, cost_price, po_cost_price,
vendor, date, expected_date, status, notes,
ordered, received, receiving_status,
received_date, last_received_date, received_by,
receiving_history
)
SELECT
po.po_id,
po.pid,
po.sku,
po.name,
COALESCE(
(
SELECT cost_each
FROM temp_po_receivings r2
WHERE r2.pid = po.pid
AND r2.po_id = po.po_id
AND r2.is_alt_po = 0
AND r2.cost_each > 0
ORDER BY r2.received_date
LIMIT 1
),
po.cost_price
) as cost_price,
po.cost_price as po_cost_price,
po.vendor,
po.date,
po.expected_date,
po.status,
po.notes,
po.ordered,
COALESCE(SUM(CASE WHEN r.is_alt_po = 0 THEN r.qty_each END), 0) as received,
CASE
WHEN COUNT(r.receiving_id) = 0 THEN 1 -- created
WHEN SUM(CASE WHEN r.is_alt_po = 0 THEN r.qty_each END) < po.ordered THEN 30 -- partial
ELSE 40 -- full
END as receiving_status,
MIN(CASE WHEN r.is_alt_po = 0 THEN r.received_date END) as received_date,
MAX(CASE WHEN r.is_alt_po = 0 THEN r.received_date END) as last_received_date,
(
SELECT r2.received_by_name
FROM temp_po_receivings r2
WHERE r2.pid = po.pid
AND r2.is_alt_po = 0
ORDER BY r2.received_date
LIMIT 1
) as received_by,
jsonb_build_object(
'ordered_qty', po.ordered,
'total_received', COALESCE(SUM(CASE WHEN r.is_alt_po = 0 THEN r.qty_each END), 0),
'remaining_unfulfilled', GREATEST(0, po.ordered - COALESCE(SUM(CASE WHEN r.is_alt_po = 0 THEN r.qty_each END), 0)),
'excess_received', GREATEST(0, COALESCE(SUM(CASE WHEN r.is_alt_po = 0 THEN r.qty_each END), 0) - po.ordered),
'po_cost', po.cost_price,
'actual_cost', COALESCE(
(
SELECT cost_each
FROM temp_po_receivings r2
WHERE r2.pid = po.pid
AND r2.is_alt_po = 0
AND r2.cost_each > 0
ORDER BY r2.received_date
LIMIT 1
),
po.cost_price
),
'fulfillment', (
SELECT jsonb_agg(
jsonb_build_object(
'receiving_id', r2.receiving_id,
'qty_applied', CASE
WHEN r2.running_total <= po.ordered THEN r2.qty_each
WHEN r2.running_total - r2.qty_each < po.ordered THEN po.ordered - (r2.running_total - r2.qty_each)
ELSE 0
END,
'qty_total', r2.qty_each,
'cost', r2.cost_each,
'date', r2.received_date,
'received_by', r2.received_by,
'received_by_name', r2.received_by_name,
'type', CASE r2.is_alt_po
WHEN 0 THEN 'original'
WHEN 1 THEN 'alternate'
ELSE 'no_po'
END,
'remaining_qty', CASE
WHEN r2.running_total <= po.ordered THEN 0
WHEN r2.running_total - r2.qty_each < po.ordered THEN r2.running_total - po.ordered
ELSE r2.qty_each
END,
'is_excess', r2.running_total > po.ordered
)
ORDER BY r2.received_date
)
FROM (
SELECT
r2.*,
SUM(r2.qty_each) OVER (
PARTITION BY r2.pid
ORDER BY r2.received_date
ROWS UNBOUNDED PRECEDING
) as running_total
FROM temp_po_receivings r2
WHERE r2.pid = po.pid
) r2
),
'alternate_po_receivings', (
SELECT jsonb_agg(
jsonb_build_object(
'receiving_id', r2.receiving_id,
'qty', r2.qty_each,
'cost', r2.cost_each,
'date', r2.received_date,
'received_by', r2.received_by,
'received_by_name', r2.received_by_name
)
ORDER BY r2.received_date
)
FROM temp_po_receivings r2
WHERE r2.pid = po.pid AND r2.is_alt_po = 1
),
'no_po_receivings', (
SELECT jsonb_agg(
jsonb_build_object(
'receiving_id', r2.receiving_id,
'qty', r2.qty_each,
'cost', r2.cost_each,
'date', r2.received_date,
'received_by', r2.received_by,
'received_by_name', r2.received_by_name
)
ORDER BY r2.received_date
)
FROM temp_po_receivings r2
WHERE r2.pid = po.pid AND r2.is_alt_po = 2
)
) as receiving_history
FROM temp_purchase_orders po
LEFT JOIN temp_po_receivings r ON po.pid = r.pid
WHERE (po.po_id, po.pid) IN (
SELECT po_id, pid FROM UNNEST($1::int[], $2::int[])
)
GROUP BY po.po_id, po.pid, po.sku, po.name, po.vendor, po.date,
po.expected_date, po.status, po.notes, po.ordered, po.cost_price
ON CONFLICT (po_id, pid) DO UPDATE SET
vendor = EXCLUDED.vendor,
date = EXCLUDED.date,
expected_date = EXCLUDED.expected_date,
status = EXCLUDED.status,
notes = EXCLUDED.notes,
ordered = EXCLUDED.ordered,
received = EXCLUDED.received,
receiving_status = EXCLUDED.receiving_status,
received_date = EXCLUDED.received_date,
last_received_date = EXCLUDED.last_received_date,
received_by = EXCLUDED.received_by,
receiving_history = EXCLUDED.receiving_history,
cost_price = EXCLUDED.cost_price,
po_cost_price = EXCLUDED.po_cost_price
RETURNING xmax
)
SELECT
COUNT(*) FILTER (WHERE xmax = 0) as inserted,
COUNT(*) FILTER (WHERE xmax <> 0) as updated
FROM inserted_pos
`, [
chunk.map(r => r.po_id),
chunk.map(r => r.pid)
]);
// Add debug logging
console.log('Insert result:', result?.rows?.[0]);
// Handle the result properly for PostgreSQL with more defensive coding
const resultRow = result?.rows?.[0] || {};
const insertCount = parseInt(resultRow.inserted || '0', 10);
const updateCount = parseInt(resultRow.updated || '0', 10);
recordsAdded += insertCount;
recordsUpdated += updateCount;
totalProcessed += chunk.length;
// Update progress
outputProgress({
status: "running",
operation: "Purchase orders final import",
message: `Processed ${totalProcessed} of ${total_pos} purchase orders`,
current: totalProcessed,
total: total_pos,
elapsed: formatElapsedTime((Date.now() - startTime) / 1000),
remaining: estimateRemaining(startTime, totalProcessed, total_pos),
rate: calculateRate(startTime, totalProcessed)
});
// Update last processed IDs for next chunk with safety check
if (chunk.length > 0) {
const lastItem = chunk[chunk.length - 1];
if (lastItem) {
lastPoId = lastItem.po_id;
lastPid = lastItem.pid;
}
}
}
// Update sync status
// Only update sync status if we get here (no errors thrown)
await localConnection.query(`
INSERT INTO sync_status (table_name, last_sync_timestamp)
VALUES ('purchase_orders', NOW())
ON CONFLICT (table_name) DO UPDATE SET
last_sync_timestamp = NOW()
`);
// Clean up temporary tables
await localConnection.query(`
DROP TABLE IF EXISTS temp_purchase_orders;
DROP TABLE IF EXISTS temp_po_receivings;
ON DUPLICATE KEY UPDATE
last_sync_timestamp = NOW(),
last_sync_id = LAST_INSERT_ID(last_sync_id)
`);
return {
status: "complete",
recordsAdded,
recordsUpdated,
totalRecords: processed
totalImported: totalItems,
recordsAdded: recordsAdded || 0,
recordsUpdated: recordsUpdated || 0,
incrementalUpdate,
lastSyncTime
};
} catch (error) {
console.error("Error during purchase orders import:", error);
// Attempt cleanup on error
try {
await localConnection.query(`
DROP TABLE IF EXISTS temp_purchase_orders;
DROP TABLE IF EXISTS temp_po_receivings;
`);
} catch (cleanupError) {
console.error('Error during cleanup:', cleanupError);
}
outputProgress({
operation: `${incrementalUpdate ? 'Incremental' : 'Full'} purchase orders import failed`,
status: "error",
error: error.message,
});
throw error;
}
}
module.exports = importPurchaseOrders;
module.exports = importPurchaseOrders;

88
inventory-server/scripts/import/utils.js
View File

@@ -1,6 +1,5 @@
const mysql = require("mysql2/promise");
const { Client } = require("ssh2");
const { Pool } = require('pg');
const dotenv = require("dotenv");
const path = require("path");
@@ -42,90 +41,23 @@ async function setupSshTunnel(sshConfig) {
async function setupConnections(sshConfig) {
const tunnel = await setupSshTunnel(sshConfig);
// Setup MySQL connection for production
const prodConnection = await mysql.createConnection({
...sshConfig.prodDbConfig,
stream: tunnel.stream,
});
// Setup PostgreSQL connection pool for local
const localPool = new Pool(sshConfig.localDbConfig);
const localConnection = await mysql.createPool({
...sshConfig.localDbConfig,
waitForConnections: true,
connectionLimit: 10,
queueLimit: 0
});
// Test the PostgreSQL connection
try {
const client = await localPool.connect();
await client.query('SELECT NOW()');
client.release();
console.log('PostgreSQL connection successful');
} catch (err) {
console.error('PostgreSQL connection error:', err);
throw err;
}
// Create a wrapper for the PostgreSQL pool to match MySQL interface
const localConnection = {
_client: null,
_transactionActive: false,
query: async (text, params) => {
// If we're not in a transaction, use the pool directly
if (!localConnection._transactionActive) {
const client = await localPool.connect();
try {
const result = await client.query(text, params);
return [result];
} finally {
client.release();
}
}
// If we're in a transaction, use the dedicated client
if (!localConnection._client) {
throw new Error('No active transaction client');
}
const result = await localConnection._client.query(text, params);
return [result];
},
beginTransaction: async () => {
if (localConnection._transactionActive) {
throw new Error('Transaction already active');
}
localConnection._client = await localPool.connect();
await localConnection._client.query('BEGIN');
localConnection._transactionActive = true;
},
commit: async () => {
if (!localConnection._transactionActive) {
throw new Error('No active transaction to commit');
}
await localConnection._client.query('COMMIT');
localConnection._client.release();
localConnection._client = null;
localConnection._transactionActive = false;
},
rollback: async () => {
if (!localConnection._transactionActive) {
throw new Error('No active transaction to rollback');
}
await localConnection._client.query('ROLLBACK');
localConnection._client.release();
localConnection._client = null;
localConnection._transactionActive = false;
},
end: async () => {
if (localConnection._client) {
localConnection._client.release();
localConnection._client = null;
}
await localPool.end();
}
return {
ssh: tunnel.ssh,
prodConnection,
localConnection
};
return { prodConnection, localConnection, tunnel };
}
// Helper function to close connections

435
inventory-server/scripts/metrics/brand-metrics.js
View File

@@ -4,19 +4,50 @@ const { getConnection } = require('./utils/db');
async function calculateBrandMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) {
const connection = await getConnection();
let success = false;
let processedOrders = 0;
const BATCH_SIZE = 5000;
try {
// Get last calculation timestamp
const [lastCalc] = await connection.query(`
SELECT last_calculation_timestamp
FROM calculate_status
WHERE module_name = 'brand_metrics'
`);
const lastCalculationTime = lastCalc[0]?.last_calculation_timestamp || '1970-01-01';
// Get total count of brands needing updates
const [brandCount] = await connection.query(`
SELECT COUNT(DISTINCT p.brand) as count
FROM products p
LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
WHERE p.brand IS NOT NULL
AND (
p.updated > ?
OR o.id IS NOT NULL
)
`, [lastCalculationTime, lastCalculationTime]);
const totalBrands = brandCount[0].count;
if (totalBrands === 0) {
console.log('No brands need metric updates');
return {
processedProducts: 0,
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
};
}
if (isCancelled) {
outputProgress({
status: 'cancelled',
operation: 'Brand metrics calculation cancelled',
current: processedCount,
total: totalProducts,
total: totalBrands,
elapsed: formatElapsedTime(startTime),
remaining: null,
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
percentage: ((processedCount / totalBrands) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -31,23 +62,15 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount =
};
}
// Get order count that will be processed
const [orderCount] = await connection.query(`
SELECT COUNT(*) as count
FROM orders o
WHERE o.canceled = false
`);
processedOrders = orderCount[0].count;
outputProgress({
status: 'running',
operation: 'Starting brand metrics calculation',
current: processedCount,
total: totalProducts,
total: totalBrands,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
remaining: estimateRemaining(startTime, processedCount, totalBrands),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
percentage: ((processedCount / totalBrands) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -55,237 +78,179 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount =
}
});
// Calculate brand metrics with optimized queries
await connection.query(`
INSERT INTO brand_metrics (
brand,
product_count,
active_products,
total_stock_units,
total_stock_cost,
total_stock_retail,
total_revenue,
avg_margin,
growth_rate
)
WITH filtered_products AS (
SELECT
p.*,
CASE
WHEN p.stock_quantity <= 5000 AND p.stock_quantity >= 0
THEN p.pid
END as valid_pid,
CASE
WHEN p.visible = true
AND p.stock_quantity <= 5000
AND p.stock_quantity >= 0
THEN p.pid
END as active_pid,
CASE
WHEN p.stock_quantity IS NULL
OR p.stock_quantity < 0
OR p.stock_quantity > 5000
THEN 0
ELSE p.stock_quantity
END as valid_stock
// Process in batches
let lastBrand = '';
while (true) {
if (isCancelled) break;
const [batch] = await connection.query(`
SELECT DISTINCT p.brand
FROM products p
FORCE INDEX (idx_brand)
LEFT JOIN orders o FORCE INDEX (idx_orders_metrics) ON p.pid = o.pid AND o.updated > ?
WHERE p.brand IS NOT NULL
),
sales_periods AS (
AND p.brand > ?
AND (
p.updated > ?
OR o.id IS NOT NULL
)
ORDER BY p.brand
LIMIT ?
`, [lastCalculationTime, lastBrand, lastCalculationTime, BATCH_SIZE]);
if (batch.length === 0) break;
// Create temporary tables for better performance
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_product_stats');
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_sales_stats');
await connection.query(`
CREATE TEMPORARY TABLE temp_product_stats (
brand VARCHAR(100) NOT NULL,
product_count INT,
active_products INT,
total_stock_units INT,
total_stock_cost DECIMAL(15,2),
total_stock_retail DECIMAL(15,2),
total_revenue DECIMAL(15,2),
avg_margin DECIMAL(5,2),
PRIMARY KEY (brand),
INDEX (total_revenue),
INDEX (product_count)
) ENGINE=MEMORY
`);
await connection.query(`
CREATE TEMPORARY TABLE temp_sales_stats (
brand VARCHAR(100) NOT NULL,
current_period_sales DECIMAL(15,2),
previous_period_sales DECIMAL(15,2),
PRIMARY KEY (brand),
INDEX (current_period_sales),
INDEX (previous_period_sales)
) ENGINE=MEMORY
`);
// Populate product stats with optimized index usage
await connection.query(`
INSERT INTO temp_product_stats
SELECT
p.brand,
SUM(o.quantity * (o.price - COALESCE(o.discount, 0))) as period_revenue,
SUM(o.quantity * (o.price - COALESCE(o.discount, 0) - p.cost_price)) as period_margin,
COUNT(DISTINCT DATE(o.date)) as period_days,
CASE
WHEN o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 3 MONTH) THEN 'current'
WHEN o.date BETWEEN DATE_SUB(CURRENT_DATE, INTERVAL 15 MONTH)
AND DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH) THEN 'previous'
END as period_type
FROM filtered_products p
JOIN orders o ON p.pid = o.pid
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 15 MONTH)
GROUP BY p.brand, period_type
),
brand_data AS (
SELECT
p.brand,
COUNT(DISTINCT p.valid_pid) as product_count,
COUNT(DISTINCT p.active_pid) as active_products,
SUM(p.valid_stock) as total_stock_units,
SUM(p.valid_stock * p.cost_price) as total_stock_cost,
SUM(p.valid_stock * p.price) as total_stock_retail,
COALESCE(SUM(o.quantity * (o.price - COALESCE(o.discount, 0))), 0) as total_revenue,
CASE
WHEN SUM(o.quantity * o.price) > 0
THEN GREATEST(
-100.0,
LEAST(
100.0,
(
SUM(o.quantity * o.price) - -- Use gross revenue (before discounts)
SUM(o.quantity * COALESCE(p.cost_price, 0)) -- Total costs
) * 100.0 /
NULLIF(SUM(o.quantity * o.price), 0) -- Divide by gross revenue
)
)
ELSE 0
END as avg_margin
FROM filtered_products p
LEFT JOIN orders o ON p.pid = o.pid AND o.canceled = false
GROUP BY p.brand
)
SELECT
bd.brand,
bd.product_count,
bd.active_products,
bd.total_stock_units,
bd.total_stock_cost,
bd.total_stock_retail,
bd.total_revenue,
bd.avg_margin,
CASE
WHEN MAX(CASE WHEN sp.period_type = 'previous' THEN sp.period_revenue END) = 0
AND MAX(CASE WHEN sp.period_type = 'current' THEN sp.period_revenue END) > 0
THEN 100.0
WHEN MAX(CASE WHEN sp.period_type = 'previous' THEN sp.period_revenue END) = 0
THEN 0.0
ELSE GREATEST(
-100.0,
LEAST(
((MAX(CASE WHEN sp.period_type = 'current' THEN sp.period_revenue END) -
MAX(CASE WHEN sp.period_type = 'previous' THEN sp.period_revenue END)) /
NULLIF(ABS(MAX(CASE WHEN sp.period_type = 'previous' THEN sp.period_revenue END)), 0)) * 100.0,
999.99
COUNT(DISTINCT p.pid) as product_count,
COUNT(DISTINCT CASE WHEN p.visible = true THEN p.pid END) as active_products,
SUM(p.stock_quantity) as total_stock_units,
SUM(p.stock_quantity * p.cost_price) as total_stock_cost,
SUM(p.stock_quantity * p.price) as total_stock_retail,
SUM(pm.total_revenue) as total_revenue,
AVG(pm.avg_margin_percent) as avg_margin
FROM products p
FORCE INDEX (idx_brand)
LEFT JOIN product_metrics pm FORCE INDEX (PRIMARY) ON p.pid = pm.pid
WHERE p.brand IN (?)
AND (
p.updated > ?
OR EXISTS (
SELECT 1 FROM orders o FORCE INDEX (idx_orders_metrics)
WHERE o.pid = p.pid
AND o.updated > ?
)
)
END as growth_rate
FROM brand_data bd
LEFT JOIN sales_periods sp ON bd.brand = sp.brand
GROUP BY bd.brand, bd.product_count, bd.active_products, bd.total_stock_units,
bd.total_stock_cost, bd.total_stock_retail, bd.total_revenue, bd.avg_margin
ON DUPLICATE KEY UPDATE
product_count = VALUES(product_count),
active_products = VALUES(active_products),
total_stock_units = VALUES(total_stock_units),
total_stock_cost = VALUES(total_stock_cost),
total_stock_retail = VALUES(total_stock_retail),
total_revenue = VALUES(total_revenue),
avg_margin = VALUES(avg_margin),
growth_rate = VALUES(growth_rate),
last_calculated_at = CURRENT_TIMESTAMP
`);
GROUP BY p.brand
`, [batch.map(row => row.brand), lastCalculationTime, lastCalculationTime]);
processedCount = Math.floor(totalProducts * 0.97);
outputProgress({
status: 'running',
operation: 'Brand metrics calculated, starting time-based metrics',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
// Calculate brand time-based metrics with optimized query
await connection.query(`
INSERT INTO brand_time_metrics (
brand,
year,
month,
product_count,
active_products,
total_stock_units,
total_stock_cost,
total_stock_retail,
total_revenue,
avg_margin
)
WITH filtered_products AS (
SELECT
p.*,
CASE WHEN p.stock_quantity <= 5000 THEN p.pid END as valid_pid,
CASE WHEN p.visible = true AND p.stock_quantity <= 5000 THEN p.pid END as active_pid,
CASE
WHEN p.stock_quantity IS NULL OR p.stock_quantity < 0 OR p.stock_quantity > 5000 THEN 0
ELSE p.stock_quantity
END as valid_stock
FROM products p
WHERE p.brand IS NOT NULL
),
monthly_metrics AS (
// Populate sales stats with optimized index usage
await connection.query(`
INSERT INTO temp_sales_stats
SELECT
p.brand,
YEAR(o.date) as year,
MONTH(o.date) as month,
COUNT(DISTINCT p.valid_pid) as product_count,
COUNT(DISTINCT p.active_pid) as active_products,
SUM(p.valid_stock) as total_stock_units,
SUM(p.valid_stock * p.cost_price) as total_stock_cost,
SUM(p.valid_stock * p.price) as total_stock_retail,
SUM(o.quantity * o.price) as total_revenue,
CASE
WHEN SUM(o.quantity * o.price) > 0
THEN GREATEST(
-100.0,
LEAST(
100.0,
(
SUM(o.quantity * o.price) - -- Use gross revenue (before discounts)
SUM(o.quantity * COALESCE(p.cost_price, 0)) -- Total costs
) * 100.0 /
NULLIF(SUM(o.quantity * o.price), 0) -- Divide by gross revenue
)
)
SUM(CASE
WHEN o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 30 DAY)
THEN o.quantity * o.price
ELSE 0
END as avg_margin
FROM filtered_products p
LEFT JOIN orders o ON p.pid = o.pid AND o.canceled = false
WHERE o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
GROUP BY p.brand, YEAR(o.date), MONTH(o.date)
)
SELECT *
FROM monthly_metrics
ON DUPLICATE KEY UPDATE
product_count = VALUES(product_count),
active_products = VALUES(active_products),
total_stock_units = VALUES(total_stock_units),
total_stock_cost = VALUES(total_stock_cost),
total_stock_retail = VALUES(total_stock_retail),
total_revenue = VALUES(total_revenue),
avg_margin = VALUES(avg_margin)
`);
END) as current_period_sales,
SUM(CASE
WHEN o.date BETWEEN DATE_SUB(CURRENT_DATE, INTERVAL 60 DAY) AND DATE_SUB(CURRENT_DATE, INTERVAL 30 DAY)
THEN o.quantity * o.price
ELSE 0
END) as previous_period_sales
FROM products p
FORCE INDEX (idx_brand)
INNER JOIN orders o FORCE INDEX (idx_orders_metrics) ON p.pid = o.pid
AND o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 60 DAY)
AND o.updated > ?
WHERE p.brand IN (?)
GROUP BY p.brand
`, [lastCalculationTime, batch.map(row => row.brand)]);
processedCount = Math.floor(totalProducts * 0.99);
outputProgress({
status: 'running',
operation: 'Brand time-based metrics calculated',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
// Update metrics using temp tables
await connection.query(`
INSERT INTO brand_metrics (
brand,
product_count,
active_products,
total_stock_units,
total_stock_cost,
total_stock_retail,
total_revenue,
avg_margin,
growth_rate,
last_calculated_at
)
SELECT
ps.brand,
COALESCE(ps.product_count, 0) as product_count,
COALESCE(ps.active_products, 0) as active_products,
COALESCE(ps.total_stock_units, 0) as total_stock_units,
COALESCE(ps.total_stock_cost, 0) as total_stock_cost,
COALESCE(ps.total_stock_retail, 0) as total_stock_retail,
COALESCE(ps.total_revenue, 0) as total_revenue,
COALESCE(ps.avg_margin, 0) as avg_margin,
CASE
WHEN COALESCE(ss.previous_period_sales, 0) = 0 AND COALESCE(ss.current_period_sales, 0) > 0 THEN 100
WHEN COALESCE(ss.previous_period_sales, 0) = 0 THEN 0
ELSE LEAST(999.99, GREATEST(-100,
((COALESCE(ss.current_period_sales, 0) / ss.previous_period_sales) - 1) * 100
))
END as growth_rate,
NOW() as last_calculated_at
FROM temp_product_stats ps
LEFT JOIN temp_sales_stats ss ON ps.brand = ss.brand
ON DUPLICATE KEY UPDATE
product_count = VALUES(product_count),
active_products = VALUES(active_products),
total_stock_units = VALUES(total_stock_units),
total_stock_cost = VALUES(total_stock_cost),
total_stock_retail = VALUES(total_stock_retail),
total_revenue = VALUES(total_revenue),
avg_margin = VALUES(avg_margin),
growth_rate = VALUES(growth_rate),
last_calculated_at = NOW()
`);
// Clean up temp tables
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_product_stats');
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_sales_stats');
lastBrand = batch[batch.length - 1].brand;
processedCount += batch.length;
outputProgress({
status: 'running',
operation: 'Processing brand metrics batch',
current: processedCount,
total: totalBrands,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalBrands),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalBrands) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
}
// If we get here, everything completed successfully
success = true;
@@ -299,7 +264,7 @@ async function calculateBrandMetrics(startTime, totalProducts, processedCount =
return {
processedProducts: processedCount,
processedOrders,
processedOrders: 0,
processedPurchaseOrders: 0,
success
};

640
inventory-server/scripts/metrics/category-metrics.js
View File

@@ -4,19 +4,52 @@ const { getConnection } = require('./utils/db');
async function calculateCategoryMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) {
const connection = await getConnection();
let success = false;
let processedOrders = 0;
const BATCH_SIZE = 5000;
try {
// Get last calculation timestamp
const [lastCalc] = await connection.query(`
SELECT last_calculation_timestamp
FROM calculate_status
WHERE module_name = 'category_metrics'
`);
const lastCalculationTime = lastCalc[0]?.last_calculation_timestamp || '1970-01-01';
// Get total count of categories needing updates
const [categoryCount] = await connection.query(`
SELECT COUNT(DISTINCT c.cat_id) as count
FROM categories c
JOIN product_categories pc ON c.cat_id = pc.cat_id
LEFT JOIN products p ON pc.pid = p.pid AND p.updated > ?
LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
WHERE c.status = 'active'
AND (
p.pid IS NOT NULL
OR o.id IS NOT NULL
)
`, [lastCalculationTime, lastCalculationTime]);
const totalCategories = categoryCount[0].count;
if (totalCategories === 0) {
console.log('No categories need metric updates');
return {
processedProducts: 0,
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
};
}
if (isCancelled) {
outputProgress({
status: 'cancelled',
operation: 'Category metrics calculation cancelled',
current: processedCount,
total: totalProducts,
total: totalCategories,
elapsed: formatElapsedTime(startTime),
remaining: null,
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
percentage: ((processedCount / totalCategories) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -31,69 +64,15 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
};
}
// Get order count that will be processed
const [orderCount] = await connection.query(`
SELECT COUNT(*) as count
FROM orders o
WHERE o.canceled = false
`);
processedOrders = orderCount[0].count;
outputProgress({
status: 'running',
operation: 'Starting category metrics calculation',
current: processedCount,
total: totalProducts,
total: totalCategories,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
remaining: estimateRemaining(startTime, processedCount, totalCategories),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
// First, calculate base category metrics
await connection.query(`
INSERT INTO category_metrics (
category_id,
product_count,
active_products,
total_value,
status,
last_calculated_at
)
SELECT
c.cat_id,
COUNT(DISTINCT p.pid) as product_count,
COUNT(DISTINCT CASE WHEN p.visible = true THEN p.pid END) as active_products,
COALESCE(SUM(p.stock_quantity * p.cost_price), 0) as total_value,
c.status,
NOW() as last_calculated_at
FROM categories c
LEFT JOIN product_categories pc ON c.cat_id = pc.cat_id
LEFT JOIN products p ON pc.pid = p.pid
GROUP BY c.cat_id, c.status
ON DUPLICATE KEY UPDATE
product_count = VALUES(product_count),
active_products = VALUES(active_products),
total_value = VALUES(total_value),
status = VALUES(status),
last_calculated_at = VALUES(last_calculated_at)
`);
processedCount = Math.floor(totalProducts * 0.90);
outputProgress({
status: 'running',
operation: 'Base category metrics calculated, updating with margin data',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
percentage: ((processedCount / totalCategories) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -101,395 +80,172 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
}
});
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
// Process in batches
let lastCatId = 0;
while (true) {
if (isCancelled) break;
// Then update with margin and turnover data
await connection.query(`
WITH category_sales AS (
SELECT
pc.cat_id,
SUM(o.quantity * o.price) as total_sales,
SUM(o.quantity * (o.price - p.cost_price)) as total_margin,
SUM(o.quantity) as units_sold,
AVG(GREATEST(p.stock_quantity, 0)) as avg_stock,
COUNT(DISTINCT DATE(o.date)) as active_days
FROM product_categories pc
JOIN products p ON pc.pid = p.pid
JOIN orders o ON p.pid = o.pid
LEFT JOIN turnover_config tc ON
(tc.category_id = pc.cat_id AND tc.vendor = p.vendor) OR
(tc.category_id = pc.cat_id AND tc.vendor IS NULL) OR
(tc.category_id IS NULL AND tc.vendor = p.vendor) OR
(tc.category_id IS NULL AND tc.vendor IS NULL)
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL COALESCE(tc.calculation_period_days, 30) DAY)
GROUP BY pc.cat_id
)
UPDATE category_metrics cm
JOIN category_sales cs ON cm.category_id = cs.cat_id
LEFT JOIN turnover_config tc ON
(tc.category_id = cm.category_id AND tc.vendor IS NULL) OR
(tc.category_id IS NULL AND tc.vendor IS NULL)
SET
cm.avg_margin = COALESCE(cs.total_margin * 100.0 / NULLIF(cs.total_sales, 0), 0),
cm.turnover_rate = CASE
WHEN cs.avg_stock > 0 AND cs.active_days > 0
THEN LEAST(
(cs.units_sold / cs.avg_stock) * (365.0 / cs.active_days),
999.99
)
ELSE 0
END,
cm.last_calculated_at = NOW()
`);
const [batch] = await connection.query(`
SELECT DISTINCT c.cat_id
FROM categories c
FORCE INDEX (PRIMARY)
JOIN product_categories pc FORCE INDEX (idx_category) ON c.cat_id = pc.cat_id
LEFT JOIN products p FORCE INDEX (PRIMARY) ON pc.pid = p.pid AND p.updated > ?
LEFT JOIN orders o FORCE INDEX (idx_orders_metrics) ON p.pid = o.pid AND o.updated > ?
WHERE c.status = 'active'
AND c.cat_id > ?
AND (
p.pid IS NOT NULL
OR o.id IS NOT NULL
)
ORDER BY c.cat_id
LIMIT ?
`, [lastCalculationTime, lastCalculationTime, lastCatId, BATCH_SIZE]);
processedCount = Math.floor(totalProducts * 0.95);
outputProgress({
status: 'running',
operation: 'Margin data updated, calculating growth rates',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (batch.length === 0) break;
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
// Create temporary tables for better performance
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_product_stats');
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_sales_stats');
// Finally update growth rates
await connection.query(`
WITH current_period AS (
SELECT
pc.cat_id,
SUM(o.quantity * (o.price - COALESCE(o.discount, 0)) /
(1 + COALESCE(ss.seasonality_factor, 0))) as revenue,
SUM(o.quantity * (o.price - COALESCE(o.discount, 0) - p.cost_price)) as gross_profit,
COUNT(DISTINCT DATE(o.date)) as days
FROM product_categories pc
JOIN products p ON pc.pid = p.pid
JOIN orders o ON p.pid = o.pid
LEFT JOIN sales_seasonality ss ON MONTH(o.date) = ss.month
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 3 MONTH)
GROUP BY pc.cat_id
),
previous_period AS (
SELECT
pc.cat_id,
SUM(o.quantity * (o.price - COALESCE(o.discount, 0)) /
(1 + COALESCE(ss.seasonality_factor, 0))) as revenue,
COUNT(DISTINCT DATE(o.date)) as days
FROM product_categories pc
JOIN products p ON pc.pid = p.pid
JOIN orders o ON p.pid = o.pid
LEFT JOIN sales_seasonality ss ON MONTH(o.date) = ss.month
WHERE o.canceled = false
AND o.date BETWEEN DATE_SUB(CURRENT_DATE, INTERVAL 15 MONTH)
AND DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
GROUP BY pc.cat_id
),
trend_data AS (
SELECT
pc.cat_id,
MONTH(o.date) as month,
SUM(o.quantity * (o.price - COALESCE(o.discount, 0)) /
(1 + COALESCE(ss.seasonality_factor, 0))) as revenue,
COUNT(DISTINCT DATE(o.date)) as days_in_month
FROM product_categories pc
JOIN products p ON pc.pid = p.pid
JOIN orders o ON p.pid = o.pid
LEFT JOIN sales_seasonality ss ON MONTH(o.date) = ss.month
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 15 MONTH)
GROUP BY pc.cat_id, MONTH(o.date)
),
trend_stats AS (
SELECT
cat_id,
COUNT(*) as n,
AVG(month) as avg_x,
AVG(revenue / NULLIF(days_in_month, 0)) as avg_y,
SUM(month * (revenue / NULLIF(days_in_month, 0))) as sum_xy,
SUM(month * month) as sum_xx
FROM trend_data
GROUP BY cat_id
HAVING COUNT(*) >= 6
),
trend_analysis AS (
SELECT
cat_id,
((n * sum_xy) - (avg_x * n * avg_y)) /
NULLIF((n * sum_xx) - (n * avg_x * avg_x), 0) as trend_slope,
avg_y as avg_daily_revenue
FROM trend_stats
),
margin_calc AS (
SELECT
pc.cat_id,
CASE
WHEN SUM(o.quantity * o.price) > 0 THEN
GREATEST(
-100.0,
LEAST(
100.0,
(
SUM(o.quantity * o.price) - -- Use gross revenue (before discounts)
SUM(o.quantity * COALESCE(p.cost_price, 0)) -- Total costs
) * 100.0 /
NULLIF(SUM(o.quantity * o.price), 0) -- Divide by gross revenue
)
)
ELSE NULL
END as avg_margin
FROM product_categories pc
JOIN products p ON pc.pid = p.pid
JOIN orders o ON p.pid = o.pid
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 3 MONTH)
GROUP BY pc.cat_id
)
UPDATE category_metrics cm
LEFT JOIN current_period cp ON cm.category_id = cp.cat_id
LEFT JOIN previous_period pp ON cm.category_id = pp.cat_id
LEFT JOIN trend_analysis ta ON cm.category_id = ta.cat_id
LEFT JOIN margin_calc mc ON cm.category_id = mc.cat_id
SET
cm.growth_rate = CASE
WHEN pp.revenue = 0 AND COALESCE(cp.revenue, 0) > 0 THEN 100.0
WHEN pp.revenue = 0 OR cp.revenue IS NULL THEN 0.0
WHEN ta.trend_slope IS NOT NULL THEN
GREATEST(
-100.0,
LEAST(
(ta.trend_slope / NULLIF(ta.avg_daily_revenue, 0)) * 365 * 100,
999.99
)
)
ELSE
GREATEST(
-100.0,
LEAST(
((COALESCE(cp.revenue, 0) - pp.revenue) /
NULLIF(ABS(pp.revenue), 0)) * 100.0,
999.99
)
)
END,
cm.avg_margin = COALESCE(mc.avg_margin, cm.avg_margin),
cm.last_calculated_at = NOW()
WHERE cp.cat_id IS NOT NULL OR pp.cat_id IS NOT NULL
`);
await connection.query(`
CREATE TEMPORARY TABLE temp_product_stats (
cat_id BIGINT NOT NULL,
product_count INT,
active_products INT,
total_value DECIMAL(15,2),
avg_margin DECIMAL(5,2),
turnover_rate DECIMAL(10,2),
PRIMARY KEY (cat_id),
INDEX (product_count),
INDEX (total_value)
) ENGINE=MEMORY
`);
processedCount = Math.floor(totalProducts * 0.97);
outputProgress({
status: 'running',
operation: 'Growth rates calculated, updating time-based metrics',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
await connection.query(`
CREATE TEMPORARY TABLE temp_sales_stats (
cat_id BIGINT NOT NULL,
recent_revenue DECIMAL(15,2),
previous_revenue DECIMAL(15,2),
PRIMARY KEY (cat_id),
INDEX (recent_revenue),
INDEX (previous_revenue)
) ENGINE=MEMORY
`);
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
// Calculate time-based metrics
await connection.query(`
INSERT INTO category_time_metrics (
category_id,
year,
month,
product_count,
active_products,
total_value,
total_revenue,
avg_margin,
turnover_rate
)
SELECT
pc.cat_id,
YEAR(o.date) as year,
MONTH(o.date) as month,
COUNT(DISTINCT p.pid) as product_count,
COUNT(DISTINCT CASE WHEN p.visible = true THEN p.pid END) as active_products,
SUM(p.stock_quantity * p.cost_price) as total_value,
SUM(o.quantity * o.price) as total_revenue,
CASE
WHEN SUM(o.quantity * o.price) > 0 THEN
LEAST(
GREATEST(
SUM(o.quantity * (o.price - GREATEST(p.cost_price, 0))) * 100.0 /
SUM(o.quantity * o.price),
-100
),
100
)
ELSE 0
END as avg_margin,
COALESCE(
LEAST(
SUM(o.quantity) / NULLIF(AVG(GREATEST(p.stock_quantity, 0)), 0),
999.99
),
0
) as turnover_rate
FROM product_categories pc
JOIN products p ON pc.pid = p.pid
JOIN orders o ON p.pid = o.pid
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
GROUP BY pc.cat_id, YEAR(o.date), MONTH(o.date)
ON DUPLICATE KEY UPDATE
product_count = VALUES(product_count),
active_products = VALUES(active_products),
total_value = VALUES(total_value),
total_revenue = VALUES(total_revenue),
avg_margin = VALUES(avg_margin),
turnover_rate = VALUES(turnover_rate)
`);
processedCount = Math.floor(totalProducts * 0.99);
outputProgress({
status: 'running',
operation: 'Time-based metrics calculated, updating category-sales metrics',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
// Calculate category-sales metrics
await connection.query(`
INSERT INTO category_sales_metrics (
category_id,
brand,
period_start,
period_end,
avg_daily_sales,
total_sold,
num_products,
avg_price,
last_calculated_at
)
WITH date_ranges AS (
// Populate product stats
await connection.query(`
INSERT INTO temp_product_stats
SELECT
DATE_SUB(CURRENT_DATE, INTERVAL 30 DAY) as period_start,
CURRENT_DATE as period_end
UNION ALL
SELECT
DATE_SUB(CURRENT_DATE, INTERVAL 90 DAY),
DATE_SUB(CURRENT_DATE, INTERVAL 31 DAY)
UNION ALL
SELECT
DATE_SUB(CURRENT_DATE, INTERVAL 180 DAY),
DATE_SUB(CURRENT_DATE, INTERVAL 91 DAY)
UNION ALL
SELECT
DATE_SUB(CURRENT_DATE, INTERVAL 365 DAY),
DATE_SUB(CURRENT_DATE, INTERVAL 181 DAY)
),
sales_data AS (
SELECT
pc.cat_id,
COALESCE(p.brand, 'Unknown') as brand,
dr.period_start,
dr.period_end,
COUNT(DISTINCT p.pid) as num_products,
SUM(o.quantity) as total_sold,
SUM(o.quantity * o.price) as total_revenue,
COUNT(DISTINCT DATE(o.date)) as num_days
FROM products p
JOIN product_categories pc ON p.pid = pc.pid
JOIN orders o ON p.pid = o.pid
CROSS JOIN date_ranges dr
WHERE o.canceled = false
AND o.date BETWEEN dr.period_start AND dr.period_end
GROUP BY pc.cat_id, p.brand, dr.period_start, dr.period_end
)
SELECT
cat_id as category_id,
brand,
period_start,
period_end,
CASE
WHEN num_days > 0
THEN total_sold / num_days
ELSE 0
END as avg_daily_sales,
total_sold,
num_products,
CASE
WHEN total_sold > 0
THEN total_revenue / total_sold
ELSE 0
END as avg_price,
NOW() as last_calculated_at
FROM sales_data
ON DUPLICATE KEY UPDATE
avg_daily_sales = VALUES(avg_daily_sales),
total_sold = VALUES(total_sold),
num_products = VALUES(num_products),
avg_price = VALUES(avg_price),
last_calculated_at = VALUES(last_calculated_at)
`);
c.cat_id,
COUNT(DISTINCT p.pid) as product_count,
COUNT(DISTINCT CASE WHEN p.visible = true THEN p.pid END) as active_products,
SUM(p.stock_quantity * p.cost_price) as total_value,
AVG(pm.avg_margin_percent) as avg_margin,
AVG(pm.turnover_rate) as turnover_rate
FROM categories c
FORCE INDEX (PRIMARY)
JOIN product_categories pc FORCE INDEX (idx_category) ON c.cat_id = pc.cat_id
LEFT JOIN products p FORCE INDEX (PRIMARY) ON pc.pid = p.pid
LEFT JOIN product_metrics pm FORCE INDEX (PRIMARY) ON p.pid = pm.pid
WHERE c.cat_id IN (?)
GROUP BY c.cat_id
`, [batch.map(row => row.cat_id)]);
processedCount = Math.floor(totalProducts * 1.0);
outputProgress({
status: 'running',
operation: 'Category-sales metrics calculated',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
// Populate sales stats
await connection.query(`
INSERT INTO temp_sales_stats
SELECT
c.cat_id,
COALESCE(SUM(CASE
WHEN o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 30 DAY)
THEN o.quantity * o.price
ELSE 0
END), 0) as recent_revenue,
COALESCE(SUM(CASE
WHEN o.date BETWEEN DATE_SUB(CURRENT_DATE, INTERVAL 60 DAY) AND DATE_SUB(CURRENT_DATE, INTERVAL 30 DAY)
THEN o.quantity * o.price
ELSE 0
END), 0) as previous_revenue
FROM categories c
FORCE INDEX (PRIMARY)
JOIN product_categories pc FORCE INDEX (idx_category) ON c.cat_id = pc.cat_id
LEFT JOIN products p FORCE INDEX (PRIMARY) ON pc.pid = p.pid
LEFT JOIN orders o FORCE INDEX (idx_orders_metrics) ON p.pid = o.pid
AND o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 60 DAY)
WHERE c.cat_id IN (?)
GROUP BY c.cat_id
`, [batch.map(row => row.cat_id)]);
// Update metrics using temp tables
await connection.query(`
INSERT INTO category_metrics (
category_id,
product_count,
active_products,
total_value,
avg_margin,
turnover_rate,
growth_rate,
status,
last_calculated_at
)
SELECT
c.cat_id,
COALESCE(ps.product_count, 0) as product_count,
COALESCE(ps.active_products, 0) as active_products,
COALESCE(ps.total_value, 0) as total_value,
COALESCE(ps.avg_margin, 0) as avg_margin,
COALESCE(ps.turnover_rate, 0) as turnover_rate,
CASE
WHEN ss.previous_revenue = 0 AND ss.recent_revenue > 0 THEN 100
WHEN ss.previous_revenue = 0 THEN 0
ELSE LEAST(999.99, GREATEST(-100,
((ss.recent_revenue / NULLIF(ss.previous_revenue, 0) - 1) * 100)
))
END as growth_rate,
c.status,
NOW() as last_calculated_at
FROM categories c
FORCE INDEX (PRIMARY)
LEFT JOIN temp_product_stats ps ON c.cat_id = ps.cat_id
LEFT JOIN temp_sales_stats ss ON c.cat_id = ss.cat_id
WHERE c.cat_id IN (?)
ON DUPLICATE KEY UPDATE
product_count = VALUES(product_count),
active_products = VALUES(active_products),
total_value = VALUES(total_value),
avg_margin = VALUES(avg_margin),
turnover_rate = VALUES(turnover_rate),
growth_rate = VALUES(growth_rate),
status = VALUES(status),
last_calculated_at = NOW()
`, [batch.map(row => row.cat_id)]);
// Clean up temp tables
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_product_stats');
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_sales_stats');
lastCatId = batch[batch.length - 1].cat_id;
processedCount += batch.length;
outputProgress({
status: 'running',
operation: 'Processing category metrics batch',
current: processedCount,
total: totalCategories,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalCategories),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalCategories) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
}
// If we get here, everything completed successfully
success = true;
@@ -503,7 +259,7 @@ async function calculateCategoryMetrics(startTime, totalProducts, processedCount
return {
processedProducts: processedCount,
processedOrders,
processedOrders: 0,
processedPurchaseOrders: 0,
success
};

211
inventory-server/scripts/metrics/financial-metrics.js
View File

@@ -4,9 +4,39 @@ const { getConnection } = require('./utils/db');
async function calculateFinancialMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) {
const connection = await getConnection();
let success = false;
let processedOrders = 0;
const BATCH_SIZE = 5000;
try {
// Get last calculation timestamp
const [lastCalc] = await connection.query(`
SELECT last_calculation_timestamp
FROM calculate_status
WHERE module_name = 'financial_metrics'
`);
const lastCalculationTime = lastCalc[0]?.last_calculation_timestamp || '1970-01-01';
// Get total count of products needing updates
if (!totalProducts) {
const [productCount] = await connection.query(`
SELECT COUNT(DISTINCT p.pid) as count
FROM products p
LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
WHERE p.updated > ?
OR o.pid IS NOT NULL
`, [lastCalculationTime, lastCalculationTime]);
totalProducts = productCount[0].count;
}
if (totalProducts === 0) {
console.log('No products need financial metric updates');
return {
processedProducts: 0,
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
};
}
if (isCancelled) {
outputProgress({
status: 'cancelled',
@@ -31,15 +61,6 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun
};
}
// Get order count that will be processed
const [orderCount] = await connection.query(`
SELECT COUNT(*) as count
FROM orders o
WHERE o.canceled = false
AND DATE(o.date) >= DATE_SUB(CURDATE(), INTERVAL 12 MONTH)
`);
processedOrders = orderCount[0].count;
outputProgress({
status: 'running',
operation: 'Starting financial metrics calculation',
@@ -56,110 +77,80 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun
}
});
// Calculate financial metrics with optimized query
await connection.query(`
WITH product_financials AS (
SELECT
p.pid,
p.cost_price * p.stock_quantity as inventory_value,
SUM(o.quantity * o.price) as total_revenue,
SUM(o.quantity * p.cost_price) as cost_of_goods_sold,
SUM(o.quantity * (o.price - p.cost_price)) as gross_profit,
MIN(o.date) as first_sale_date,
MAX(o.date) as last_sale_date,
DATEDIFF(MAX(o.date), MIN(o.date)) + 1 as calculation_period_days,
COUNT(DISTINCT DATE(o.date)) as active_days
// Process in batches
let lastPid = 0;
while (true) {
if (isCancelled) break;
const [batch] = await connection.query(`
SELECT DISTINCT p.pid
FROM products p
LEFT JOIN orders o ON p.pid = o.pid
WHERE o.canceled = false
AND DATE(o.date) >= DATE_SUB(CURDATE(), INTERVAL 12 MONTH)
GROUP BY p.pid
)
UPDATE product_metrics pm
JOIN product_financials pf ON pm.pid = pf.pid
SET
pm.inventory_value = COALESCE(pf.inventory_value, 0),
pm.total_revenue = COALESCE(pf.total_revenue, 0),
pm.cost_of_goods_sold = COALESCE(pf.cost_of_goods_sold, 0),
pm.gross_profit = COALESCE(pf.gross_profit, 0),
pm.gmroi = CASE
WHEN COALESCE(pf.inventory_value, 0) > 0 AND pf.active_days > 0 THEN
(COALESCE(pf.gross_profit, 0) * (365.0 / pf.active_days)) / COALESCE(pf.inventory_value, 0)
ELSE 0
END,
pm.last_calculated_at = CURRENT_TIMESTAMP
`);
LEFT JOIN orders o ON p.pid = o.pid
WHERE p.pid > ?
AND (
p.updated > ?
OR EXISTS (
SELECT 1 FROM orders o2
WHERE o2.pid = p.pid
AND o2.updated > ?
)
)
ORDER BY p.pid
LIMIT ?
`, [lastPid, lastCalculationTime, lastCalculationTime, BATCH_SIZE]);
processedCount = Math.floor(totalProducts * 0.65);
outputProgress({
status: 'running',
operation: 'Base financial metrics calculated, updating time aggregates',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (batch.length === 0) break;
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
// Update financial metrics for this batch
await connection.query(`
UPDATE product_metrics pm
JOIN (
SELECT
p.pid,
p.cost_price * p.stock_quantity as inventory_value,
SUM(o.quantity * o.price) as total_revenue,
SUM(o.quantity * p.cost_price) as cost_of_goods_sold,
SUM(o.quantity * (o.price - p.cost_price)) as gross_profit,
COUNT(DISTINCT DATE(o.date)) as active_days
FROM products p
LEFT JOIN orders o ON p.pid = o.pid
AND o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 90 DAY)
WHERE p.pid IN (?)
GROUP BY p.pid
) fin ON pm.pid = fin.pid
SET
pm.inventory_value = COALESCE(fin.inventory_value, 0),
pm.total_revenue = COALESCE(fin.total_revenue, 0),
pm.cost_of_goods_sold = COALESCE(fin.cost_of_goods_sold, 0),
pm.gross_profit = COALESCE(fin.gross_profit, 0),
pm.gmroi = CASE
WHEN COALESCE(fin.inventory_value, 0) > 0 AND fin.active_days > 0
THEN (COALESCE(fin.gross_profit, 0) * (365.0 / fin.active_days)) / COALESCE(fin.inventory_value, 0)
ELSE 0
END,
pm.last_calculated_at = NOW()
`, [batch.map(row => row.pid)]);
// Update time-based aggregates with optimized query
await connection.query(`
WITH monthly_financials AS (
SELECT
p.pid,
YEAR(o.date) as year,
MONTH(o.date) as month,
p.cost_price * p.stock_quantity as inventory_value,
SUM(o.quantity * (o.price - p.cost_price)) as gross_profit,
COUNT(DISTINCT DATE(o.date)) as active_days,
MIN(o.date) as period_start,
MAX(o.date) as period_end
FROM products p
LEFT JOIN orders o ON p.pid = o.pid
WHERE o.canceled = false
GROUP BY p.pid, YEAR(o.date), MONTH(o.date)
)
UPDATE product_time_aggregates pta
JOIN monthly_financials mf ON pta.pid = mf.pid
AND pta.year = mf.year
AND pta.month = mf.month
SET
pta.inventory_value = COALESCE(mf.inventory_value, 0),
pta.gmroi = CASE
WHEN COALESCE(mf.inventory_value, 0) > 0 AND mf.active_days > 0 THEN
(COALESCE(mf.gross_profit, 0) * (365.0 / mf.active_days)) / COALESCE(mf.inventory_value, 0)
ELSE 0
END
`);
lastPid = batch[batch.length - 1].pid;
processedCount += batch.length;
processedCount = Math.floor(totalProducts * 0.70);
outputProgress({
status: 'running',
operation: 'Time-based aggregates updated',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
outputProgress({
status: 'running',
operation: 'Processing financial metrics batch',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
}
// If we get here, everything completed successfully
success = true;
@@ -173,7 +164,7 @@ async function calculateFinancialMetrics(startTime, totalProducts, processedCoun
return {
processedProducts: processedCount,
processedOrders,
processedOrders: 0,
processedPurchaseOrders: 0,
success
};

161
inventory-server/scripts/metrics/product-metrics.js
View File

@@ -16,16 +16,42 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
const BATCH_SIZE = 5000;
try {
// Skip flags are inherited from the parent scope
const SKIP_PRODUCT_BASE_METRICS = 0;
const SKIP_PRODUCT_TIME_AGGREGATES = 0;
// Get last calculation timestamp
const [lastCalc] = await connection.query(`
SELECT last_calculation_timestamp
FROM calculate_status
WHERE module_name = 'product_metrics'
`);
const lastCalculationTime = lastCalc[0]?.last_calculation_timestamp || '1970-01-01';
// Get total product count if not provided
if (!totalProducts) {
const [productCount] = await connection.query('SELECT COUNT(*) as count FROM products');
const [productCount] = await connection.query(`
SELECT COUNT(DISTINCT p.pid) as count
FROM products p
LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
LEFT JOIN purchase_orders po ON p.pid = po.pid AND po.updated > ?
WHERE p.updated > ?
OR o.pid IS NOT NULL
OR po.pid IS NOT NULL
`, [lastCalculationTime, lastCalculationTime, lastCalculationTime]);
totalProducts = productCount[0].count;
}
if (totalProducts === 0) {
console.log('No products need updating');
return {
processedProducts: 0,
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
};
}
// Skip flags are inherited from the parent scope
const SKIP_PRODUCT_BASE_METRICS = 0;
const SKIP_PRODUCT_TIME_AGGREGATES = 0;
if (isCancelled) {
outputProgress({
status: 'cancelled',
@@ -93,10 +119,39 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
processedOrders = orderCount[0].count;
// Clear temporary tables
await connection.query('TRUNCATE TABLE temp_sales_metrics');
await connection.query('TRUNCATE TABLE temp_purchase_metrics');
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_sales_metrics');
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_purchase_metrics');
// Populate temp_sales_metrics with base stats and sales averages
// Create optimized temporary tables with indexes
await connection.query(`
CREATE TEMPORARY TABLE temp_sales_metrics (
pid BIGINT NOT NULL,
daily_sales_avg DECIMAL(10,3),
weekly_sales_avg DECIMAL(10,3),
monthly_sales_avg DECIMAL(10,3),
total_revenue DECIMAL(10,2),
avg_margin_percent DECIMAL(5,2),
first_sale_date DATE,
last_sale_date DATE,
PRIMARY KEY (pid),
INDEX (daily_sales_avg),
INDEX (total_revenue)
) ENGINE=MEMORY
`);
await connection.query(`
CREATE TEMPORARY TABLE temp_purchase_metrics (
pid BIGINT NOT NULL,
avg_lead_time_days DECIMAL(5,1),
last_purchase_date DATE,
first_received_date DATE,
last_received_date DATE,
PRIMARY KEY (pid),
INDEX (avg_lead_time_days)
) ENGINE=MEMORY
`);
// Populate temp_sales_metrics with base stats and sales averages using FORCE INDEX
await connection.query(`
INSERT INTO temp_sales_metrics
SELECT
@@ -113,13 +168,21 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
MIN(o.date) as first_sale_date,
MAX(o.date) as last_sale_date
FROM products p
LEFT JOIN orders o ON p.pid = o.pid
AND o.canceled = false
AND o.date >= DATE_SUB(CURDATE(), INTERVAL 90 DAY)
FORCE INDEX (PRIMARY)
LEFT JOIN orders o FORCE INDEX (idx_orders_metrics) ON p.pid = o.pid
AND o.canceled = false
AND o.date >= DATE_SUB(CURDATE(), INTERVAL 90 DAY)
WHERE p.updated > ?
OR EXISTS (
SELECT 1 FROM orders o2 FORCE INDEX (idx_orders_metrics)
WHERE o2.pid = p.pid
AND o2.canceled = false
AND o2.updated > ?
)
GROUP BY p.pid
`);
`, [lastCalculationTime, lastCalculationTime]);
// Populate temp_purchase_metrics
// Populate temp_purchase_metrics with optimized index usage
await connection.query(`
INSERT INTO temp_purchase_metrics
SELECT
@@ -129,21 +192,38 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
MIN(po.received_date) as first_received_date,
MAX(po.received_date) as last_received_date
FROM products p
LEFT JOIN purchase_orders po ON p.pid = po.pid
AND po.received_date IS NOT NULL
AND po.date >= DATE_SUB(CURDATE(), INTERVAL 365 DAY)
FORCE INDEX (PRIMARY)
LEFT JOIN purchase_orders po FORCE INDEX (idx_po_metrics) ON p.pid = po.pid
AND po.received_date IS NOT NULL
AND po.date >= DATE_SUB(CURDATE(), INTERVAL 365 DAY)
WHERE p.updated > ?
OR EXISTS (
SELECT 1 FROM purchase_orders po2 FORCE INDEX (idx_po_metrics)
WHERE po2.pid = p.pid
AND po2.updated > ?
)
GROUP BY p.pid
`);
`, [lastCalculationTime, lastCalculationTime]);
// Process updates in batches
// Process updates in batches, but only for affected products
let lastPid = 0;
while (true) {
if (isCancelled) break;
const [batch] = await connection.query(
'SELECT pid FROM products WHERE pid > ? ORDER BY pid LIMIT ?',
[lastPid, BATCH_SIZE]
);
const [batch] = await connection.query(`
SELECT DISTINCT p.pid
FROM products p
LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
LEFT JOIN purchase_orders po ON p.pid = po.pid AND po.updated > ?
WHERE p.pid > ?
AND (
p.updated > ?
OR o.pid IS NOT NULL
OR po.pid IS NOT NULL
)
ORDER BY p.pid
LIMIT ?
`, [lastCalculationTime, lastCalculationTime, lastPid, lastCalculationTime, BATCH_SIZE]);
if (batch.length === 0) break;
@@ -153,7 +233,7 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
LEFT JOIN temp_sales_metrics sm ON pm.pid = sm.pid
LEFT JOIN temp_purchase_metrics lm ON pm.pid = lm.pid
SET
pm.inventory_value = p.stock_quantity * NULLIF(p.cost_price, 0),
pm.inventory_value = p.stock_quantity * p.cost_price,
pm.daily_sales_avg = COALESCE(sm.daily_sales_avg, 0),
pm.weekly_sales_avg = COALESCE(sm.weekly_sales_avg, 0),
pm.monthly_sales_avg = COALESCE(sm.monthly_sales_avg, 0),
@@ -164,12 +244,12 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
pm.avg_lead_time_days = COALESCE(lm.avg_lead_time_days, 30),
pm.days_of_inventory = CASE
WHEN COALESCE(sm.daily_sales_avg, 0) > 0
THEN FLOOR(p.stock_quantity / NULLIF(sm.daily_sales_avg, 0))
THEN FLOOR(p.stock_quantity / sm.daily_sales_avg)
ELSE NULL
END,
pm.weeks_of_inventory = CASE
WHEN COALESCE(sm.weekly_sales_avg, 0) > 0
THEN FLOOR(p.stock_quantity / NULLIF(sm.weekly_sales_avg, 0))
THEN FLOOR(p.stock_quantity / sm.weekly_sales_avg)
ELSE NULL
END,
pm.stock_status = CASE
@@ -181,21 +261,10 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
WHEN p.stock_quantity / NULLIF(sm.daily_sales_avg, 0) > ? THEN 'Overstocked'
ELSE 'Healthy'
END,
pm.safety_stock = CASE
WHEN COALESCE(sm.daily_sales_avg, 0) > 0 THEN
CEIL(sm.daily_sales_avg * SQRT(COALESCE(lm.avg_lead_time_days, 30)) * 1.96)
ELSE ?
END,
pm.reorder_point = CASE
WHEN COALESCE(sm.daily_sales_avg, 0) > 0 THEN
CEIL(sm.daily_sales_avg * COALESCE(lm.avg_lead_time_days, 30)) +
CEIL(sm.daily_sales_avg * SQRT(COALESCE(lm.avg_lead_time_days, 30)) * 1.96)
ELSE ?
END,
pm.reorder_qty = CASE
WHEN COALESCE(sm.daily_sales_avg, 0) > 0 AND NULLIF(p.cost_price, 0) IS NOT NULL THEN
WHEN COALESCE(sm.daily_sales_avg, 0) > 0 THEN
GREATEST(
CEIL(SQRT((2 * (sm.daily_sales_avg * 365) * 25) / (NULLIF(p.cost_price, 0) * 0.25))),
CEIL(sm.daily_sales_avg * COALESCE(lm.avg_lead_time_days, 30) * 1.96),
?
)
ELSE ?
@@ -206,22 +275,18 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
ELSE 0
END,
pm.last_calculated_at = NOW()
WHERE p.pid IN (${batch.map(() => '?').join(',')})
`,
[
WHERE p.pid IN (?)
`, [
defaultThresholds.low_stock_threshold,
defaultThresholds.critical_days,
defaultThresholds.reorder_days,
defaultThresholds.overstock_days,
defaultThresholds.low_stock_threshold,
defaultThresholds.low_stock_threshold,
defaultThresholds.low_stock_threshold,
defaultThresholds.low_stock_threshold,
defaultThresholds.overstock_days,
defaultThresholds.overstock_days,
...batch.map(row => row.pid)
]
);
batch.map(row => row.pid)
]);
lastPid = batch[batch.length - 1].pid;
processedCount += batch.length;
@@ -547,7 +612,7 @@ async function calculateProductMetrics(startTime, totalProducts, processedCount
// If we get here, everything completed successfully
success = true;
// Update calculate_status
// Update calculate_status with current timestamp
await connection.query(`
INSERT INTO calculate_status (module_name, last_calculation_timestamp)
VALUES ('product_metrics', NOW())
@@ -618,9 +683,9 @@ function calculateReorderQuantities(stock, stock_status, daily_sales_avg, avg_le
if (daily_sales_avg > 0) {
const annual_demand = daily_sales_avg * 365;
const order_cost = 25; // Fixed cost per order
const holding_cost = config.cost_price * 0.25; // 25% of unit cost as annual holding cost
const holding_cost_percent = 0.25; // 25% annual holding cost
reorder_qty = Math.ceil(Math.sqrt((2 * annual_demand * order_cost) / holding_cost));
reorder_qty = Math.ceil(Math.sqrt((2 * annual_demand * order_cost) / holding_cost_percent));
} else {
// If no sales data, use a basic calculation
reorder_qty = Math.max(safety_stock, config.low_stock_threshold);

580
inventory-server/scripts/metrics/sales-forecasts.js
View File

@@ -4,19 +4,50 @@ const { getConnection } = require('./utils/db');
async function calculateSalesForecasts(startTime, totalProducts, processedCount = 0, isCancelled = false) {
const connection = await getConnection();
let success = false;
let processedOrders = 0;
const BATCH_SIZE = 5000;
try {
// Get last calculation timestamp
const [lastCalc] = await connection.query(`
SELECT last_calculation_timestamp
FROM calculate_status
WHERE module_name = 'sales_forecasts'
`);
const lastCalculationTime = lastCalc[0]?.last_calculation_timestamp || '1970-01-01';
// Get total count of products needing updates
const [productCount] = await connection.query(`
SELECT COUNT(DISTINCT p.pid) as count
FROM products p
LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
WHERE p.visible = true
AND (
p.updated > ?
OR o.id IS NOT NULL
)
`, [lastCalculationTime, lastCalculationTime]);
const totalProductsToUpdate = productCount[0].count;
if (totalProductsToUpdate === 0) {
console.log('No products need forecast updates');
return {
processedProducts: 0,
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
};
}
if (isCancelled) {
outputProgress({
status: 'cancelled',
operation: 'Sales forecasts calculation cancelled',
operation: 'Sales forecast calculation cancelled',
current: processedCount,
total: totalProducts,
total: totalProductsToUpdate,
elapsed: formatElapsedTime(startTime),
remaining: null,
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
percentage: ((processedCount / totalProductsToUpdate) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -31,24 +62,15 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount
};
}
// Get order count that will be processed
const [orderCount] = await connection.query(`
SELECT COUNT(*) as count
FROM orders o
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 90 DAY)
`);
processedOrders = orderCount[0].count;
outputProgress({
status: 'running',
operation: 'Starting sales forecasts calculation',
operation: 'Starting sales forecast calculation',
current: processedCount,
total: totalProducts,
total: totalProductsToUpdate,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
remaining: estimateRemaining(startTime, processedCount, totalProductsToUpdate),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
percentage: ((processedCount / totalProductsToUpdate) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -56,365 +78,201 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount
}
});
// First, create a temporary table for forecast dates
await connection.query(`
CREATE TEMPORARY TABLE IF NOT EXISTS temp_forecast_dates (
forecast_date DATE,
day_of_week INT,
month INT,
PRIMARY KEY (forecast_date)
)
`);
// Process in batches
let lastPid = '';
while (true) {
if (isCancelled) break;
await connection.query(`
INSERT INTO temp_forecast_dates
SELECT
DATE_ADD(CURRENT_DATE, INTERVAL n DAY) as forecast_date,
DAYOFWEEK(DATE_ADD(CURRENT_DATE, INTERVAL n DAY)) as day_of_week,
MONTH(DATE_ADD(CURRENT_DATE, INTERVAL n DAY)) as month
FROM (
SELECT a.N + b.N * 10 as n
FROM
(SELECT 0 as N UNION SELECT 1 UNION SELECT 2 UNION SELECT 3 UNION SELECT 4 UNION
SELECT 5 UNION SELECT 6 UNION SELECT 7 UNION SELECT 8 UNION SELECT 9) a,
(SELECT 0 as N UNION SELECT 1 UNION SELECT 2) b
ORDER BY n
LIMIT 31
) numbers
`);
const [batch] = await connection.query(`
SELECT DISTINCT p.pid
FROM products p
FORCE INDEX (PRIMARY)
LEFT JOIN orders o FORCE INDEX (idx_orders_metrics) ON p.pid = o.pid AND o.updated > ?
WHERE p.visible = true
AND p.pid > ?
AND (
p.updated > ?
OR o.id IS NOT NULL
)
ORDER BY p.pid
LIMIT ?
`, [lastCalculationTime, lastPid, lastCalculationTime, BATCH_SIZE]);
processedCount = Math.floor(totalProducts * 0.92);
outputProgress({
status: 'running',
operation: 'Forecast dates prepared, calculating daily sales stats',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (batch.length === 0) break;
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
// Create temporary tables for better performance
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_historical_sales');
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_sales_stats');
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_recent_trend');
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_confidence_calc');
// Create temporary table for daily sales stats
await connection.query(`
CREATE TEMPORARY TABLE IF NOT EXISTS temp_daily_sales AS
SELECT
o.pid,
DAYOFWEEK(o.date) as day_of_week,
SUM(o.quantity) as daily_quantity,
SUM(o.price * o.quantity) as daily_revenue,
COUNT(DISTINCT DATE(o.date)) as day_count
FROM orders o
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 90 DAY)
GROUP BY o.pid, DAYOFWEEK(o.date)
`);
// Create optimized temporary tables with indexes
await connection.query(`
CREATE TEMPORARY TABLE temp_historical_sales (
pid BIGINT NOT NULL,
sale_date DATE NOT NULL,
daily_quantity INT,
daily_revenue DECIMAL(15,2),
PRIMARY KEY (pid, sale_date),
INDEX (sale_date)
) ENGINE=MEMORY
`);
processedCount = Math.floor(totalProducts * 0.94);
outputProgress({
status: 'running',
operation: 'Daily sales stats calculated, preparing product stats',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
await connection.query(`
CREATE TEMPORARY TABLE temp_sales_stats (
pid BIGINT NOT NULL,
avg_daily_units DECIMAL(10,2),
avg_daily_revenue DECIMAL(15,2),
std_daily_units DECIMAL(10,2),
days_with_sales INT,
first_sale DATE,
last_sale DATE,
PRIMARY KEY (pid),
INDEX (days_with_sales),
INDEX (last_sale)
) ENGINE=MEMORY
`);
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
await connection.query(`
CREATE TEMPORARY TABLE temp_recent_trend (
pid BIGINT NOT NULL,
recent_avg_units DECIMAL(10,2),
recent_avg_revenue DECIMAL(15,2),
PRIMARY KEY (pid)
) ENGINE=MEMORY
`);
// Create temporary table for product stats
await connection.query(`
CREATE TEMPORARY TABLE IF NOT EXISTS temp_product_stats AS
SELECT
pid,
AVG(daily_revenue) as overall_avg_revenue,
SUM(day_count) as total_days
FROM temp_daily_sales
GROUP BY pid
`);
await connection.query(`
CREATE TEMPORARY TABLE temp_confidence_calc (
pid BIGINT NOT NULL,
confidence_level TINYINT,
PRIMARY KEY (pid)
) ENGINE=MEMORY
`);
processedCount = Math.floor(totalProducts * 0.96);
outputProgress({
status: 'running',
operation: 'Product stats prepared, calculating product-level forecasts',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
// Calculate product-level forecasts
await connection.query(`
INSERT INTO sales_forecasts (
pid,
forecast_date,
forecast_units,
forecast_revenue,
confidence_level,
last_calculated_at
)
WITH daily_stats AS (
// Populate historical sales with optimized index usage
await connection.query(`
INSERT INTO temp_historical_sales
SELECT
ds.pid,
AVG(ds.daily_quantity) as avg_daily_qty,
STDDEV(ds.daily_quantity) as std_daily_qty,
COUNT(DISTINCT ds.day_count) as data_points,
SUM(ds.day_count) as total_days,
AVG(ds.daily_revenue) as avg_daily_revenue,
STDDEV(ds.daily_revenue) as std_daily_revenue,
MIN(ds.daily_quantity) as min_daily_qty,
MAX(ds.daily_quantity) as max_daily_qty,
-- Calculate variance without using LAG
COALESCE(
STDDEV(ds.daily_quantity) / NULLIF(AVG(ds.daily_quantity), 0),
0
) as daily_variance_ratio
FROM temp_daily_sales ds
GROUP BY ds.pid
HAVING AVG(ds.daily_quantity) > 0
)
SELECT
ds.pid,
fd.forecast_date,
GREATEST(0,
ROUND(
ds.avg_daily_qty *
(1 + COALESCE(sf.seasonality_factor, 0)) *
CASE
WHEN ds.std_daily_qty / NULLIF(ds.avg_daily_qty, 0) > 1.5 THEN 0.85
WHEN ds.std_daily_qty / NULLIF(ds.avg_daily_qty, 0) > 1.0 THEN 0.9
WHEN ds.std_daily_qty / NULLIF(ds.avg_daily_qty, 0) > 0.5 THEN 0.95
ELSE 1.0
END,
2
)
) as forecast_units,
GREATEST(0,
ROUND(
COALESCE(
CASE
WHEN ds.data_points >= 4 THEN ds.avg_daily_revenue
ELSE ps.overall_avg_revenue
END *
(1 + COALESCE(sf.seasonality_factor, 0)) *
CASE
WHEN ds.std_daily_revenue / NULLIF(ds.avg_daily_revenue, 0) > 1.5 THEN 0.85
WHEN ds.std_daily_revenue / NULLIF(ds.avg_daily_revenue, 0) > 1.0 THEN 0.9
WHEN ds.std_daily_revenue / NULLIF(ds.avg_daily_revenue, 0) > 0.5 THEN 0.95
ELSE 1.0
END,
0
),
2
)
) as forecast_revenue,
CASE
WHEN ds.total_days >= 60 AND ds.daily_variance_ratio < 0.5 THEN 90
WHEN ds.total_days >= 60 THEN 85
WHEN ds.total_days >= 30 AND ds.daily_variance_ratio < 0.5 THEN 80
WHEN ds.total_days >= 30 THEN 75
WHEN ds.total_days >= 14 AND ds.daily_variance_ratio < 0.5 THEN 70
WHEN ds.total_days >= 14 THEN 65
ELSE 60
END as confidence_level,
NOW() as last_calculated_at
FROM daily_stats ds
JOIN temp_product_stats ps ON ds.pid = ps.pid
CROSS JOIN temp_forecast_dates fd
LEFT JOIN sales_seasonality sf ON fd.month = sf.month
GROUP BY ds.pid, fd.forecast_date, ps.overall_avg_revenue, sf.seasonality_factor
ON DUPLICATE KEY UPDATE
forecast_units = VALUES(forecast_units),
forecast_revenue = VALUES(forecast_revenue),
confidence_level = VALUES(confidence_level),
last_calculated_at = NOW()
`);
o.pid,
DATE(o.date) as sale_date,
SUM(o.quantity) as daily_quantity,
SUM(o.quantity * o.price) as daily_revenue
FROM orders o
FORCE INDEX (idx_orders_metrics)
WHERE o.canceled = false
AND o.pid IN (?)
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 180 DAY)
GROUP BY o.pid, DATE(o.date)
`, [batch.map(row => row.pid)]);
processedCount = Math.floor(totalProducts * 0.98);
outputProgress({
status: 'running',
operation: 'Product forecasts calculated, preparing category stats',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
// Populate sales stats
await connection.query(`
INSERT INTO temp_sales_stats
SELECT
pid,
AVG(daily_quantity) as avg_daily_units,
AVG(daily_revenue) as avg_daily_revenue,
STDDEV(daily_quantity) as std_daily_units,
COUNT(*) as days_with_sales,
MIN(sale_date) as first_sale,
MAX(sale_date) as last_sale
FROM temp_historical_sales
GROUP BY pid
`);
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
// Populate recent trend
await connection.query(`
INSERT INTO temp_recent_trend
SELECT
h.pid,
AVG(h.daily_quantity) as recent_avg_units,
AVG(h.daily_revenue) as recent_avg_revenue
FROM temp_historical_sales h
WHERE h.sale_date >= DATE_SUB(CURRENT_DATE, INTERVAL 30 DAY)
GROUP BY h.pid
`);
// Create temporary table for category stats
await connection.query(`
CREATE TEMPORARY TABLE IF NOT EXISTS temp_category_sales AS
SELECT
pc.cat_id,
DAYOFWEEK(o.date) as day_of_week,
SUM(o.quantity) as daily_quantity,
SUM(o.price * o.quantity) as daily_revenue,
COUNT(DISTINCT DATE(o.date)) as day_count
FROM orders o
JOIN product_categories pc ON o.pid = pc.pid
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 90 DAY)
GROUP BY pc.cat_id, DAYOFWEEK(o.date)
`);
// Calculate confidence levels
await connection.query(`
INSERT INTO temp_confidence_calc
SELECT
s.pid,
LEAST(100, GREATEST(0, ROUND(
(s.days_with_sales / 180.0 * 50) + -- Up to 50 points for history length
(CASE
WHEN s.std_daily_units = 0 OR s.avg_daily_units = 0 THEN 0
WHEN (s.std_daily_units / s.avg_daily_units) <= 0.5 THEN 30
WHEN (s.std_daily_units / s.avg_daily_units) <= 1.0 THEN 20
WHEN (s.std_daily_units / s.avg_daily_units) <= 2.0 THEN 10
ELSE 0
END) + -- Up to 30 points for consistency
(CASE
WHEN DATEDIFF(CURRENT_DATE, s.last_sale) <= 7 THEN 20
WHEN DATEDIFF(CURRENT_DATE, s.last_sale) <= 30 THEN 10
ELSE 0
END) -- Up to 20 points for recency
))) as confidence_level
FROM temp_sales_stats s
`);
await connection.query(`
CREATE TEMPORARY TABLE IF NOT EXISTS temp_category_stats AS
SELECT
cat_id,
AVG(daily_revenue) as overall_avg_revenue,
SUM(day_count) as total_days
FROM temp_category_sales
GROUP BY cat_id
`);
processedCount = Math.floor(totalProducts * 0.99);
outputProgress({
status: 'running',
operation: 'Category stats prepared, calculating category-level forecasts',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
// Calculate category-level forecasts
await connection.query(`
INSERT INTO category_forecasts (
category_id,
forecast_date,
forecast_units,
forecast_revenue,
confidence_level,
last_calculated_at
)
SELECT
cs.cat_id as category_id,
fd.forecast_date,
GREATEST(0,
AVG(cs.daily_quantity) *
(1 + COALESCE(sf.seasonality_factor, 0))
) as forecast_units,
GREATEST(0,
COALESCE(
// Generate forecasts using temp tables
await connection.query(`
REPLACE INTO sales_forecasts
(pid, forecast_date, forecast_units, forecast_revenue, confidence_level, last_calculated_at)
SELECT
s.pid,
DATE_ADD(CURRENT_DATE, INTERVAL n.days DAY),
GREATEST(0, ROUND(
CASE
WHEN SUM(cs.day_count) >= 4 THEN AVG(cs.daily_revenue)
ELSE ct.overall_avg_revenue
END *
(1 + COALESCE(sf.seasonality_factor, 0)) *
(0.95 + (RAND() * 0.1)),
0
)
) as forecast_revenue,
CASE
WHEN ct.total_days >= 60 THEN 90
WHEN ct.total_days >= 30 THEN 80
WHEN ct.total_days >= 14 THEN 70
ELSE 60
END as confidence_level,
NOW() as last_calculated_at
FROM temp_category_sales cs
JOIN temp_category_stats ct ON cs.cat_id = ct.cat_id
CROSS JOIN temp_forecast_dates fd
LEFT JOIN sales_seasonality sf ON fd.month = sf.month
GROUP BY cs.cat_id, fd.forecast_date, ct.overall_avg_revenue, ct.total_days, sf.seasonality_factor
HAVING AVG(cs.daily_quantity) > 0
ON DUPLICATE KEY UPDATE
forecast_units = VALUES(forecast_units),
forecast_revenue = VALUES(forecast_revenue),
confidence_level = VALUES(confidence_level),
last_calculated_at = NOW()
`);
WHEN s.days_with_sales >= n.days THEN COALESCE(t.recent_avg_units, s.avg_daily_units)
ELSE s.avg_daily_units * (s.days_with_sales / n.days)
END
)),
GREATEST(0, ROUND(
CASE
WHEN s.days_with_sales >= n.days THEN COALESCE(t.recent_avg_revenue, s.avg_daily_revenue)
ELSE s.avg_daily_revenue * (s.days_with_sales / n.days)
END,
2
)),
c.confidence_level,
NOW()
FROM temp_sales_stats s
CROSS JOIN (
SELECT 30 as days
UNION SELECT 60
UNION SELECT 90
) n
LEFT JOIN temp_recent_trend t ON s.pid = t.pid
LEFT JOIN temp_confidence_calc c ON s.pid = c.pid;
`);
// Clean up temporary tables
await connection.query(`
DROP TEMPORARY TABLE IF EXISTS temp_forecast_dates;
DROP TEMPORARY TABLE IF EXISTS temp_daily_sales;
DROP TEMPORARY TABLE IF EXISTS temp_product_stats;
DROP TEMPORARY TABLE IF EXISTS temp_category_sales;
DROP TEMPORARY TABLE IF EXISTS temp_category_stats;
`);
// Clean up temp tables
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_historical_sales');
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_sales_stats');
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_recent_trend');
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_confidence_calc');
processedCount = Math.floor(totalProducts * 1.0);
outputProgress({
status: 'running',
operation: 'Category forecasts calculated and temporary tables cleaned up',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
lastPid = batch[batch.length - 1].pid;
processedCount += batch.length;
outputProgress({
status: 'running',
operation: 'Processing sales forecast batch',
current: processedCount,
total: totalProductsToUpdate,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProductsToUpdate),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProductsToUpdate) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
}
// If we get here, everything completed successfully
success = true;
@@ -428,7 +286,7 @@ async function calculateSalesForecasts(startTime, totalProducts, processedCount
return {
processedProducts: processedCount,
processedOrders,
processedOrders: 0,
processedPurchaseOrders: 0,
success
};

369
inventory-server/scripts/metrics/time-aggregates.js
View File

@@ -4,9 +4,39 @@ const { getConnection } = require('./utils/db');
async function calculateTimeAggregates(startTime, totalProducts, processedCount = 0, isCancelled = false) {
const connection = await getConnection();
let success = false;
let processedOrders = 0;
const BATCH_SIZE = 5000;
try {
// Get last calculation timestamp
const [lastCalc] = await connection.query(`
SELECT last_calculation_timestamp
FROM calculate_status
WHERE module_name = 'time_aggregates'
`);
const lastCalculationTime = lastCalc[0]?.last_calculation_timestamp || '1970-01-01';
// Get total count of products needing updates
if (!totalProducts) {
const [productCount] = await connection.query(`
SELECT COUNT(DISTINCT p.pid) as count
FROM products p
LEFT JOIN orders o ON p.pid = o.pid AND o.updated > ?
WHERE p.updated > ?
OR o.pid IS NOT NULL
`, [lastCalculationTime, lastCalculationTime]);
totalProducts = productCount[0].count;
}
if (totalProducts === 0) {
console.log('No products need time aggregate updates');
return {
processedProducts: 0,
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
};
}
if (isCancelled) {
outputProgress({
status: 'cancelled',
@@ -31,14 +61,6 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount
};
}
// Get order count that will be processed
const [orderCount] = await connection.query(`
SELECT COUNT(*) as count
FROM orders o
WHERE o.canceled = false
`);
processedOrders = orderCount[0].count;
outputProgress({
status: 'running',
operation: 'Starting time aggregates calculation',
@@ -55,223 +77,140 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount
}
});
// Initial insert of time-based aggregates
await connection.query(`
INSERT INTO product_time_aggregates (
pid,
year,
month,
total_quantity_sold,
total_revenue,
total_cost,
order_count,
stock_received,
stock_ordered,
avg_price,
profit_margin,
inventory_value,
gmroi
)
WITH monthly_sales AS (
// Process in batches
let lastPid = 0;
while (true) {
if (isCancelled) break;
const [batch] = await connection.query(`
SELECT DISTINCT p.pid
FROM products p
FORCE INDEX (PRIMARY)
LEFT JOIN orders o FORCE INDEX (idx_orders_metrics) ON p.pid = o.pid
WHERE p.pid > ?
AND (
p.updated > ?
OR EXISTS (
SELECT 1
FROM orders o2 FORCE INDEX (idx_orders_metrics)
WHERE o2.pid = p.pid
AND o2.updated > ?
)
)
ORDER BY p.pid
LIMIT ?
`, [lastPid, lastCalculationTime, lastCalculationTime, BATCH_SIZE]);
if (batch.length === 0) break;
// Calculate and update time aggregates for this batch using temporary table
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_time_aggregates');
await connection.query(`
CREATE TEMPORARY TABLE temp_time_aggregates (
pid BIGINT NOT NULL,
year INT NOT NULL,
month INT NOT NULL,
total_quantity_sold INT DEFAULT 0,
total_revenue DECIMAL(10,3) DEFAULT 0,
total_cost DECIMAL(10,3) DEFAULT 0,
order_count INT DEFAULT 0,
stock_received INT DEFAULT 0,
stock_ordered INT DEFAULT 0,
avg_price DECIMAL(10,3),
profit_margin DECIMAL(10,3),
inventory_value DECIMAL(10,3),
gmroi DECIMAL(10,3),
PRIMARY KEY (pid, year, month),
INDEX (pid),
INDEX (year, month)
) ENGINE=MEMORY
`);
// Populate temporary table
await connection.query(`
INSERT INTO temp_time_aggregates
SELECT
o.pid,
p.pid,
YEAR(o.date) as year,
MONTH(o.date) as month,
SUM(o.quantity) as total_quantity_sold,
SUM((o.price - COALESCE(o.discount, 0)) * o.quantity) as total_revenue,
SUM(COALESCE(p.cost_price, 0) * o.quantity) as total_cost,
SUM(o.quantity * o.price) as total_revenue,
SUM(o.quantity * p.cost_price) as total_cost,
COUNT(DISTINCT o.order_number) as order_count,
AVG(o.price - COALESCE(o.discount, 0)) as avg_price,
COALESCE(SUM(CASE WHEN po.received_date IS NOT NULL THEN po.received ELSE 0 END), 0) as stock_received,
COALESCE(SUM(po.ordered), 0) as stock_ordered,
AVG(o.price) as avg_price,
CASE
WHEN SUM((o.price - COALESCE(o.discount, 0)) * o.quantity) > 0
THEN ((SUM((o.price - COALESCE(o.discount, 0)) * o.quantity) - SUM(COALESCE(p.cost_price, 0) * o.quantity))
/ SUM((o.price - COALESCE(o.discount, 0)) * o.quantity)) * 100
WHEN SUM(o.quantity * o.price) > 0
THEN ((SUM(o.quantity * o.price) - SUM(o.quantity * p.cost_price)) / SUM(o.quantity * o.price)) * 100
ELSE 0
END as profit_margin,
p.cost_price * p.stock_quantity as inventory_value,
COUNT(DISTINCT DATE(o.date)) as active_days
FROM orders o
JOIN products p ON o.pid = p.pid
WHERE o.canceled = false
GROUP BY o.pid, YEAR(o.date), MONTH(o.date)
),
monthly_stock AS (
SELECT
pid,
YEAR(date) as year,
MONTH(date) as month,
SUM(received) as stock_received,
SUM(ordered) as stock_ordered
FROM purchase_orders
GROUP BY pid, YEAR(date), MONTH(date)
),
base_products AS (
SELECT
p.pid,
p.cost_price * p.stock_quantity as inventory_value
CASE
WHEN p.cost_price * p.stock_quantity > 0
THEN (SUM(o.quantity * (o.price - p.cost_price))) / (p.cost_price * p.stock_quantity)
ELSE 0
END as gmroi
FROM products p
)
SELECT
COALESCE(s.pid, ms.pid) as pid,
COALESCE(s.year, ms.year) as year,
COALESCE(s.month, ms.month) as month,
COALESCE(s.total_quantity_sold, 0) as total_quantity_sold,
COALESCE(s.total_revenue, 0) as total_revenue,
COALESCE(s.total_cost, 0) as total_cost,
COALESCE(s.order_count, 0) as order_count,
COALESCE(ms.stock_received, 0) as stock_received,
COALESCE(ms.stock_ordered, 0) as stock_ordered,
COALESCE(s.avg_price, 0) as avg_price,
COALESCE(s.profit_margin, 0) as profit_margin,
COALESCE(s.inventory_value, bp.inventory_value, 0) as inventory_value,
CASE
WHEN COALESCE(s.inventory_value, bp.inventory_value, 0) > 0
AND COALESCE(s.active_days, 0) > 0
THEN (COALESCE(s.total_revenue - s.total_cost, 0) * (365.0 / s.active_days))
/ COALESCE(s.inventory_value, bp.inventory_value)
ELSE 0
END as gmroi
FROM (
SELECT * FROM monthly_sales s
UNION ALL
SELECT
ms.pid,
ms.year,
ms.month,
0 as total_quantity_sold,
0 as total_revenue,
0 as total_cost,
0 as order_count,
NULL as avg_price,
0 as profit_margin,
NULL as inventory_value,
0 as active_days
FROM monthly_stock ms
WHERE NOT EXISTS (
SELECT 1 FROM monthly_sales s2
WHERE s2.pid = ms.pid
AND s2.year = ms.year
AND s2.month = ms.month
)
) s
LEFT JOIN monthly_stock ms
ON s.pid = ms.pid
AND s.year = ms.year
AND s.month = ms.month
JOIN base_products bp ON COALESCE(s.pid, ms.pid) = bp.pid
UNION
SELECT
ms.pid,
ms.year,
ms.month,
0 as total_quantity_sold,
0 as total_revenue,
0 as total_cost,
0 as order_count,
ms.stock_received,
ms.stock_ordered,
0 as avg_price,
0 as profit_margin,
bp.inventory_value,
0 as gmroi
FROM monthly_stock ms
JOIN base_products bp ON ms.pid = bp.pid
WHERE NOT EXISTS (
SELECT 1 FROM (
SELECT * FROM monthly_sales
UNION ALL
SELECT
ms2.pid,
ms2.year,
ms2.month,
0, 0, 0, 0, NULL, 0, NULL, 0
FROM monthly_stock ms2
WHERE NOT EXISTS (
SELECT 1 FROM monthly_sales s2
WHERE s2.pid = ms2.pid
AND s2.year = ms2.year
AND s2.month = ms2.month
)
) s
WHERE s.pid = ms.pid
AND s.year = ms.year
AND s.month = ms.month
)
ON DUPLICATE KEY UPDATE
total_quantity_sold = VALUES(total_quantity_sold),
total_revenue = VALUES(total_revenue),
total_cost = VALUES(total_cost),
order_count = VALUES(order_count),
stock_received = VALUES(stock_received),
stock_ordered = VALUES(stock_ordered),
avg_price = VALUES(avg_price),
profit_margin = VALUES(profit_margin),
inventory_value = VALUES(inventory_value),
gmroi = VALUES(gmroi)
`);
processedCount = Math.floor(totalProducts * 0.60);
outputProgress({
status: 'running',
operation: 'Base time aggregates calculated, updating financial metrics',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders: 0,
success
};
// Update with financial metrics
await connection.query(`
UPDATE product_time_aggregates pta
JOIN (
SELECT
p.pid,
YEAR(o.date) as year,
MONTH(o.date) as month,
p.cost_price * p.stock_quantity as inventory_value,
SUM(o.quantity * (o.price - p.cost_price)) as gross_profit,
COUNT(DISTINCT DATE(o.date)) as active_days
FROM products p
LEFT JOIN orders o ON p.pid = o.pid
WHERE o.canceled = false
FORCE INDEX (PRIMARY)
INNER JOIN orders o FORCE INDEX (idx_orders_metrics) ON p.pid = o.pid
AND o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
LEFT JOIN purchase_orders po FORCE INDEX (idx_po_metrics) ON p.pid = po.pid
AND po.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
WHERE p.pid IN (?)
GROUP BY p.pid, YEAR(o.date), MONTH(o.date)
) fin ON pta.pid = fin.pid
AND pta.year = fin.year
AND pta.month = fin.month
SET
pta.inventory_value = COALESCE(fin.inventory_value, 0)
`);
HAVING year IS NOT NULL AND month IS NOT NULL
`, [batch.map(row => row.pid)]);
processedCount = Math.floor(totalProducts * 0.65);
outputProgress({
status: 'running',
operation: 'Financial metrics updated',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
// Update from temporary table
await connection.query(`
INSERT INTO product_time_aggregates (
pid, year, month,
total_quantity_sold, total_revenue, total_cost,
order_count, stock_received, stock_ordered,
avg_price, profit_margin, inventory_value, gmroi
)
SELECT
pid, year, month,
total_quantity_sold, total_revenue, total_cost,
order_count, stock_received, stock_ordered,
avg_price, profit_margin, inventory_value, gmroi
FROM temp_time_aggregates
ON DUPLICATE KEY UPDATE
total_quantity_sold = VALUES(total_quantity_sold),
total_revenue = VALUES(total_revenue),
total_cost = VALUES(total_cost),
order_count = VALUES(order_count),
stock_received = VALUES(stock_received),
stock_ordered = VALUES(stock_ordered),
avg_price = VALUES(avg_price),
profit_margin = VALUES(profit_margin),
inventory_value = VALUES(inventory_value),
gmroi = VALUES(gmroi)
`);
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_time_aggregates');
lastPid = batch[batch.length - 1].pid;
processedCount += batch.length;
outputProgress({
status: 'running',
operation: 'Processing time aggregates batch',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
}
// If we get here, everything completed successfully
success = true;
@@ -285,7 +224,7 @@ async function calculateTimeAggregates(startTime, totalProducts, processedCount
return {
processedProducts: processedCount,
processedOrders,
processedOrders: 0,
processedPurchaseOrders: 0,
success
};

513
inventory-server/scripts/metrics/vendor-metrics.js
View File

@@ -4,20 +4,57 @@ const { getConnection } = require('./utils/db');
async function calculateVendorMetrics(startTime, totalProducts, processedCount = 0, isCancelled = false) {
const connection = await getConnection();
let success = false;
let processedOrders = 0;
let processedPurchaseOrders = 0;
const BATCH_SIZE = 5000;
try {
// Get last calculation timestamp
const [lastCalc] = await connection.query(`
SELECT last_calculation_timestamp
FROM calculate_status
WHERE module_name = 'vendor_metrics'
`);
const lastCalculationTime = lastCalc[0]?.last_calculation_timestamp || '1970-01-01';
// Get total count of vendors needing updates using EXISTS for better performance
const [vendorCount] = await connection.query(`
SELECT COUNT(DISTINCT v.vendor) as count
FROM vendor_details v
WHERE v.status = 'active'
AND (
EXISTS (
SELECT 1 FROM products p
WHERE p.vendor = v.vendor
AND p.updated > ?
)
OR EXISTS (
SELECT 1 FROM purchase_orders po
WHERE po.vendor = v.vendor
AND po.updated > ?
)
)
`, [lastCalculationTime, lastCalculationTime]);
const totalVendors = vendorCount[0].count;
if (totalVendors === 0) {
console.log('No vendors need metric updates');
return {
processedProducts: 0,
processedOrders: 0,
processedPurchaseOrders: 0,
success: true
};
}
if (isCancelled) {
outputProgress({
status: 'cancelled',
operation: 'Vendor metrics calculation cancelled',
current: processedCount,
total: totalProducts,
total: totalVendors,
elapsed: formatElapsedTime(startTime),
remaining: null,
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
percentage: ((processedCount / totalVendors) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -26,37 +63,21 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount =
});
return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders,
processedOrders: 0,
processedPurchaseOrders: 0,
success
};
}
// Get counts of records that will be processed
const [[orderCount], [poCount]] = await Promise.all([
connection.query(`
SELECT COUNT(*) as count
FROM orders o
WHERE o.canceled = false
`),
connection.query(`
SELECT COUNT(*) as count
FROM purchase_orders po
WHERE po.status != 0
`)
]);
processedOrders = orderCount.count;
processedPurchaseOrders = poCount.count;
outputProgress({
status: 'running',
operation: 'Starting vendor metrics calculation',
current: processedCount,
total: totalProducts,
total: totalVendors,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
remaining: estimateRemaining(startTime, processedCount, totalVendors),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
percentage: ((processedCount / totalVendors) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
@@ -64,278 +85,192 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount =
}
});
// First ensure all vendors exist in vendor_details
await connection.query(`
INSERT IGNORE INTO vendor_details (vendor, status, created_at, updated_at)
SELECT DISTINCT
vendor,
'active' as status,
NOW() as created_at,
NOW() as updated_at
FROM products
WHERE vendor IS NOT NULL
`);
// Process in batches
let lastVendor = '';
while (true) {
if (isCancelled) break;
processedCount = Math.floor(totalProducts * 0.8);
outputProgress({
status: 'running',
operation: 'Vendor details updated, calculating metrics',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
// Get batch of vendors using EXISTS for better performance
const [batch] = await connection.query(`
SELECT DISTINCT v.vendor
FROM vendor_details v
WHERE v.status = 'active'
AND v.vendor > ?
AND (
EXISTS (
SELECT 1
FROM products p
WHERE p.vendor = v.vendor
AND p.updated > ?
)
OR EXISTS (
SELECT 1
FROM purchase_orders po
WHERE po.vendor = v.vendor
AND po.updated > ?
)
)
ORDER BY v.vendor
LIMIT ?
`, [lastVendor, lastCalculationTime, lastCalculationTime, BATCH_SIZE]);
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders,
success
};
if (batch.length === 0) break;
// Now calculate vendor metrics
await connection.query(`
INSERT INTO vendor_metrics (
vendor,
total_revenue,
total_orders,
total_late_orders,
avg_lead_time_days,
on_time_delivery_rate,
order_fill_rate,
avg_order_value,
active_products,
total_products,
total_purchase_value,
avg_margin_percent,
status,
last_calculated_at
)
WITH vendor_sales AS (
// Create temporary tables with optimized structure and indexes
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_purchase_stats');
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_product_stats');
await connection.query(`
CREATE TEMPORARY TABLE temp_purchase_stats (
vendor VARCHAR(100) NOT NULL,
avg_lead_time_days DECIMAL(10,2),
total_orders INT,
total_late_orders INT,
total_purchase_value DECIMAL(15,2),
avg_order_value DECIMAL(15,2),
on_time_delivery_rate DECIMAL(5,2),
order_fill_rate DECIMAL(5,2),
PRIMARY KEY (vendor),
INDEX (total_orders),
INDEX (total_purchase_value)
) ENGINE=MEMORY
`);
await connection.query(`
CREATE TEMPORARY TABLE temp_product_stats (
vendor VARCHAR(100) NOT NULL,
total_products INT,
active_products INT,
avg_margin_percent DECIMAL(5,2),
total_revenue DECIMAL(15,2),
PRIMARY KEY (vendor),
INDEX (total_products),
INDEX (total_revenue)
) ENGINE=MEMORY
`);
// Populate purchase_stats temp table with optimized index usage
await connection.query(`
INSERT INTO temp_purchase_stats
SELECT
po.vendor,
AVG(DATEDIFF(po.received_date, po.date)) as avg_lead_time_days,
COUNT(DISTINCT po.po_id) as total_orders,
COUNT(CASE WHEN DATEDIFF(po.received_date, po.date) > 30 THEN 1 END) as total_late_orders,
SUM(po.ordered * po.po_cost_price) as total_purchase_value,
AVG(po.ordered * po.po_cost_price) as avg_order_value,
(COUNT(CASE WHEN DATEDIFF(po.received_date, po.date) <= 30 THEN 1 END) / COUNT(*)) * 100 as on_time_delivery_rate,
(SUM(LEAST(po.received, po.ordered)) / NULLIF(SUM(po.ordered), 0)) * 100 as order_fill_rate
FROM purchase_orders po
FORCE INDEX (idx_vendor)
WHERE po.vendor IN (?)
AND po.received_date IS NOT NULL
AND po.date >= DATE_SUB(CURRENT_DATE, INTERVAL 365 DAY)
AND po.updated > ?
GROUP BY po.vendor
`, [batch.map(row => row.vendor), lastCalculationTime]);
// Populate product stats with optimized index usage
await connection.query(`
INSERT INTO temp_product_stats
SELECT
p.vendor,
SUM(o.quantity * o.price) as total_revenue,
COUNT(DISTINCT o.id) as total_orders,
COUNT(DISTINCT p.pid) as active_products,
SUM(o.quantity * (o.price - p.cost_price)) as total_margin
COUNT(DISTINCT p.pid) as product_count,
COUNT(DISTINCT CASE WHEN p.visible = true THEN p.pid END) as active_products,
AVG(pm.avg_margin_percent) as avg_margin,
SUM(pm.total_revenue) as total_revenue
FROM products p
JOIN orders o ON p.pid = o.pid
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
FORCE INDEX (idx_vendor)
LEFT JOIN product_metrics pm FORCE INDEX (PRIMARY) ON p.pid = pm.pid
WHERE p.vendor IN (?)
AND (
p.updated > ?
OR EXISTS (
SELECT 1 FROM orders o FORCE INDEX (idx_orders_metrics)
WHERE o.pid = p.pid
AND o.updated > ?
)
)
GROUP BY p.vendor
),
vendor_po AS (
SELECT
p.vendor,
COUNT(DISTINCT CASE WHEN po.receiving_status = 40 THEN po.id END) as received_orders,
COUNT(DISTINCT po.id) as total_orders,
AVG(CASE
WHEN po.receiving_status = 40
THEN DATEDIFF(po.received_date, po.date)
END) as avg_lead_time_days,
SUM(po.ordered * po.po_cost_price) as total_purchase_value
FROM products p
JOIN purchase_orders po ON p.pid = po.pid
WHERE po.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
GROUP BY p.vendor
),
vendor_products AS (
SELECT
`, [batch.map(row => row.vendor), lastCalculationTime, lastCalculationTime]);
// Update metrics using temp tables with optimized join order
await connection.query(`
INSERT INTO vendor_metrics (
vendor,
COUNT(DISTINCT pid) as total_products
FROM products
GROUP BY vendor
)
SELECT
vs.vendor,
COALESCE(vs.total_revenue, 0) as total_revenue,
COALESCE(vp.total_orders, 0) as total_orders,
COALESCE(vp.total_orders - vp.received_orders, 0) as total_late_orders,
COALESCE(vp.avg_lead_time_days, 0) as avg_lead_time_days,
CASE
WHEN vp.total_orders > 0
THEN (vp.received_orders / vp.total_orders) * 100
ELSE 0
END as on_time_delivery_rate,
CASE
WHEN vp.total_orders > 0
THEN (vp.received_orders / vp.total_orders) * 100
ELSE 0
END as order_fill_rate,
CASE
WHEN vs.total_orders > 0
THEN vs.total_revenue / vs.total_orders
ELSE 0
END as avg_order_value,
COALESCE(vs.active_products, 0) as active_products,
COALESCE(vpr.total_products, 0) as total_products,
COALESCE(vp.total_purchase_value, 0) as total_purchase_value,
CASE
WHEN vs.total_revenue > 0
THEN (vs.total_margin / vs.total_revenue) * 100
ELSE 0
END as avg_margin_percent,
'active' as status,
NOW() as last_calculated_at
FROM vendor_sales vs
LEFT JOIN vendor_po vp ON vs.vendor = vp.vendor
LEFT JOIN vendor_products vpr ON vs.vendor = vpr.vendor
WHERE vs.vendor IS NOT NULL
ON DUPLICATE KEY UPDATE
total_revenue = VALUES(total_revenue),
total_orders = VALUES(total_orders),
total_late_orders = VALUES(total_late_orders),
avg_lead_time_days = VALUES(avg_lead_time_days),
on_time_delivery_rate = VALUES(on_time_delivery_rate),
order_fill_rate = VALUES(order_fill_rate),
avg_order_value = VALUES(avg_order_value),
active_products = VALUES(active_products),
total_products = VALUES(total_products),
total_purchase_value = VALUES(total_purchase_value),
avg_margin_percent = VALUES(avg_margin_percent),
status = VALUES(status),
last_calculated_at = VALUES(last_calculated_at)
`);
processedCount = Math.floor(totalProducts * 0.9);
outputProgress({
status: 'running',
operation: 'Vendor metrics calculated, updating time-based metrics',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
if (isCancelled) return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders,
success
};
// Calculate time-based metrics
await connection.query(`
INSERT INTO vendor_time_metrics (
vendor,
year,
month,
total_orders,
late_orders,
avg_lead_time_days,
total_purchase_value,
total_revenue,
avg_margin_percent
)
WITH monthly_orders AS (
avg_lead_time_days,
on_time_delivery_rate,
order_fill_rate,
total_orders,
total_late_orders,
total_purchase_value,
avg_order_value,
active_products,
total_products,
total_revenue,
avg_margin_percent,
status,
last_calculated_at
)
SELECT
p.vendor,
YEAR(o.date) as year,
MONTH(o.date) as month,
COUNT(DISTINCT o.id) as total_orders,
SUM(o.quantity * o.price) as total_revenue,
SUM(o.quantity * (o.price - p.cost_price)) as total_margin
FROM products p
JOIN orders o ON p.pid = o.pid
WHERE o.canceled = false
AND o.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
AND p.vendor IS NOT NULL
GROUP BY p.vendor, YEAR(o.date), MONTH(o.date)
),
monthly_po AS (
SELECT
p.vendor,
YEAR(po.date) as year,
MONTH(po.date) as month,
COUNT(DISTINCT po.id) as total_po,
COUNT(DISTINCT CASE
WHEN po.receiving_status = 40 AND po.received_date > po.expected_date
THEN po.id
END) as late_orders,
AVG(CASE
WHEN po.receiving_status = 40
THEN DATEDIFF(po.received_date, po.date)
END) as avg_lead_time_days,
SUM(po.ordered * po.po_cost_price) as total_purchase_value
FROM products p
JOIN purchase_orders po ON p.pid = po.pid
WHERE po.date >= DATE_SUB(CURRENT_DATE, INTERVAL 12 MONTH)
AND p.vendor IS NOT NULL
GROUP BY p.vendor, YEAR(po.date), MONTH(po.date)
)
SELECT
mo.vendor,
mo.year,
mo.month,
COALESCE(mp.total_po, 0) as total_orders,
COALESCE(mp.late_orders, 0) as late_orders,
COALESCE(mp.avg_lead_time_days, 0) as avg_lead_time_days,
COALESCE(mp.total_purchase_value, 0) as total_purchase_value,
mo.total_revenue,
CASE
WHEN mo.total_revenue > 0
THEN (mo.total_margin / mo.total_revenue) * 100
ELSE 0
END as avg_margin_percent
FROM monthly_orders mo
LEFT JOIN monthly_po mp ON mo.vendor = mp.vendor
AND mo.year = mp.year
AND mo.month = mp.month
UNION
SELECT
mp.vendor,
mp.year,
mp.month,
mp.total_po as total_orders,
mp.late_orders,
mp.avg_lead_time_days,
mp.total_purchase_value,
0 as total_revenue,
0 as avg_margin_percent
FROM monthly_po mp
LEFT JOIN monthly_orders mo ON mp.vendor = mo.vendor
AND mp.year = mo.year
AND mp.month = mo.month
WHERE mo.vendor IS NULL
ON DUPLICATE KEY UPDATE
total_orders = VALUES(total_orders),
late_orders = VALUES(late_orders),
avg_lead_time_days = VALUES(avg_lead_time_days),
total_purchase_value = VALUES(total_purchase_value),
total_revenue = VALUES(total_revenue),
avg_margin_percent = VALUES(avg_margin_percent)
`);
v.vendor,
COALESCE(ps.avg_lead_time_days, 0) as avg_lead_time_days,
COALESCE(ps.on_time_delivery_rate, 0) as on_time_delivery_rate,
COALESCE(ps.order_fill_rate, 0) as order_fill_rate,
COALESCE(ps.total_orders, 0) as total_orders,
COALESCE(ps.total_late_orders, 0) as total_late_orders,
COALESCE(ps.total_purchase_value, 0) as total_purchase_value,
COALESCE(ps.avg_order_value, 0) as avg_order_value,
COALESCE(prs.active_products, 0) as active_products,
COALESCE(prs.total_products, 0) as total_products,
COALESCE(prs.total_revenue, 0) as total_revenue,
COALESCE(prs.avg_margin_percent, 0) as avg_margin_percent,
v.status,
NOW() as last_calculated_at
FROM vendor_details v
FORCE INDEX (PRIMARY)
LEFT JOIN temp_purchase_stats ps ON v.vendor = ps.vendor
LEFT JOIN temp_product_stats prs ON v.vendor = prs.vendor
WHERE v.vendor IN (?)
ON DUPLICATE KEY UPDATE
avg_lead_time_days = VALUES(avg_lead_time_days),
on_time_delivery_rate = VALUES(on_time_delivery_rate),
order_fill_rate = VALUES(order_fill_rate),
total_orders = VALUES(total_orders),
total_late_orders = VALUES(total_late_orders),
total_purchase_value = VALUES(total_purchase_value),
avg_order_value = VALUES(avg_order_value),
active_products = VALUES(active_products),
total_products = VALUES(total_products),
total_revenue = VALUES(total_revenue),
avg_margin_percent = VALUES(avg_margin_percent),
status = VALUES(status),
last_calculated_at = NOW()
`, [batch.map(row => row.vendor)]);
processedCount = Math.floor(totalProducts * 0.95);
outputProgress({
status: 'running',
operation: 'Time-based vendor metrics calculated',
current: processedCount,
total: totalProducts,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalProducts),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalProducts) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
// Clean up temp tables
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_purchase_stats');
await connection.query('DROP TEMPORARY TABLE IF EXISTS temp_product_stats');
lastVendor = batch[batch.length - 1].vendor;
processedCount += batch.length;
outputProgress({
status: 'running',
operation: 'Processing vendor metrics batch',
current: processedCount,
total: totalVendors,
elapsed: formatElapsedTime(startTime),
remaining: estimateRemaining(startTime, processedCount, totalVendors),
rate: calculateRate(startTime, processedCount),
percentage: ((processedCount / totalVendors) * 100).toFixed(1),
timing: {
start_time: new Date(startTime).toISOString(),
end_time: new Date().toISOString(),
elapsed_seconds: Math.round((Date.now() - startTime) / 1000)
}
});
}
// If we get here, everything completed successfully
success = true;
@@ -349,8 +284,8 @@ async function calculateVendorMetrics(startTime, totalProducts, processedCount =
return {
processedProducts: processedCount,
processedOrders,
processedPurchaseOrders,
processedOrders: 0,
processedPurchaseOrders: 0,
success
};

0
inventory-server/old/old_csv/import-csv.js → inventory-server/scripts/old_csv/import-csv.js
View File

0
inventory-server/old/old_csv/update-csv.js → inventory-server/scripts/old_csv/update-csv.js
View File

367
inventory-server/scripts/reset-db.js
View File

@@ -1,4 +1,4 @@
const { Client } = require('pg');
const mysql = require('mysql2/promise');
const path = require('path');
const dotenv = require('dotenv');
const fs = require('fs');
@@ -10,7 +10,7 @@ const dbConfig = {
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
port: process.env.DB_PORT || 5432
multipleStatements: true
};
// Helper function to output progress in JSON format
@@ -54,44 +54,14 @@ function splitSQLStatements(sql) {
let currentStatement = '';
let inString = false;
let stringChar = '';
let inDollarQuote = false;
let dollarQuoteTag = '';
// Process character by character
for (let i = 0; i < sql.length; i++) {
const char = sql[i];
const nextChar = sql[i + 1] || '';
// Handle dollar quotes
if (char === '$' && !inString) {
// Look ahead to find the dollar quote tag
let tag = '$';
let j = i + 1;
while (j < sql.length && sql[j] !== '$') {
tag += sql[j];
j++;
}
tag += '$';
if (j < sql.length) { // Found closing $
if (!inDollarQuote) {
inDollarQuote = true;
dollarQuoteTag = tag;
currentStatement += tag;
i = j;
continue;
} else if (sql.substring(i, j + 1) === dollarQuoteTag) {
inDollarQuote = false;
dollarQuoteTag = '';
currentStatement += tag;
i = j;
continue;
}
}
}
// Handle string literals (only if not in dollar quote)
if (!inDollarQuote && (char === "'" || char === '"') && sql[i - 1] !== '\\') {
// Handle string literals
if ((char === "'" || char === '"') && sql[i - 1] !== '\\') {
if (!inString) {
inString = true;
stringChar = char;
@@ -100,25 +70,23 @@ function splitSQLStatements(sql) {
}
}
// Handle comments (only if not in string or dollar quote)
if (!inString && !inDollarQuote) {
if (char === '-' && nextChar === '-') {
// Skip to end of line
while (i < sql.length && sql[i] !== '\n') i++;
continue;
}
if (char === '/' && nextChar === '*') {
// Skip until closing */
i += 2;
while (i < sql.length && (sql[i] !== '*' || sql[i + 1] !== '/')) i++;
i++; // Skip the closing /
continue;
}
// Handle comments
if (!inString && char === '-' && nextChar === '-') {
// Skip to end of line
while (i < sql.length && sql[i] !== '\n') i++;
continue;
}
// Handle statement boundaries (only if not in string or dollar quote)
if (!inString && !inDollarQuote && char === ';') {
if (!inString && char === '/' && nextChar === '*') {
// Skip until closing */
i += 2;
while (i < sql.length && (sql[i] !== '*' || sql[i + 1] !== '/')) i++;
i++; // Skip the closing /
continue;
}
// Handle statement boundaries
if (!inString && char === ';') {
if (currentStatement.trim()) {
statements.push(currentStatement.trim());
}
@@ -152,42 +120,46 @@ async function resetDatabase() {
}
});
const client = new Client(dbConfig);
await client.connect();
const connection = await mysql.createConnection(dbConfig);
try {
// Check PostgreSQL version and user
// Check MySQL privileges
outputProgress({
operation: 'Checking database',
message: 'Verifying PostgreSQL version and user privileges...'
operation: 'Checking privileges',
message: 'Verifying MySQL user privileges...'
});
const versionResult = await client.query('SELECT version()');
const userResult = await client.query('SELECT current_user, current_database()');
const [grants] = await connection.query('SHOW GRANTS');
outputProgress({
operation: 'Database info',
operation: 'User privileges',
message: {
version: versionResult.rows[0].version,
user: userResult.rows[0].current_user,
database: userResult.rows[0].current_database
grants: grants.map(g => Object.values(g)[0])
}
});
// Enable warnings as errors
await connection.query('SET SESSION sql_notes = 1');
// Log database config (without sensitive info)
outputProgress({
operation: 'Database config',
message: `Using database: ${dbConfig.database} on host: ${dbConfig.host}`
});
// Get list of all tables in the current database
outputProgress({
operation: 'Getting table list',
message: 'Retrieving all table names...'
});
const tablesResult = await client.query(`
SELECT string_agg(tablename, ', ') as tables
FROM pg_tables
WHERE schemaname = 'public'
AND tablename NOT IN ('users', 'permissions', 'user_permissions', 'calculate_history', 'import_history');
const [tables] = await connection.query(`
SELECT GROUP_CONCAT(table_name) as tables
FROM information_schema.tables
WHERE table_schema = DATABASE()
AND table_name NOT IN ('users', 'import_history')
`);
if (!tablesResult.rows[0].tables) {
if (!tables[0].tables) {
outputProgress({
operation: 'No tables found',
message: 'Database is already empty'
@@ -198,73 +170,20 @@ async function resetDatabase() {
message: 'Dropping all existing tables...'
});
// Disable triggers/foreign key checks
await client.query('SET session_replication_role = \'replica\';');
// Drop all tables except users
const tables = tablesResult.rows[0].tables.split(', ');
for (const table of tables) {
if (!['users'].includes(table)) {
await client.query(`DROP TABLE IF EXISTS "${table}" CASCADE`);
}
}
// Only drop types if we're not preserving history tables
const historyTablesExist = await client.query(`
SELECT EXISTS (
SELECT FROM pg_tables
WHERE schemaname = 'public'
AND tablename IN ('calculate_history', 'import_history')
);
`);
if (!historyTablesExist.rows[0].exists) {
await client.query('DROP TYPE IF EXISTS calculation_status CASCADE;');
await client.query('DROP TYPE IF EXISTS module_name CASCADE;');
}
// Re-enable triggers/foreign key checks
await client.query('SET session_replication_role = \'origin\';');
await connection.query('SET FOREIGN_KEY_CHECKS = 0');
const dropQuery = `
DROP TABLE IF EXISTS
${tables[0].tables
.split(',')
.filter(table => table !== 'users')
.map(table => '`' + table + '`')
.join(', ')}
`;
await connection.query(dropQuery);
await connection.query('SET FOREIGN_KEY_CHECKS = 1');
}
// Create enum types if they don't exist
outputProgress({
operation: 'Creating enum types',
message: 'Setting up required enum types...'
});
// Check if types exist before creating
const typesExist = await client.query(`
SELECT EXISTS (
SELECT 1 FROM pg_type
WHERE typname = 'calculation_status'
) as calc_status_exists,
EXISTS (
SELECT 1 FROM pg_type
WHERE typname = 'module_name'
) as module_name_exists;
`);
if (!typesExist.rows[0].calc_status_exists) {
await client.query(`CREATE TYPE calculation_status AS ENUM ('running', 'completed', 'failed', 'cancelled')`);
}
if (!typesExist.rows[0].module_name_exists) {
await client.query(`
CREATE TYPE module_name AS ENUM (
'product_metrics',
'time_aggregates',
'financial_metrics',
'vendor_metrics',
'category_metrics',
'brand_metrics',
'sales_forecasts',
'abc_classification'
)
`);
}
// Read and execute main schema first (core tables)
// Read and execute main schema (core tables)
outputProgress({
operation: 'Running database setup',
message: 'Creating core tables...'
@@ -304,24 +223,35 @@ async function resetDatabase() {
for (let i = 0; i < statements.length; i++) {
const stmt = statements[i];
try {
const result = await client.query(stmt);
const [result, fields] = await connection.query(stmt);
// Check for warnings
const [warnings] = await connection.query('SHOW WARNINGS');
if (warnings && warnings.length > 0) {
outputProgress({
status: 'warning',
operation: 'SQL Warning',
statement: i + 1,
warnings: warnings
});
}
// Verify if table was created (if this was a CREATE TABLE statement)
if (stmt.trim().toLowerCase().startsWith('create table')) {
const tableName = stmt.match(/create\s+table\s+(?:if\s+not\s+exists\s+)?["]?(\w+)["]?/i)?.[1];
const tableName = stmt.match(/create\s+table\s+(?:if\s+not\s+exists\s+)?`?(\w+)`?/i)?.[1];
if (tableName) {
const tableExists = await client.query(`
const [tableExists] = await connection.query(`
SELECT COUNT(*) as count
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name = $1
WHERE table_schema = DATABASE()
AND table_name = ?
`, [tableName]);
outputProgress({
operation: 'Table Creation Verification',
message: {
table: tableName,
exists: tableExists.rows[0].count > 0
exists: tableExists[0].count > 0
}
});
}
@@ -333,7 +263,7 @@ async function resetDatabase() {
statement: i + 1,
total: statements.length,
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : ''),
rowCount: result.rowCount
affectedRows: result.affectedRows
}
});
} catch (sqlError) {
@@ -341,6 +271,8 @@ async function resetDatabase() {
status: 'error',
operation: 'SQL Error',
error: sqlError.message,
sqlState: sqlError.sqlState,
errno: sqlError.errno,
statement: stmt,
statementNumber: i + 1
});
@@ -348,12 +280,66 @@ async function resetDatabase() {
}
}
// Verify core tables were created
const existingTables = (await client.query(`
SELECT table_name
// List all tables in the database after schema execution
outputProgress({
operation: 'Debug database',
message: {
currentDatabase: (await connection.query('SELECT DATABASE() as db'))[0][0].db
}
});
const [allTables] = await connection.query(`
SELECT
table_schema,
table_name,
engine,
create_time,
table_rows
FROM information_schema.tables
WHERE table_schema = 'public'
`)).rows.map(t => t.table_name);
WHERE table_schema = DATABASE()
`);
if (allTables.length === 0) {
outputProgress({
operation: 'Warning',
message: 'No tables found in database after schema execution'
});
} else {
outputProgress({
operation: 'Tables after schema execution',
message: {
count: allTables.length,
tables: allTables.map(t => ({
schema: t.table_schema,
name: t.table_name,
engine: t.engine,
created: t.create_time,
rows: t.table_rows
}))
}
});
}
// Also check table status
const [tableStatus] = await connection.query('SHOW TABLE STATUS');
outputProgress({
operation: 'Table Status',
message: {
tables: tableStatus.map(t => ({
name: t.Name,
engine: t.Engine,
version: t.Version,
rowFormat: t.Row_format,
rows: t.Rows,
createTime: t.Create_time,
updateTime: t.Update_time
}))
}
});
// Verify core tables were created using SHOW TABLES
const [showTables] = await connection.query('SHOW TABLES');
const existingTables = showTables.map(t => Object.values(t)[0]);
outputProgress({
operation: 'Core tables verification',
@@ -373,12 +359,22 @@ async function resetDatabase() {
);
}
// Verify all core tables use InnoDB
const [engineStatus] = await connection.query('SHOW TABLE STATUS WHERE Name IN (?)', [CORE_TABLES]);
const nonInnoDBTables = engineStatus.filter(t => t.Engine !== 'InnoDB');
if (nonInnoDBTables.length > 0) {
throw new Error(
`Tables using non-InnoDB engine: ${nonInnoDBTables.map(t => t.Name).join(', ')}`
);
}
outputProgress({
operation: 'Core tables created',
message: `Successfully created tables: ${CORE_TABLES.join(', ')}`
});
// Now read and execute config schema (since core tables exist)
// Read and execute config schema
outputProgress({
operation: 'Running config setup',
message: 'Creating configuration tables...'
@@ -404,7 +400,18 @@ async function resetDatabase() {
for (let i = 0; i < configStatements.length; i++) {
const stmt = configStatements[i];
try {
const result = await client.query(stmt);
const [result, fields] = await connection.query(stmt);
// Check for warnings
const [warnings] = await connection.query('SHOW WARNINGS');
if (warnings && warnings.length > 0) {
outputProgress({
status: 'warning',
operation: 'Config SQL Warning',
statement: i + 1,
warnings: warnings
});
}
outputProgress({
operation: 'Config SQL Progress',
@@ -412,7 +419,7 @@ async function resetDatabase() {
statement: i + 1,
total: configStatements.length,
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : ''),
rowCount: result.rowCount
affectedRows: result.affectedRows
}
});
} catch (sqlError) {
@@ -420,6 +427,8 @@ async function resetDatabase() {
status: 'error',
operation: 'Config SQL Error',
error: sqlError.message,
sqlState: sqlError.sqlState,
errno: sqlError.errno,
statement: stmt,
statementNumber: i + 1
});
@@ -427,6 +436,33 @@ async function resetDatabase() {
}
}
// Verify config tables were created
const [showConfigTables] = await connection.query('SHOW TABLES');
const existingConfigTables = showConfigTables.map(t => Object.values(t)[0]);
outputProgress({
operation: 'Config tables verification',
message: {
found: existingConfigTables,
expected: CONFIG_TABLES
}
});
const missingConfigTables = CONFIG_TABLES.filter(
t => !existingConfigTables.includes(t)
);
if (missingConfigTables.length > 0) {
throw new Error(
`Failed to create config tables: ${missingConfigTables.join(', ')}`
);
}
outputProgress({
operation: 'Config tables created',
message: `Successfully created tables: ${CONFIG_TABLES.join(', ')}`
});
// Read and execute metrics schema (metrics tables)
outputProgress({
operation: 'Running metrics setup',
@@ -453,7 +489,18 @@ async function resetDatabase() {
for (let i = 0; i < metricsStatements.length; i++) {
const stmt = metricsStatements[i];
try {
const result = await client.query(stmt);
const [result, fields] = await connection.query(stmt);
// Check for warnings
const [warnings] = await connection.query('SHOW WARNINGS');
if (warnings && warnings.length > 0) {
outputProgress({
status: 'warning',
operation: 'Metrics SQL Warning',
statement: i + 1,
warnings: warnings
});
}
outputProgress({
operation: 'Metrics SQL Progress',
@@ -461,7 +508,7 @@ async function resetDatabase() {
statement: i + 1,
total: metricsStatements.length,
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : ''),
rowCount: result.rowCount
affectedRows: result.affectedRows
}
});
} catch (sqlError) {
@@ -469,6 +516,8 @@ async function resetDatabase() {
status: 'error',
operation: 'Metrics SQL Error',
error: sqlError.message,
sqlState: sqlError.sqlState,
errno: sqlError.errno,
statement: stmt,
statementNumber: i + 1
});
@@ -490,19 +539,9 @@ async function resetDatabase() {
});
process.exit(1);
} finally {
await client.end();
await connection.end();
}
}
// Export if required as a module
if (typeof module !== 'undefined' && module.exports) {
module.exports = resetDatabase;
}
// Run if called directly
if (require.main === module) {
resetDatabase().catch(error => {
console.error('Error:', error);
process.exit(1);
});
}
// Run the reset
resetDatabase();

133
inventory-server/scripts/reset-metrics.js
View File

@@ -1,4 +1,4 @@
const { Client } = require('pg');
const mysql = require('mysql2/promise');
const path = require('path');
const fs = require('fs');
require('dotenv').config({ path: path.resolve(__dirname, '../.env') });
@@ -8,7 +8,7 @@ const dbConfig = {
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
port: process.env.DB_PORT || 5432
multipleStatements: true
};
function outputProgress(data) {
@@ -34,8 +34,8 @@ const METRICS_TABLES = [
'sales_forecasts',
'temp_purchase_metrics',
'temp_sales_metrics',
'vendor_metrics',
'vendor_time_metrics',
'vendor_metrics', //before vendor_details for foreign key
'vendor_time_metrics', //before vendor_details for foreign key
'vendor_details'
];
@@ -90,31 +90,31 @@ function splitSQLStatements(sql) {
}
async function resetMetrics() {
let client;
let connection;
try {
outputProgress({
operation: 'Starting metrics reset',
message: 'Connecting to database...'
});
client = new Client(dbConfig);
await client.connect();
connection = await mysql.createConnection(dbConfig);
await connection.beginTransaction();
// First verify current state
const initialTables = await client.query(`
SELECT tablename as name
FROM pg_tables
WHERE schemaname = 'public'
AND tablename = ANY($1)
const [initialTables] = await connection.query(`
SELECT TABLE_NAME as name
FROM information_schema.tables
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME IN (?)
`, [METRICS_TABLES]);
outputProgress({
operation: 'Initial state',
message: `Found ${initialTables.rows.length} existing metrics tables: ${initialTables.rows.map(t => t.name).join(', ')}`
message: `Found ${initialTables.length} existing metrics tables: ${initialTables.map(t => t.name).join(', ')}`
});
// Disable foreign key checks at the start
await client.query('SET session_replication_role = \'replica\'');
await connection.query('SET FOREIGN_KEY_CHECKS = 0');
// Drop all metrics tables in reverse order to handle dependencies
outputProgress({
@@ -124,17 +124,17 @@ async function resetMetrics() {
for (const table of [...METRICS_TABLES].reverse()) {
try {
await client.query(`DROP TABLE IF EXISTS "${table}" CASCADE`);
await connection.query(`DROP TABLE IF EXISTS ${table}`);
// Verify the table was actually dropped
const checkDrop = await client.query(`
const [checkDrop] = await connection.query(`
SELECT COUNT(*) as count
FROM pg_tables
WHERE schemaname = 'public'
AND tablename = $1
FROM information_schema.tables
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = ?
`, [table]);
if (parseInt(checkDrop.rows[0].count) > 0) {
if (checkDrop[0].count > 0) {
throw new Error(`Failed to drop table ${table} - table still exists`);
}
@@ -153,15 +153,15 @@ async function resetMetrics() {
}
// Verify all tables were dropped
const afterDrop = await client.query(`
SELECT tablename as name
FROM pg_tables
WHERE schemaname = 'public'
AND tablename = ANY($1)
const [afterDrop] = await connection.query(`
SELECT TABLE_NAME as name
FROM information_schema.tables
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME IN (?)
`, [METRICS_TABLES]);
if (afterDrop.rows.length > 0) {
throw new Error(`Failed to drop all tables. Remaining tables: ${afterDrop.rows.map(t => t.name).join(', ')}`);
if (afterDrop.length > 0) {
throw new Error(`Failed to drop all tables. Remaining tables: ${afterDrop.map(t => t.name).join(', ')}`);
}
// Read metrics schema
@@ -187,26 +187,39 @@ async function resetMetrics() {
for (let i = 0; i < statements.length; i++) {
const stmt = statements[i];
try {
const result = await client.query(stmt);
await connection.query(stmt);
// Check for warnings
const [warnings] = await connection.query('SHOW WARNINGS');
if (warnings && warnings.length > 0) {
outputProgress({
status: 'warning',
operation: 'SQL Warning',
message: {
statement: i + 1,
warnings: warnings
}
});
}
// If this is a CREATE TABLE statement, verify the table was created
if (stmt.trim().toLowerCase().startsWith('create table')) {
const tableName = stmt.match(/create\s+table\s+(?:if\s+not\s+exists\s+)?["]?(\w+)["]?/i)?.[1];
const tableName = stmt.match(/create\s+table\s+(?:if\s+not\s+exists\s+)?`?(\w+)`?/i)?.[1];
if (tableName) {
const checkCreate = await client.query(`
SELECT tablename as name
FROM pg_tables
WHERE schemaname = 'public'
AND tablename = $1
const [checkCreate] = await connection.query(`
SELECT TABLE_NAME as name, CREATE_TIME as created
FROM information_schema.tables
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = ?
`, [tableName]);
if (checkCreate.rows.length === 0) {
if (checkCreate.length === 0) {
throw new Error(`Failed to create table ${tableName} - table does not exist after CREATE statement`);
}
outputProgress({
operation: 'Table created',
message: `Successfully created table: ${tableName}`
message: `Successfully created table: ${tableName} at ${checkCreate[0].created}`
});
}
}
@@ -216,8 +229,7 @@ async function resetMetrics() {
message: {
statement: i + 1,
total: statements.length,
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : ''),
rowCount: result.rowCount
preview: stmt.substring(0, 100) + (stmt.length > 100 ? '...' : '')
}
});
} catch (sqlError) {
@@ -226,6 +238,8 @@ async function resetMetrics() {
operation: 'SQL Error',
message: {
error: sqlError.message,
sqlState: sqlError.sqlState,
errno: sqlError.errno,
statement: stmt,
statementNumber: i + 1
}
@@ -235,7 +249,7 @@ async function resetMetrics() {
}
// Re-enable foreign key checks after all tables are created
await client.query('SET session_replication_role = \'origin\'');
await connection.query('SET FOREIGN_KEY_CHECKS = 1');
// Verify metrics tables were created
outputProgress({
@@ -243,36 +257,37 @@ async function resetMetrics() {
message: 'Checking all metrics tables were created...'
});
const metricsTablesResult = await client.query(`
SELECT tablename as name
FROM pg_tables
WHERE schemaname = 'public'
AND tablename = ANY($1)
const [metricsTablesResult] = await connection.query(`
SELECT
TABLE_NAME as name,
TABLE_ROWS as \`rows\`,
CREATE_TIME as created
FROM information_schema.tables
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME IN (?)
`, [METRICS_TABLES]);
outputProgress({
operation: 'Tables found',
message: `Found ${metricsTablesResult.rows.length} tables: ${metricsTablesResult.rows.map(t => t.name).join(', ')}`
message: `Found ${metricsTablesResult.length} tables: ${metricsTablesResult.map(t =>
`${t.name} (created: ${t.created})`
).join(', ')}`
});
const existingMetricsTables = metricsTablesResult.rows.map(t => t.name);
const existingMetricsTables = metricsTablesResult.map(t => t.name);
const missingMetricsTables = METRICS_TABLES.filter(t => !existingMetricsTables.includes(t));
if (missingMetricsTables.length > 0) {
// Do one final check of the actual tables
const finalCheck = await client.query(`
SELECT tablename as name
FROM pg_tables
WHERE schemaname = 'public'
`);
const [finalCheck] = await connection.query('SHOW TABLES');
outputProgress({
operation: 'Final table check',
message: `All database tables: ${finalCheck.rows.map(t => t.name).join(', ')}`
message: `All database tables: ${finalCheck.map(t => Object.values(t)[0]).join(', ')}`
});
throw new Error(`Failed to create metrics tables: ${missingMetricsTables.join(', ')}`);
}
await client.query('COMMIT');
await connection.commit();
outputProgress({
status: 'complete',
@@ -287,17 +302,17 @@ async function resetMetrics() {
stack: error.stack
});
if (client) {
await client.query('ROLLBACK');
if (connection) {
await connection.rollback();
// Make sure to re-enable foreign key checks even if there's an error
await client.query('SET session_replication_role = \'origin\'').catch(() => {});
await connection.query('SET FOREIGN_KEY_CHECKS = 1').catch(() => {});
}
throw error;
} finally {
if (client) {
if (connection) {
// One final attempt to ensure foreign key checks are enabled
await client.query('SET session_replication_role = \'origin\'').catch(() => {});
await client.end();
await connection.query('SET FOREIGN_KEY_CHECKS = 1').catch(() => {});
await connection.end();
}
}
}

0
inventory-server/old/scripts.js → inventory-server/scripts/scripts.js
View File

0
inventory-server/old/test-prod-connection.js → inventory-server/scripts/test-prod-connection.js
View File

335
inventory-server/src/routes/ai-prompts.js
View File

@@ -1,335 +0,0 @@
const express = require('express');
const router = express.Router();
// Get all AI prompts
router.get('/', async (req, res) => {
try {
const pool = req.app.locals.pool;
if (!pool) {
throw new Error('Database pool not initialized');
}
const result = await pool.query(`
SELECT * FROM ai_prompts
ORDER BY prompt_type ASC, company ASC
`);
res.json(result.rows);
} catch (error) {
console.error('Error fetching AI prompts:', error);
res.status(500).json({
error: 'Failed to fetch AI prompts',
details: error instanceof Error ? error.message : 'Unknown error'
});
}
});
// Get prompt by ID
router.get('/:id', async (req, res) => {
try {
const { id } = req.params;
const pool = req.app.locals.pool;
if (!pool) {
throw new Error('Database pool not initialized');
}
const result = await pool.query(`
SELECT * FROM ai_prompts
WHERE id = $1
`, [id]);
if (result.rows.length === 0) {
return res.status(404).json({ error: 'AI prompt not found' });
}
res.json(result.rows[0]);
} catch (error) {
console.error('Error fetching AI prompt:', error);
res.status(500).json({
error: 'Failed to fetch AI prompt',
details: error instanceof Error ? error.message : 'Unknown error'
});
}
});
// Get prompt by company
router.get('/company/:companyId', async (req, res) => {
try {
const { companyId } = req.params;
const pool = req.app.locals.pool;
if (!pool) {
throw new Error('Database pool not initialized');
}
const result = await pool.query(`
SELECT * FROM ai_prompts
WHERE company = $1
`, [companyId]);
if (result.rows.length === 0) {
return res.status(404).json({ error: 'AI prompt not found for this company' });
}
res.json(result.rows[0]);
} catch (error) {
console.error('Error fetching AI prompt by company:', error);
res.status(500).json({
error: 'Failed to fetch AI prompt by company',
details: error instanceof Error ? error.message : 'Unknown error'
});
}
});
// Get general prompt
router.get('/type/general', async (req, res) => {
try {
const pool = req.app.locals.pool;
if (!pool) {
throw new Error('Database pool not initialized');
}
const result = await pool.query(`
SELECT * FROM ai_prompts
WHERE prompt_type = 'general'
`);
if (result.rows.length === 0) {
return res.status(404).json({ error: 'General AI prompt not found' });
}
res.json(result.rows[0]);
} catch (error) {
console.error('Error fetching general AI prompt:', error);
res.status(500).json({
error: 'Failed to fetch general AI prompt',
details: error instanceof Error ? error.message : 'Unknown error'
});
}
});
// Get system prompt
router.get('/type/system', async (req, res) => {
try {
const pool = req.app.locals.pool;
if (!pool) {
throw new Error('Database pool not initialized');
}
const result = await pool.query(`
SELECT * FROM ai_prompts
WHERE prompt_type = 'system'
`);
if (result.rows.length === 0) {
return res.status(404).json({ error: 'System AI prompt not found' });
}
res.json(result.rows[0]);
} catch (error) {
console.error('Error fetching system AI prompt:', error);
res.status(500).json({
error: 'Failed to fetch system AI prompt',
details: error instanceof Error ? error.message : 'Unknown error'
});
}
});
// Create new AI prompt
router.post('/', async (req, res) => {
try {
const {
prompt_text,
prompt_type,
company
} = req.body;
// Validate required fields
if (!prompt_text || !prompt_type) {
return res.status(400).json({ error: 'Prompt text and type are required' });
}
// Validate prompt type
if (!['general', 'company_specific', 'system'].includes(prompt_type)) {
return res.status(400).json({ error: 'Prompt type must be either "general", "company_specific", or "system"' });
}
// Validate company is provided for company-specific prompts
if (prompt_type === 'company_specific' && !company) {
return res.status(400).json({ error: 'Company is required for company-specific prompts' });
}
// Validate company is not provided for general or system prompts
if ((prompt_type === 'general' || prompt_type === 'system') && company) {
return res.status(400).json({ error: 'Company should not be provided for general or system prompts' });
}
const pool = req.app.locals.pool;
if (!pool) {
throw new Error('Database pool not initialized');
}
const result = await pool.query(`
INSERT INTO ai_prompts (
prompt_text,
prompt_type,
company
) VALUES ($1, $2, $3)
RETURNING *
`, [
prompt_text,
prompt_type,
company
]);
res.status(201).json(result.rows[0]);
} catch (error) {
console.error('Error creating AI prompt:', error);
// Check for unique constraint violations
if (error instanceof Error && error.message.includes('unique constraint')) {
if (error.message.includes('unique_company_prompt')) {
return res.status(409).json({
error: 'A prompt already exists for this company',
details: error.message
});
} else if (error.message.includes('idx_unique_general_prompt')) {
return res.status(409).json({
error: 'A general prompt already exists',
details: error.message
});
} else if (error.message.includes('idx_unique_system_prompt')) {
return res.status(409).json({
error: 'A system prompt already exists',
details: error.message
});
}
}
res.status(500).json({
error: 'Failed to create AI prompt',
details: error instanceof Error ? error.message : 'Unknown error'
});
}
});
// Update AI prompt
router.put('/:id', async (req, res) => {
try {
const { id } = req.params;
const {
prompt_text,
prompt_type,
company
} = req.body;
// Validate required fields
if (!prompt_text || !prompt_type) {
return res.status(400).json({ error: 'Prompt text and type are required' });
}
// Validate prompt type
if (!['general', 'company_specific', 'system'].includes(prompt_type)) {
return res.status(400).json({ error: 'Prompt type must be either "general", "company_specific", or "system"' });
}
// Validate company is provided for company-specific prompts
if (prompt_type === 'company_specific' && !company) {
return res.status(400).json({ error: 'Company is required for company-specific prompts' });
}
// Validate company is not provided for general or system prompts
if ((prompt_type === 'general' || prompt_type === 'system') && company) {
return res.status(400).json({ error: 'Company should not be provided for general or system prompts' });
}
const pool = req.app.locals.pool;
if (!pool) {
throw new Error('Database pool not initialized');
}
// Check if the prompt exists
const checkResult = await pool.query('SELECT * FROM ai_prompts WHERE id = $1', [id]);
if (checkResult.rows.length === 0) {
return res.status(404).json({ error: 'AI prompt not found' });
}
const result = await pool.query(`
UPDATE ai_prompts
SET
prompt_text = $1,
prompt_type = $2,
company = $3
WHERE id = $4
RETURNING *
`, [
prompt_text,
prompt_type,
company,
id
]);
res.json(result.rows[0]);
} catch (error) {
console.error('Error updating AI prompt:', error);
// Check for unique constraint violations
if (error instanceof Error && error.message.includes('unique constraint')) {
if (error.message.includes('unique_company_prompt')) {
return res.status(409).json({
error: 'A prompt already exists for this company',
details: error.message
});
} else if (error.message.includes('idx_unique_general_prompt')) {
return res.status(409).json({
error: 'A general prompt already exists',
details: error.message
});
} else if (error.message.includes('idx_unique_system_prompt')) {
return res.status(409).json({
error: 'A system prompt already exists',
details: error.message
});
}
}
res.status(500).json({
error: 'Failed to update AI prompt',
details: error instanceof Error ? error.message : 'Unknown error'
});
}
});
// Delete AI prompt
router.delete('/:id', async (req, res) => {
try {
const { id } = req.params;
const pool = req.app.locals.pool;
if (!pool) {
throw new Error('Database pool not initialized');
}
const result = await pool.query('DELETE FROM ai_prompts WHERE id = $1 RETURNING *', [id]);
if (result.rows.length === 0) {
return res.status(404).json({ error: 'AI prompt not found' });
}
res.json({ message: 'AI prompt deleted successfully' });
} catch (error) {
console.error('Error deleting AI prompt:', error);
res.status(500).json({
error: 'Failed to delete AI prompt',
details: error instanceof Error ? error.message : 'Unknown error'
});
}
});
// Error handling middleware
router.use((err, req, res, next) => {
console.error('AI prompts route error:', err);
res.status(500).json({
error: 'Internal server error',
details: err.message
});
});
module.exports = router;

1326
inventory-server/src/routes/ai-validation.js
View File

File diff suppressed because it is too large Load Diff

147
inventory-server/src/routes/analytics.js
View File

@@ -6,24 +6,24 @@ router.get('/stats', async (req, res) => {
try {
const pool = req.app.locals.pool;
const { rows: [results] } = await pool.query(`
const [results] = await pool.query(`
SELECT
COALESCE(
ROUND(
(SUM(o.price * o.quantity - p.cost_price * o.quantity) /
NULLIF(SUM(o.price * o.quantity), 0) * 100)::numeric, 1
NULLIF(SUM(o.price * o.quantity), 0)) * 100, 1
),
0
) as profitMargin,
COALESCE(
ROUND(
(AVG(p.price / NULLIF(p.cost_price, 0) - 1) * 100)::numeric, 1
(AVG(p.price / NULLIF(p.cost_price, 0) - 1) * 100), 1
),
0
) as averageMarkup,
COALESCE(
ROUND(
(SUM(o.quantity) / NULLIF(AVG(p.stock_quantity), 0))::numeric, 2
SUM(o.quantity) / NULLIF(AVG(p.stock_quantity), 0), 2
),
0
) as stockTurnoverRate,
@@ -31,23 +31,23 @@ router.get('/stats', async (req, res) => {
COALESCE(COUNT(DISTINCT p.categories), 0) as categoryCount,
COALESCE(
ROUND(
AVG(o.price * o.quantity)::numeric, 2
AVG(o.price * o.quantity), 2
),
0
) as averageOrderValue
FROM products p
LEFT JOIN orders o ON p.pid = o.pid
WHERE o.date >= CURRENT_DATE - INTERVAL '30 days'
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
`);
// Ensure all values are numbers
const stats = {
profitMargin: Number(results.profitmargin) || 0,
averageMarkup: Number(results.averagemarkup) || 0,
stockTurnoverRate: Number(results.stockturnoverrate) || 0,
vendorCount: Number(results.vendorcount) || 0,
categoryCount: Number(results.categorycount) || 0,
averageOrderValue: Number(results.averageordervalue) || 0
profitMargin: Number(results[0].profitMargin) || 0,
averageMarkup: Number(results[0].averageMarkup) || 0,
stockTurnoverRate: Number(results[0].stockTurnoverRate) || 0,
vendorCount: Number(results[0].vendorCount) || 0,
categoryCount: Number(results[0].categoryCount) || 0,
averageOrderValue: Number(results[0].averageOrderValue) || 0
};
res.json(stats);
@@ -63,13 +63,13 @@ router.get('/profit', async (req, res) => {
const pool = req.app.locals.pool;
// Get profit margins by category with full path
const { rows: byCategory } = await pool.query(`
const [byCategory] = await pool.query(`
WITH RECURSIVE category_path AS (
SELECT
c.cat_id,
c.name,
c.parent_id,
c.name::text as path
CAST(c.name AS CHAR(1000)) as path
FROM categories c
WHERE c.parent_id IS NULL
@@ -79,7 +79,7 @@ router.get('/profit', async (req, res) => {
c.cat_id,
c.name,
c.parent_id,
cp.path || ' > ' || c.name
CONCAT(cp.path, ' > ', c.name)
FROM categories c
JOIN category_path cp ON c.parent_id = cp.cat_id
)
@@ -88,46 +88,53 @@ router.get('/profit', async (req, res) => {
cp.path as categoryPath,
ROUND(
(SUM(o.price * o.quantity - p.cost_price * o.quantity) /
NULLIF(SUM(o.price * o.quantity), 0) * 100)::numeric, 1
NULLIF(SUM(o.price * o.quantity), 0)) * 100, 1
) as profitMargin,
ROUND(SUM(o.price * o.quantity)::numeric, 3) as revenue,
ROUND(SUM(p.cost_price * o.quantity)::numeric, 3) as cost
CAST(SUM(o.price * o.quantity) AS DECIMAL(15,3)) as revenue,
CAST(SUM(p.cost_price * o.quantity) AS DECIMAL(15,3)) as cost
FROM products p
LEFT JOIN orders o ON p.pid = o.pid
JOIN product_categories pc ON p.pid = pc.pid
JOIN categories c ON pc.cat_id = c.cat_id
JOIN category_path cp ON c.cat_id = cp.cat_id
WHERE o.date >= CURRENT_DATE - INTERVAL '30 days'
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
GROUP BY c.name, cp.path
ORDER BY profitMargin DESC
LIMIT 10
`);
// Get profit margin trend over time
const { rows: overTime } = await pool.query(`
const [overTime] = await pool.query(`
SELECT
to_char(o.date, 'YYYY-MM-DD') as date,
formatted_date as date,
ROUND(
(SUM(o.price * o.quantity - p.cost_price * o.quantity) /
NULLIF(SUM(o.price * o.quantity), 0) * 100)::numeric, 1
NULLIF(SUM(o.price * o.quantity), 0)) * 100, 1
) as profitMargin,
ROUND(SUM(o.price * o.quantity)::numeric, 3) as revenue,
ROUND(SUM(p.cost_price * o.quantity)::numeric, 3) as cost
CAST(SUM(o.price * o.quantity) AS DECIMAL(15,3)) as revenue,
CAST(SUM(p.cost_price * o.quantity) AS DECIMAL(15,3)) as cost
FROM products p
LEFT JOIN orders o ON p.pid = o.pid
WHERE o.date >= CURRENT_DATE - INTERVAL '30 days'
GROUP BY to_char(o.date, 'YYYY-MM-DD')
ORDER BY date
CROSS JOIN (
SELECT DATE_FORMAT(o.date, '%Y-%m-%d') as formatted_date
FROM orders o
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
GROUP BY DATE_FORMAT(o.date, '%Y-%m-%d')
) dates
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
AND DATE_FORMAT(o.date, '%Y-%m-%d') = dates.formatted_date
GROUP BY formatted_date
ORDER BY formatted_date
`);
// Get top performing products with category paths
const { rows: topProducts } = await pool.query(`
const [topProducts] = await pool.query(`
WITH RECURSIVE category_path AS (
SELECT
c.cat_id,
c.name,
c.parent_id,
c.name::text as path
CAST(c.name AS CHAR(1000)) as path
FROM categories c
WHERE c.parent_id IS NULL
@@ -137,7 +144,7 @@ router.get('/profit', async (req, res) => {
c.cat_id,
c.name,
c.parent_id,
cp.path || ' > ' || c.name
CONCAT(cp.path, ' > ', c.name)
FROM categories c
JOIN category_path cp ON c.parent_id = cp.cat_id
)
@@ -147,18 +154,18 @@ router.get('/profit', async (req, res) => {
cp.path as categoryPath,
ROUND(
(SUM(o.price * o.quantity - p.cost_price * o.quantity) /
NULLIF(SUM(o.price * o.quantity), 0) * 100)::numeric, 1
NULLIF(SUM(o.price * o.quantity), 0)) * 100, 1
) as profitMargin,
ROUND(SUM(o.price * o.quantity)::numeric, 3) as revenue,
ROUND(SUM(p.cost_price * o.quantity)::numeric, 3) as cost
CAST(SUM(o.price * o.quantity) AS DECIMAL(15,3)) as revenue,
CAST(SUM(p.cost_price * o.quantity) AS DECIMAL(15,3)) as cost
FROM products p
LEFT JOIN orders o ON p.pid = o.pid
JOIN product_categories pc ON p.pid = pc.pid
JOIN categories c ON pc.cat_id = c.cat_id
JOIN category_path cp ON c.cat_id = cp.cat_id
WHERE o.date >= CURRENT_DATE - INTERVAL '30 days'
WHERE o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
GROUP BY p.pid, p.title, c.name, cp.path
HAVING SUM(o.price * o.quantity) > 0
HAVING revenue > 0
ORDER BY profitMargin DESC
LIMIT 10
`);
@@ -178,7 +185,7 @@ router.get('/vendors', async (req, res) => {
console.log('Fetching vendor performance data...');
// First check if we have any vendors with sales
const { rows: [checkData] } = await pool.query(`
const [checkData] = await pool.query(`
SELECT COUNT(DISTINCT p.vendor) as vendor_count,
COUNT(DISTINCT o.order_number) as order_count
FROM products p
@@ -186,39 +193,39 @@ router.get('/vendors', async (req, res) => {
WHERE p.vendor IS NOT NULL
`);
console.log('Vendor data check:', checkData);
console.log('Vendor data check:', checkData[0]);
// Get vendor performance metrics
const { rows: performance } = await pool.query(`
const [performance] = await pool.query(`
WITH monthly_sales AS (
SELECT
p.vendor,
ROUND(SUM(CASE
WHEN o.date >= CURRENT_DATE - INTERVAL '30 days'
CAST(SUM(CASE
WHEN o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
THEN o.price * o.quantity
ELSE 0
END)::numeric, 3) as current_month,
ROUND(SUM(CASE
WHEN o.date >= CURRENT_DATE - INTERVAL '60 days'
AND o.date < CURRENT_DATE - INTERVAL '30 days'
END) AS DECIMAL(15,3)) as current_month,
CAST(SUM(CASE
WHEN o.date >= DATE_SUB(CURDATE(), INTERVAL 60 DAY)
AND o.date < DATE_SUB(CURDATE(), INTERVAL 30 DAY)
THEN o.price * o.quantity
ELSE 0
END)::numeric, 3) as previous_month
END) AS DECIMAL(15,3)) as previous_month
FROM products p
LEFT JOIN orders o ON p.pid = o.pid
WHERE p.vendor IS NOT NULL
AND o.date >= CURRENT_DATE - INTERVAL '60 days'
AND o.date >= DATE_SUB(CURDATE(), INTERVAL 60 DAY)
GROUP BY p.vendor
)
SELECT
p.vendor,
ROUND(SUM(o.price * o.quantity)::numeric, 3) as salesVolume,
CAST(SUM(o.price * o.quantity) AS DECIMAL(15,3)) as salesVolume,
COALESCE(ROUND(
(SUM(o.price * o.quantity - p.cost_price * o.quantity) /
NULLIF(SUM(o.price * o.quantity), 0) * 100)::numeric, 1
NULLIF(SUM(o.price * o.quantity), 0)) * 100, 1
), 0) as profitMargin,
COALESCE(ROUND(
(SUM(o.quantity) / NULLIF(AVG(p.stock_quantity), 0))::numeric, 1
SUM(o.quantity) / NULLIF(AVG(p.stock_quantity), 0), 1
), 0) as stockTurnover,
COUNT(DISTINCT p.pid) as productCount,
ROUND(
@@ -229,7 +236,7 @@ router.get('/vendors', async (req, res) => {
LEFT JOIN orders o ON p.pid = o.pid
LEFT JOIN monthly_sales ms ON p.vendor = ms.vendor
WHERE p.vendor IS NOT NULL
AND o.date >= CURRENT_DATE - INTERVAL '30 days'
AND o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
GROUP BY p.vendor, ms.current_month, ms.previous_month
ORDER BY salesVolume DESC
LIMIT 10
@@ -237,7 +244,45 @@ router.get('/vendors', async (req, res) => {
console.log('Performance data:', performance);
res.json({ performance });
// Get vendor comparison data
const [comparison] = await pool.query(`
SELECT
p.vendor,
CAST(COALESCE(ROUND(SUM(o.price * o.quantity) / NULLIF(COUNT(DISTINCT p.pid), 0), 2), 0) AS DECIMAL(15,3)) as salesPerProduct,
COALESCE(ROUND(AVG((o.price - p.cost_price) / NULLIF(o.price, 0) * 100), 1), 0) as averageMargin,
COUNT(DISTINCT p.pid) as size
FROM products p
LEFT JOIN orders o ON p.pid = o.pid AND o.date >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
WHERE p.vendor IS NOT NULL
GROUP BY p.vendor
ORDER BY salesPerProduct DESC
LIMIT 20
`);
console.log('Comparison data:', comparison);
// Get vendor sales trends
const [trends] = await pool.query(`
SELECT
p.vendor,
DATE_FORMAT(o.date, '%b %Y') as month,
CAST(COALESCE(SUM(o.price * o.quantity), 0) AS DECIMAL(15,3)) as sales
FROM products p
LEFT JOIN orders o ON p.pid = o.pid
WHERE p.vendor IS NOT NULL
AND o.date >= DATE_SUB(CURDATE(), INTERVAL 6 MONTH)
GROUP BY
p.vendor,
DATE_FORMAT(o.date, '%b %Y'),
DATE_FORMAT(o.date, '%Y-%m')
ORDER BY
p.vendor,
DATE_FORMAT(o.date, '%Y-%m')
`);
console.log('Trends data:', trends);
res.json({ performance, comparison, trends });
} catch (error) {
console.error('Error fetching vendor performance:', error);
res.status(500).json({ error: 'Failed to fetch vendor performance' });

28
inventory-server/src/routes/categories.js
View File

@@ -6,7 +6,7 @@ router.get('/', async (req, res) => {
const pool = req.app.locals.pool;
try {
// Get all categories with metrics and hierarchy info
const { rows: categories } = await pool.query(`
const [categories] = await pool.query(`
SELECT
c.cat_id,
c.name,
@@ -18,7 +18,7 @@ router.get('/', async (req, res) => {
p.type as parent_type,
COALESCE(cm.product_count, 0) as product_count,
COALESCE(cm.active_products, 0) as active_products,
ROUND(COALESCE(cm.total_value, 0)::numeric, 3) as total_value,
CAST(COALESCE(cm.total_value, 0) AS DECIMAL(15,3)) as total_value,
COALESCE(cm.avg_margin, 0) as avg_margin,
COALESCE(cm.turnover_rate, 0) as turnover_rate,
COALESCE(cm.growth_rate, 0) as growth_rate
@@ -39,22 +39,22 @@ router.get('/', async (req, res) => {
`);
// Get overall stats
const { rows: [stats] } = await pool.query(`
const [stats] = await pool.query(`
SELECT
COUNT(DISTINCT c.cat_id) as totalCategories,
COUNT(DISTINCT CASE WHEN c.status = 'active' THEN c.cat_id END) as activeCategories,
ROUND(COALESCE(SUM(cm.total_value), 0)::numeric, 3) as totalValue,
COALESCE(ROUND(AVG(NULLIF(cm.avg_margin, 0))::numeric, 1), 0) as avgMargin,
COALESCE(ROUND(AVG(NULLIF(cm.growth_rate, 0))::numeric, 1), 0) as avgGrowth
CAST(COALESCE(SUM(cm.total_value), 0) AS DECIMAL(15,3)) as totalValue,
COALESCE(ROUND(AVG(NULLIF(cm.avg_margin, 0)), 1), 0) as avgMargin,
COALESCE(ROUND(AVG(NULLIF(cm.growth_rate, 0)), 1), 0) as avgGrowth
FROM categories c
LEFT JOIN category_metrics cm ON c.cat_id = cm.category_id
`);
// Get type counts for filtering
const { rows: typeCounts } = await pool.query(`
const [typeCounts] = await pool.query(`
SELECT
type,
COUNT(*)::integer as count
COUNT(*) as count
FROM categories
GROUP BY type
ORDER BY type
@@ -81,14 +81,14 @@ router.get('/', async (req, res) => {
})),
typeCounts: typeCounts.map(tc => ({
type: tc.type,
count: tc.count // Already cast to integer in the query
count: parseInt(tc.count)
})),
stats: {
totalCategories: parseInt(stats.totalcategories),
activeCategories: parseInt(stats.activecategories),
totalValue: parseFloat(stats.totalvalue),
avgMargin: parseFloat(stats.avgmargin),
avgGrowth: parseFloat(stats.avggrowth)
totalCategories: parseInt(stats[0].totalCategories),
activeCategories: parseInt(stats[0].activeCategories),
totalValue: parseFloat(stats[0].totalValue),
avgMargin: parseFloat(stats[0].avgMargin),
avgGrowth: parseFloat(stats[0].avgGrowth)
}
});
} catch (error) {

72
inventory-server/src/routes/config.js
View File

@@ -13,22 +13,22 @@ router.get('/', async (req, res) => {
try {
console.log('[Config Route] Fetching configuration values...');
const { rows: stockThresholds } = await pool.query('SELECT * FROM stock_thresholds WHERE id = 1');
const [stockThresholds] = await pool.query('SELECT * FROM stock_thresholds WHERE id = 1');
console.log('[Config Route] Stock thresholds:', stockThresholds);
const { rows: leadTimeThresholds } = await pool.query('SELECT * FROM lead_time_thresholds WHERE id = 1');
const [leadTimeThresholds] = await pool.query('SELECT * FROM lead_time_thresholds WHERE id = 1');
console.log('[Config Route] Lead time thresholds:', leadTimeThresholds);
const { rows: salesVelocityConfig } = await pool.query('SELECT * FROM sales_velocity_config WHERE id = 1');
const [salesVelocityConfig] = await pool.query('SELECT * FROM sales_velocity_config WHERE id = 1');
console.log('[Config Route] Sales velocity config:', salesVelocityConfig);
const { rows: abcConfig } = await pool.query('SELECT * FROM abc_classification_config WHERE id = 1');
const [abcConfig] = await pool.query('SELECT * FROM abc_classification_config WHERE id = 1');
console.log('[Config Route] ABC config:', abcConfig);
const { rows: safetyStockConfig } = await pool.query('SELECT * FROM safety_stock_config WHERE id = 1');
const [safetyStockConfig] = await pool.query('SELECT * FROM safety_stock_config WHERE id = 1');
console.log('[Config Route] Safety stock config:', safetyStockConfig);
const { rows: turnoverConfig } = await pool.query('SELECT * FROM turnover_config WHERE id = 1');
const [turnoverConfig] = await pool.query('SELECT * FROM turnover_config WHERE id = 1');
console.log('[Config Route] Turnover config:', turnoverConfig);
const response = {
@@ -53,14 +53,14 @@ router.put('/stock-thresholds/:id', async (req, res) => {
const pool = req.app.locals.pool;
try {
const { critical_days, reorder_days, overstock_days, low_stock_threshold, min_reorder_quantity } = req.body;
const { rows } = await pool.query(
const [result] = await pool.query(
`UPDATE stock_thresholds
SET critical_days = $1,
reorder_days = $2,
overstock_days = $3,
low_stock_threshold = $4,
min_reorder_quantity = $5
WHERE id = $6`,
SET critical_days = ?,
reorder_days = ?,
overstock_days = ?,
low_stock_threshold = ?,
min_reorder_quantity = ?
WHERE id = ?`,
[critical_days, reorder_days, overstock_days, low_stock_threshold, min_reorder_quantity, req.params.id]
);
res.json({ success: true });
@@ -75,12 +75,12 @@ router.put('/lead-time-thresholds/:id', async (req, res) => {
const pool = req.app.locals.pool;
try {
const { target_days, warning_days, critical_days } = req.body;
const { rows } = await pool.query(
const [result] = await pool.query(
`UPDATE lead_time_thresholds
SET target_days = $1,
warning_days = $2,
critical_days = $3
WHERE id = $4`,
SET target_days = ?,
warning_days = ?,
critical_days = ?
WHERE id = ?`,
[target_days, warning_days, critical_days, req.params.id]
);
res.json({ success: true });
@@ -95,12 +95,12 @@ router.put('/sales-velocity/:id', async (req, res) => {
const pool = req.app.locals.pool;
try {
const { daily_window_days, weekly_window_days, monthly_window_days } = req.body;
const { rows } = await pool.query(
const [result] = await pool.query(
`UPDATE sales_velocity_config
SET daily_window_days = $1,
weekly_window_days = $2,
monthly_window_days = $3
WHERE id = $4`,
SET daily_window_days = ?,
weekly_window_days = ?,
monthly_window_days = ?
WHERE id = ?`,
[daily_window_days, weekly_window_days, monthly_window_days, req.params.id]
);
res.json({ success: true });
@@ -115,12 +115,12 @@ router.put('/abc-classification/:id', async (req, res) => {
const pool = req.app.locals.pool;
try {
const { a_threshold, b_threshold, classification_period_days } = req.body;
const { rows } = await pool.query(
const [result] = await pool.query(
`UPDATE abc_classification_config
SET a_threshold = $1,
b_threshold = $2,
classification_period_days = $3
WHERE id = $4`,
SET a_threshold = ?,
b_threshold = ?,
classification_period_days = ?
WHERE id = ?`,
[a_threshold, b_threshold, classification_period_days, req.params.id]
);
res.json({ success: true });
@@ -135,11 +135,11 @@ router.put('/safety-stock/:id', async (req, res) => {
const pool = req.app.locals.pool;
try {
const { coverage_days, service_level } = req.body;
const { rows } = await pool.query(
const [result] = await pool.query(
`UPDATE safety_stock_config
SET coverage_days = $1,
service_level = $2
WHERE id = $3`,
SET coverage_days = ?,
service_level = ?
WHERE id = ?`,
[coverage_days, service_level, req.params.id]
);
res.json({ success: true });
@@ -154,11 +154,11 @@ router.put('/turnover/:id', async (req, res) => {
const pool = req.app.locals.pool;
try {
const { calculation_period_days, target_rate } = req.body;
const { rows } = await pool.query(
const [result] = await pool.query(
`UPDATE turnover_config
SET calculation_period_days = $1,
target_rate = $2
WHERE id = $3`,
SET calculation_period_days = ?,
target_rate = ?
WHERE id = ?`,
[calculation_period_days, target_rate, req.params.id]
);
res.json({ success: true });

443
inventory-server/src/routes/csv.js
View File

@@ -2,7 +2,6 @@ const express = require('express');
const router = express.Router();
const { spawn } = require('child_process');
const path = require('path');
const db = require('../utils/db');
// Debug middleware MUST be first
router.use((req, res, next) => {
@@ -10,11 +9,9 @@ router.use((req, res, next) => {
next();
});
// Store active processes and their progress
// Store active import process and its progress
let activeImport = null;
let importProgress = null;
let activeFullUpdate = null;
let activeFullReset = null;
// SSE clients for progress updates
const updateClients = new Set();
@@ -22,16 +19,17 @@ const importClients = new Set();
const resetClients = new Set();
const resetMetricsClients = new Set();
const calculateMetricsClients = new Set();
const fullUpdateClients = new Set();
const fullResetClients = new Set();
// Helper to send progress to specific clients
function sendProgressToClients(clients, data) {
// If data is a string, send it directly
// If it's an object, convert it to JSON
const message = typeof data === 'string'
? `data: ${data}\n\n`
: `data: ${JSON.stringify(data)}\n\n`;
function sendProgressToClients(clients, progress) {
const data = typeof progress === 'string' ? { progress } : progress;
// Ensure we have a status field
if (!data.status) {
data.status = 'running';
}
const message = `data: ${JSON.stringify(data)}\n\n`;
clients.forEach(client => {
try {
@@ -47,128 +45,8 @@ function sendProgressToClients(clients, data) {
});
}
// Helper to run a script and stream progress
function runScript(scriptPath, type, clients) {
return new Promise((resolve, reject) => {
// Kill any existing process of this type
let activeProcess;
switch (type) {
case 'update':
if (activeFullUpdate) {
try { activeFullUpdate.kill(); } catch (e) { }
}
activeProcess = activeFullUpdate;
break;
case 'reset':
if (activeFullReset) {
try { activeFullReset.kill(); } catch (e) { }
}
activeProcess = activeFullReset;
break;
}
const child = spawn('node', [scriptPath], {
stdio: ['inherit', 'pipe', 'pipe']
});
switch (type) {
case 'update':
activeFullUpdate = child;
break;
case 'reset':
activeFullReset = child;
break;
}
let output = '';
child.stdout.on('data', (data) => {
const text = data.toString();
output += text;
// Split by lines to handle multiple JSON outputs
const lines = text.split('\n');
lines.filter(line => line.trim()).forEach(line => {
try {
// Try to parse as JSON but don't let it affect the display
const jsonData = JSON.parse(line);
// Only end the process if we get a final status
if (jsonData.status === 'complete' || jsonData.status === 'error' || jsonData.status === 'cancelled') {
if (jsonData.status === 'complete' && !jsonData.operation?.includes('complete')) {
// Don't close for intermediate completion messages
sendProgressToClients(clients, line);
return;
}
// Close only on final completion/error/cancellation
switch (type) {
case 'update':
activeFullUpdate = null;
break;
case 'reset':
activeFullReset = null;
break;
}
if (jsonData.status === 'error') {
reject(new Error(jsonData.error || 'Unknown error'));
} else {
resolve({ output });
}
}
} catch (e) {
// Not JSON, just display as is
}
// Always send the raw line
sendProgressToClients(clients, line);
});
});
child.stderr.on('data', (data) => {
const text = data.toString();
console.error(text);
// Send stderr output directly too
sendProgressToClients(clients, text);
});
child.on('close', (code) => {
switch (type) {
case 'update':
activeFullUpdate = null;
break;
case 'reset':
activeFullReset = null;
break;
}
if (code !== 0) {
const error = `Script ${scriptPath} exited with code ${code}`;
sendProgressToClients(clients, error);
reject(new Error(error));
}
// Don't resolve here - let the completion message from the script trigger the resolve
});
child.on('error', (err) => {
switch (type) {
case 'update':
activeFullUpdate = null;
break;
case 'reset':
activeFullReset = null;
break;
}
sendProgressToClients(clients, err.message);
reject(err);
});
});
}
// Progress endpoints
router.get('/:type/progress', (req, res) => {
const { type } = req.params;
if (!['update', 'reset'].includes(type)) {
return res.status(400).json({ error: 'Invalid operation type' });
}
router.get('/update/progress', (req, res) => {
res.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
@@ -177,19 +55,105 @@ router.get('/:type/progress', (req, res) => {
'Access-Control-Allow-Credentials': 'true'
});
// Add this client to the correct set
const clients = type === 'update' ? fullUpdateClients : fullResetClients;
clients.add(res);
// Send an initial message to test the connection
res.write('data: {"status":"running","operation":"Initializing connection..."}\n\n');
// Send initial connection message
sendProgressToClients(new Set([res]), JSON.stringify({
status: 'running',
operation: 'Initializing connection...'
}));
// Add this client to the update set
updateClients.add(res);
// Handle client disconnect
// Remove client when connection closes
req.on('close', () => {
clients.delete(res);
updateClients.delete(res);
});
});
router.get('/import/progress', (req, res) => {
res.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'Access-Control-Allow-Origin': req.headers.origin || '*',
'Access-Control-Allow-Credentials': 'true'
});
// Send an initial message to test the connection
res.write('data: {"status":"running","operation":"Initializing connection..."}\n\n');
// Add this client to the import set
importClients.add(res);
// Remove client when connection closes
req.on('close', () => {
importClients.delete(res);
});
});
router.get('/reset/progress', (req, res) => {
res.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'Access-Control-Allow-Origin': req.headers.origin || '*',
'Access-Control-Allow-Credentials': 'true'
});
// Send an initial message to test the connection
res.write('data: {"status":"running","operation":"Initializing connection..."}\n\n');
// Add this client to the reset set
resetClients.add(res);
// Remove client when connection closes
req.on('close', () => {
resetClients.delete(res);
});
});
// Add reset-metrics progress endpoint
router.get('/reset-metrics/progress', (req, res) => {
res.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'Access-Control-Allow-Origin': req.headers.origin || '*',
'Access-Control-Allow-Credentials': 'true'
});
// Send an initial message to test the connection
res.write('data: {"status":"running","operation":"Initializing connection..."}\n\n');
// Add this client to the reset-metrics set
resetMetricsClients.add(res);
// Remove client when connection closes
req.on('close', () => {
resetMetricsClients.delete(res);
});
});
// Add calculate-metrics progress endpoint
router.get('/calculate-metrics/progress', (req, res) => {
res.writeHead(200, {
'Content-Type': 'text/event-stream',
'Cache-Control': 'no-cache',
'Connection': 'keep-alive',
'Access-Control-Allow-Origin': req.headers.origin || '*',
'Access-Control-Allow-Credentials': 'true'
});
// Send current progress if it exists
if (importProgress) {
res.write(`data: ${JSON.stringify(importProgress)}\n\n`);
} else {
res.write('data: {"status":"running","operation":"Initializing connection..."}\n\n');
}
// Add this client to the calculate-metrics set
calculateMetricsClients.add(res);
// Remove client when connection closes
req.on('close', () => {
calculateMetricsClients.delete(res);
});
});
@@ -210,6 +174,7 @@ router.get('/status', (req, res) => {
// Add calculate-metrics status endpoint
router.get('/calculate-metrics/status', (req, res) => {
console.log('Calculate metrics status endpoint hit');
const calculateMetrics = require('../../scripts/calculate-metrics');
const progress = calculateMetrics.getProgress();
@@ -406,35 +371,49 @@ router.post('/import', async (req, res) => {
// Route to cancel active process
router.post('/cancel', (req, res) => {
let killed = false;
// Get the operation type from the request
const { type } = req.query;
const clients = type === 'update' ? fullUpdateClients : fullResetClients;
const activeProcess = type === 'update' ? activeFullUpdate : activeFullReset;
if (activeProcess) {
try {
activeProcess.kill('SIGTERM');
if (type === 'update') {
activeFullUpdate = null;
} else {
activeFullReset = null;
}
killed = true;
sendProgressToClients(clients, JSON.stringify({
status: 'cancelled',
operation: 'Operation cancelled'
}));
} catch (err) {
console.error(`Error killing ${type} process:`, err);
}
if (!activeImport) {
return res.status(404).json({ error: 'No active process to cancel' });
}
if (killed) {
try {
// If it's the prod import module, call its cancel function
if (typeof activeImport.cancelImport === 'function') {
activeImport.cancelImport();
} else {
// Otherwise it's a child process
activeImport.kill('SIGTERM');
}
// Get the operation type from the request
const { operation } = req.query;
// Send cancel message only to the appropriate client set
const cancelMessage = {
status: 'cancelled',
operation: 'Operation cancelled'
};
switch (operation) {
case 'update':
sendProgressToClients(updateClients, cancelMessage);
break;
case 'import':
sendProgressToClients(importClients, cancelMessage);
break;
case 'reset':
sendProgressToClients(resetClients, cancelMessage);
break;
case 'calculate-metrics':
sendProgressToClients(calculateMetricsClients, cancelMessage);
break;
}
res.json({ success: true });
} else {
res.status(404).json({ error: 'No active process to cancel' });
} catch (error) {
// Even if there's an error, try to clean up
activeImport = null;
importProgress = null;
res.status(500).json({ error: 'Failed to cancel process' });
}
});
@@ -573,6 +552,20 @@ router.post('/reset-metrics', async (req, res) => {
}
});
// Add calculate-metrics status endpoint
router.get('/calculate-metrics/status', (req, res) => {
const calculateMetrics = require('../../scripts/calculate-metrics');
const progress = calculateMetrics.getProgress();
// Only consider it active if both the process is running and we have progress
const isActive = !!activeImport && !!progress;
res.json({
active: isActive,
progress: isActive ? progress : null
});
});
// Add calculate-metrics endpoint
router.post('/calculate-metrics', async (req, res) => {
if (activeImport) {
@@ -718,116 +711,4 @@ router.post('/import-from-prod', async (req, res) => {
}
});
// POST /csv/full-update - Run full update script
router.post('/full-update', async (req, res) => {
try {
const scriptPath = path.join(__dirname, '../../scripts/full-update.js');
runScript(scriptPath, 'update', fullUpdateClients)
.catch(error => {
console.error('Update failed:', error);
});
res.status(202).json({ message: 'Update started' });
} catch (error) {
res.status(500).json({ error: error.message });
}
});
// POST /csv/full-reset - Run full reset script
router.post('/full-reset', async (req, res) => {
try {
const scriptPath = path.join(__dirname, '../../scripts/full-reset.js');
runScript(scriptPath, 'reset', fullResetClients)
.catch(error => {
console.error('Reset failed:', error);
});
res.status(202).json({ message: 'Reset started' });
} catch (error) {
res.status(500).json({ error: error.message });
}
});
// GET /history/import - Get recent import history
router.get('/history/import', async (req, res) => {
try {
const pool = req.app.locals.pool;
const { rows } = await pool.query(`
SELECT
id,
start_time,
end_time,
status,
error_message,
rows_processed::integer,
files_processed::integer
FROM import_history
ORDER BY start_time DESC
LIMIT 20
`);
res.json(rows || []);
} catch (error) {
console.error('Error fetching import history:', error);
res.status(500).json({ error: error.message });
}
});
// GET /history/calculate - Get recent calculation history
router.get('/history/calculate', async (req, res) => {
try {
const pool = req.app.locals.pool;
const { rows } = await pool.query(`
SELECT
id,
start_time,
end_time,
status,
error_message,
modules_processed::integer,
total_modules::integer
FROM calculate_history
ORDER BY start_time DESC
LIMIT 20
`);
res.json(rows || []);
} catch (error) {
console.error('Error fetching calculate history:', error);
res.status(500).json({ error: error.message });
}
});
// GET /status/modules - Get module calculation status
router.get('/status/modules', async (req, res) => {
try {
const pool = req.app.locals.pool;
const { rows } = await pool.query(`
SELECT
module_name,
last_calculation_timestamp::timestamp
FROM calculate_status
ORDER BY module_name
`);
res.json(rows || []);
} catch (error) {
console.error('Error fetching module status:', error);
res.status(500).json({ error: error.message });
}
});
// GET /status/tables - Get table sync status
router.get('/status/tables', async (req, res) => {
try {
const pool = req.app.locals.pool;
const { rows } = await pool.query(`
SELECT
table_name,
last_sync_timestamp::timestamp
FROM sync_status
ORDER BY table_name
`);
res.json(rows || []);
} catch (error) {
console.error('Error fetching table status:', error);
res.status(500).json({ error: error.message });
}
});
module.exports = router;

116
inventory-server/src/routes/dashboard.js
View File

@@ -19,15 +19,16 @@ async function executeQuery(sql, params = []) {
router.get('/stock/metrics', async (req, res) => {
try {
// Get stock metrics
const { rows: [stockMetrics] } = await executeQuery(`
const [rows] = await executeQuery(`
SELECT
COALESCE(COUNT(*), 0)::integer as total_products,
COALESCE(COUNT(CASE WHEN stock_quantity > 0 THEN 1 END), 0)::integer as products_in_stock,
COALESCE(SUM(CASE WHEN stock_quantity > 0 THEN stock_quantity END), 0)::integer as total_units,
ROUND(COALESCE(SUM(CASE WHEN stock_quantity > 0 THEN stock_quantity * cost_price END), 0)::numeric, 3) as total_cost,
ROUND(COALESCE(SUM(CASE WHEN stock_quantity > 0 THEN stock_quantity * price END), 0)::numeric, 3) as total_retail
COALESCE(COUNT(*), 0) as total_products,
COALESCE(COUNT(CASE WHEN stock_quantity > 0 THEN 1 END), 0) as products_in_stock,
COALESCE(SUM(CASE WHEN stock_quantity > 0 THEN stock_quantity END), 0) as total_units,
COALESCE(SUM(CASE WHEN stock_quantity > 0 THEN stock_quantity * cost_price END), 0) as total_cost,
COALESCE(SUM(CASE WHEN stock_quantity > 0 THEN stock_quantity * price END), 0) as total_retail
FROM products
`);
const stockMetrics = rows[0];
console.log('Raw stockMetrics from database:', stockMetrics);
console.log('stockMetrics.total_products:', stockMetrics.total_products);
@@ -37,26 +38,26 @@ router.get('/stock/metrics', async (req, res) => {
console.log('stockMetrics.total_retail:', stockMetrics.total_retail);
// Get brand stock values with Other category
const { rows: brandValues } = await executeQuery(`
const [brandValues] = await executeQuery(`
WITH brand_totals AS (
SELECT
COALESCE(brand, 'Unbranded') as brand,
COUNT(DISTINCT pid)::integer as variant_count,
COALESCE(SUM(stock_quantity), 0)::integer as stock_units,
ROUND(COALESCE(SUM(stock_quantity * cost_price), 0)::numeric, 3) as stock_cost,
ROUND(COALESCE(SUM(stock_quantity * price), 0)::numeric, 3) as stock_retail
COUNT(DISTINCT pid) as variant_count,
COALESCE(SUM(stock_quantity), 0) as stock_units,
CAST(COALESCE(SUM(stock_quantity * cost_price), 0) AS DECIMAL(15,3)) as stock_cost,
CAST(COALESCE(SUM(stock_quantity * price), 0) AS DECIMAL(15,3)) as stock_retail
FROM products
WHERE stock_quantity > 0
GROUP BY COALESCE(brand, 'Unbranded')
HAVING ROUND(COALESCE(SUM(stock_quantity * cost_price), 0)::numeric, 3) > 0
HAVING stock_cost > 0
),
other_brands AS (
SELECT
'Other' as brand,
SUM(variant_count)::integer as variant_count,
SUM(stock_units)::integer as stock_units,
ROUND(SUM(stock_cost)::numeric, 3) as stock_cost,
ROUND(SUM(stock_retail)::numeric, 3) as stock_retail
SUM(variant_count) as variant_count,
SUM(stock_units) as stock_units,
CAST(SUM(stock_cost) AS DECIMAL(15,3)) as stock_cost,
CAST(SUM(stock_retail) AS DECIMAL(15,3)) as stock_retail
FROM brand_totals
WHERE stock_cost <= 5000
),
@@ -100,50 +101,51 @@ router.get('/stock/metrics', async (req, res) => {
// Returns purchase order metrics by vendor
router.get('/purchase/metrics', async (req, res) => {
try {
const { rows: [poMetrics] } = await executeQuery(`
const [rows] = await executeQuery(`
SELECT
COALESCE(COUNT(DISTINCT CASE
WHEN po.receiving_status < $1
WHEN po.receiving_status < ${ReceivingStatus.PartialReceived}
THEN po.po_id
END), 0)::integer as active_pos,
END), 0) as active_pos,
COALESCE(COUNT(DISTINCT CASE
WHEN po.receiving_status < $1
AND po.expected_date < CURRENT_DATE
WHEN po.receiving_status < ${ReceivingStatus.PartialReceived}
AND po.expected_date < CURDATE()
THEN po.po_id
END), 0)::integer as overdue_pos,
END), 0) as overdue_pos,
COALESCE(SUM(CASE
WHEN po.receiving_status < $1
WHEN po.receiving_status < ${ReceivingStatus.PartialReceived}
THEN po.ordered
ELSE 0
END), 0)::integer as total_units,
ROUND(COALESCE(SUM(CASE
WHEN po.receiving_status < $1
END), 0) as total_units,
CAST(COALESCE(SUM(CASE
WHEN po.receiving_status < ${ReceivingStatus.PartialReceived}
THEN po.ordered * po.cost_price
ELSE 0
END), 0)::numeric, 3) as total_cost,
ROUND(COALESCE(SUM(CASE
WHEN po.receiving_status < $1
END), 0) AS DECIMAL(15,3)) as total_cost,
CAST(COALESCE(SUM(CASE
WHEN po.receiving_status < ${ReceivingStatus.PartialReceived}
THEN po.ordered * p.price
ELSE 0
END), 0)::numeric, 3) as total_retail
END), 0) AS DECIMAL(15,3)) as total_retail
FROM purchase_orders po
JOIN products p ON po.pid = p.pid
`, [ReceivingStatus.PartialReceived]);
`);
const poMetrics = rows[0];
const { rows: vendorOrders } = await executeQuery(`
const [vendorOrders] = await executeQuery(`
SELECT
po.vendor,
COUNT(DISTINCT po.po_id)::integer as orders,
COALESCE(SUM(po.ordered), 0)::integer as units,
ROUND(COALESCE(SUM(po.ordered * po.cost_price), 0)::numeric, 3) as cost,
ROUND(COALESCE(SUM(po.ordered * p.price), 0)::numeric, 3) as retail
COUNT(DISTINCT po.po_id) as orders,
COALESCE(SUM(po.ordered), 0) as units,
CAST(COALESCE(SUM(po.ordered * po.cost_price), 0) AS DECIMAL(15,3)) as cost,
CAST(COALESCE(SUM(po.ordered * p.price), 0) AS DECIMAL(15,3)) as retail
FROM purchase_orders po
JOIN products p ON po.pid = p.pid
WHERE po.receiving_status < $1
WHERE po.receiving_status < ${ReceivingStatus.PartialReceived}
GROUP BY po.vendor
HAVING ROUND(COALESCE(SUM(po.ordered * po.cost_price), 0)::numeric, 3) > 0
HAVING cost > 0
ORDER BY cost DESC
`, [ReceivingStatus.PartialReceived]);
`);
// Format response to match PurchaseMetricsData interface
const response = {
@@ -173,21 +175,21 @@ router.get('/purchase/metrics', async (req, res) => {
router.get('/replenishment/metrics', async (req, res) => {
try {
// Get summary metrics
const { rows: [metrics] } = await executeQuery(`
const [metrics] = await executeQuery(`
SELECT
COUNT(DISTINCT p.pid)::integer as products_to_replenish,
COUNT(DISTINCT p.pid) as products_to_replenish,
COALESCE(SUM(CASE
WHEN p.stock_quantity < 0 THEN ABS(p.stock_quantity) + pm.reorder_qty
ELSE pm.reorder_qty
END), 0)::integer as total_units_needed,
ROUND(COALESCE(SUM(CASE
END), 0) as total_units_needed,
CAST(COALESCE(SUM(CASE
WHEN p.stock_quantity < 0 THEN (ABS(p.stock_quantity) + pm.reorder_qty) * p.cost_price
ELSE pm.reorder_qty * p.cost_price
END), 0)::numeric, 3) as total_cost,
ROUND(COALESCE(SUM(CASE
END), 0) AS DECIMAL(15,3)) as total_cost,
CAST(COALESCE(SUM(CASE
WHEN p.stock_quantity < 0 THEN (ABS(p.stock_quantity) + pm.reorder_qty) * p.price
ELSE pm.reorder_qty * p.price
END), 0)::numeric, 3) as total_retail
END), 0) AS DECIMAL(15,3)) as total_retail
FROM products p
JOIN product_metrics pm ON p.pid = pm.pid
WHERE p.replenishable = true
@@ -197,23 +199,23 @@ router.get('/replenishment/metrics', async (req, res) => {
`);
// Get top variants to replenish
const { rows: variants } = await executeQuery(`
const [variants] = await executeQuery(`
SELECT
p.pid,
p.title,
p.stock_quantity::integer as current_stock,
p.stock_quantity as current_stock,
CASE
WHEN p.stock_quantity < 0 THEN ABS(p.stock_quantity) + pm.reorder_qty
ELSE pm.reorder_qty
END::integer as replenish_qty,
ROUND(CASE
END as replenish_qty,
CAST(CASE
WHEN p.stock_quantity < 0 THEN (ABS(p.stock_quantity) + pm.reorder_qty) * p.cost_price
ELSE pm.reorder_qty * p.cost_price
END::numeric, 3) as replenish_cost,
ROUND(CASE
END AS DECIMAL(15,3)) as replenish_cost,
CAST(CASE
WHEN p.stock_quantity < 0 THEN (ABS(p.stock_quantity) + pm.reorder_qty) * p.price
ELSE pm.reorder_qty * p.price
END::numeric, 3) as replenish_retail,
END AS DECIMAL(15,3)) as replenish_retail,
pm.stock_status
FROM products p
JOIN product_metrics pm ON p.pid = pm.pid
@@ -232,10 +234,10 @@ router.get('/replenishment/metrics', async (req, res) => {
// Format response
const response = {
productsToReplenish: parseInt(metrics.products_to_replenish) || 0,
unitsToReplenish: parseInt(metrics.total_units_needed) || 0,
replenishmentCost: parseFloat(metrics.total_cost) || 0,
replenishmentRetail: parseFloat(metrics.total_retail) || 0,
productsToReplenish: parseInt(metrics[0].products_to_replenish) || 0,
unitsToReplenish: parseInt(metrics[0].total_units_needed) || 0,
replenishmentCost: parseFloat(metrics[0].total_cost) || 0,
replenishmentRetail: parseFloat(metrics[0].total_retail) || 0,
topVariants: variants.map(v => ({
id: v.pid,
title: v.title,

1117
inventory-server/src/routes/import.js
View File

File diff suppressed because it is too large Load Diff

18
inventory-server/src/routes/metrics.js
View File

@@ -5,28 +5,26 @@ const router = express.Router();
router.get('/trends', async (req, res) => {
const pool = req.app.locals.pool;
try {
const { rows } = await pool.query(`
const [rows] = await pool.query(`
WITH MonthlyMetrics AS (
SELECT
make_date(pta.year, pta.month, 1) as date,
ROUND(COALESCE(SUM(pta.total_revenue), 0)::numeric, 3) as revenue,
ROUND(COALESCE(SUM(pta.total_cost), 0)::numeric, 3) as cost,
ROUND(COALESCE(SUM(pm.inventory_value), 0)::numeric, 3) as inventory_value,
DATE(CONCAT(pta.year, '-', LPAD(pta.month, 2, '0'), '-01')) as date,
CAST(COALESCE(SUM(pta.total_revenue), 0) AS DECIMAL(15,3)) as revenue,
CAST(COALESCE(SUM(pta.total_cost), 0) AS DECIMAL(15,3)) as cost,
CAST(COALESCE(SUM(pm.inventory_value), 0) AS DECIMAL(15,3)) as inventory_value,
CASE
WHEN SUM(pm.inventory_value) > 0
THEN ROUND((SUM(pta.total_revenue - pta.total_cost) / SUM(pm.inventory_value) * 100)::numeric, 3)
THEN CAST((SUM(pta.total_revenue - pta.total_cost) / SUM(pm.inventory_value)) * 100 AS DECIMAL(15,3))
ELSE 0
END as gmroi
FROM product_time_aggregates pta
JOIN product_metrics pm ON pta.pid = pm.pid
WHERE (pta.year * 100 + pta.month) >=
EXTRACT(YEAR FROM CURRENT_DATE - INTERVAL '12 months')::integer * 100 +
EXTRACT(MONTH FROM CURRENT_DATE - INTERVAL '12 months')::integer
WHERE (pta.year * 100 + pta.month) >= DATE_FORMAT(DATE_SUB(CURDATE(), INTERVAL 12 MONTH), '%Y%m')
GROUP BY pta.year, pta.month
ORDER BY date ASC
)
SELECT
to_char(date, 'Mon YY') as date,
DATE_FORMAT(date, '%b %y') as date,
revenue,
inventory_value,
gmroi

68
inventory-server/src/routes/orders.js
View File

@@ -20,46 +20,39 @@ router.get('/', async (req, res) => {
// Build the WHERE clause
const conditions = ['o1.canceled = false'];
const params = [];
let paramCounter = 1;
if (search) {
conditions.push(`(o1.order_number ILIKE $${paramCounter} OR o1.customer ILIKE $${paramCounter})`);
params.push(`%${search}%`);
paramCounter++;
conditions.push('(o1.order_number LIKE ? OR o1.customer LIKE ?)');
params.push(`%${search}%`, `%${search}%`);
}
if (status !== 'all') {
conditions.push(`o1.status = $${paramCounter}`);
conditions.push('o1.status = ?');
params.push(status);
paramCounter++;
}
if (fromDate) {
conditions.push(`DATE(o1.date) >= DATE($${paramCounter})`);
conditions.push('DATE(o1.date) >= DATE(?)');
params.push(fromDate.toISOString());
paramCounter++;
}
if (toDate) {
conditions.push(`DATE(o1.date) <= DATE($${paramCounter})`);
conditions.push('DATE(o1.date) <= DATE(?)');
params.push(toDate.toISOString());
paramCounter++;
}
if (minAmount > 0) {
conditions.push(`total_amount >= $${paramCounter}`);
conditions.push('total_amount >= ?');
params.push(minAmount);
paramCounter++;
}
if (maxAmount) {
conditions.push(`total_amount <= $${paramCounter}`);
conditions.push('total_amount <= ?');
params.push(maxAmount);
paramCounter++;
}
// Get total count for pagination
const { rows: [countResult] } = await pool.query(`
const [countResult] = await pool.query(`
SELECT COUNT(DISTINCT o1.order_number) as total
FROM orders o1
LEFT JOIN (
@@ -70,7 +63,7 @@ router.get('/', async (req, res) => {
WHERE ${conditions.join(' AND ')}
`, params);
const total = countResult.total;
const total = countResult[0].total;
// Get paginated results
const query = `
@@ -82,7 +75,7 @@ router.get('/', async (req, res) => {
o1.payment_method,
o1.shipping_method,
COUNT(o2.pid) as items_count,
ROUND(SUM(o2.price * o2.quantity)::numeric, 3) as total_amount
CAST(SUM(o2.price * o2.quantity) AS DECIMAL(15,3)) as total_amount
FROM orders o1
JOIN orders o2 ON o1.order_number = o2.order_number
WHERE ${conditions.join(' AND ')}
@@ -98,37 +91,36 @@ router.get('/', async (req, res) => {
? `${sortColumn} ${sortDirection}`
: `o1.${sortColumn} ${sortDirection}`
}
LIMIT $${paramCounter} OFFSET $${paramCounter + 1}
LIMIT ? OFFSET ?
`;
params.push(limit, offset);
const { rows } = await pool.query(query, params);
const [rows] = await pool.query(query, [...params, limit, offset]);
// Get order statistics
const { rows: [orderStats] } = await pool.query(`
const [stats] = await pool.query(`
WITH CurrentStats AS (
SELECT
COUNT(DISTINCT order_number) as total_orders,
ROUND(SUM(price * quantity)::numeric, 3) as total_revenue
CAST(SUM(price * quantity) AS DECIMAL(15,3)) as total_revenue
FROM orders
WHERE canceled = false
AND DATE(date) >= CURRENT_DATE - INTERVAL '30 days'
AND DATE(date) >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
),
PreviousStats AS (
SELECT
COUNT(DISTINCT order_number) as prev_orders,
ROUND(SUM(price * quantity)::numeric, 3) as prev_revenue
CAST(SUM(price * quantity) AS DECIMAL(15,3)) as prev_revenue
FROM orders
WHERE canceled = false
AND DATE(date) BETWEEN CURRENT_DATE - INTERVAL '60 days' AND CURRENT_DATE - INTERVAL '30 days'
AND DATE(date) BETWEEN DATE_SUB(CURDATE(), INTERVAL 60 DAY) AND DATE_SUB(CURDATE(), INTERVAL 30 DAY)
),
OrderValues AS (
SELECT
order_number,
ROUND(SUM(price * quantity)::numeric, 3) as order_value
CAST(SUM(price * quantity) AS DECIMAL(15,3)) as order_value
FROM orders
WHERE canceled = false
AND DATE(date) >= CURRENT_DATE - INTERVAL '30 days'
AND DATE(date) >= DATE_SUB(CURDATE(), INTERVAL 30 DAY)
GROUP BY order_number
)
SELECT
@@ -136,27 +128,29 @@ router.get('/', async (req, res) => {
cs.total_revenue,
CASE
WHEN ps.prev_orders > 0
THEN ROUND(((cs.total_orders - ps.prev_orders)::numeric / ps.prev_orders * 100), 1)
THEN ((cs.total_orders - ps.prev_orders) / ps.prev_orders * 100)
ELSE 0
END as order_growth,
CASE
WHEN ps.prev_revenue > 0
THEN ROUND(((cs.total_revenue - ps.prev_revenue)::numeric / ps.prev_revenue * 100), 1)
THEN ((cs.total_revenue - ps.prev_revenue) / ps.prev_revenue * 100)
ELSE 0
END as revenue_growth,
CASE
WHEN cs.total_orders > 0
THEN ROUND((cs.total_revenue::numeric / cs.total_orders), 3)
THEN CAST((cs.total_revenue / cs.total_orders) AS DECIMAL(15,3))
ELSE 0
END as average_order_value,
CASE
WHEN ps.prev_orders > 0
THEN ROUND((ps.prev_revenue::numeric / ps.prev_orders), 3)
THEN CAST((ps.prev_revenue / ps.prev_orders) AS DECIMAL(15,3))
ELSE 0
END as prev_average_order_value
FROM CurrentStats cs
CROSS JOIN PreviousStats ps
`);
const orderStats = stats[0];
res.json({
orders: rows.map(row => ({
@@ -195,7 +189,7 @@ router.get('/:orderNumber', async (req, res) => {
const pool = req.app.locals.pool;
try {
// Get order details
const { rows: orderRows } = await pool.query(`
const [orderRows] = await pool.query(`
SELECT DISTINCT
o1.order_number,
o1.customer,
@@ -206,10 +200,10 @@ router.get('/:orderNumber', async (req, res) => {
o1.shipping_address,
o1.billing_address,
COUNT(o2.pid) as items_count,
ROUND(SUM(o2.price * o2.quantity)::numeric, 3) as total_amount
CAST(SUM(o2.price * o2.quantity) AS DECIMAL(15,3)) as total_amount
FROM orders o1
JOIN orders o2 ON o1.order_number = o2.order_number
WHERE o1.order_number = $1 AND o1.canceled = false
WHERE o1.order_number = ? AND o1.canceled = false
GROUP BY
o1.order_number,
o1.customer,
@@ -226,17 +220,17 @@ router.get('/:orderNumber', async (req, res) => {
}
// Get order items
const { rows: itemRows } = await pool.query(`
const [itemRows] = await pool.query(`
SELECT
o.pid,
p.title,
p.SKU,
o.quantity,
o.price,
ROUND((o.price * o.quantity)::numeric, 3) as total
CAST((o.price * o.quantity) AS DECIMAL(15,3)) as total
FROM orders o
JOIN products p ON o.pid = p.pid
WHERE o.order_number = $1 AND o.canceled = false
WHERE o.order_number = ? AND o.canceled = false
`, [req.params.orderNumber]);
const order = {

140
inventory-server/src/routes/products.js
View File

@@ -20,7 +20,7 @@ router.get('/brands', async (req, res) => {
const pool = req.app.locals.pool;
console.log('Fetching brands from database...');
const { rows } = await pool.query(`
const [results] = await pool.query(`
SELECT DISTINCT COALESCE(p.brand, 'Unbranded') as brand
FROM products p
JOIN purchase_orders po ON p.pid = po.pid
@@ -30,8 +30,8 @@ router.get('/brands', async (req, res) => {
ORDER BY COALESCE(p.brand, 'Unbranded')
`);
console.log(`Found ${rows.length} brands:`, rows.slice(0, 3));
res.json(rows.map(r => r.brand));
console.log(`Found ${results.length} brands:`, results.slice(0, 3));
res.json(results.map(r => r.brand));
} catch (error) {
console.error('Error fetching brands:', error);
res.status(500).json({ error: 'Failed to fetch brands' });
@@ -50,7 +50,6 @@ router.get('/', async (req, res) => {
const conditions = ['p.visible = true'];
const params = [];
let paramCounter = 1;
// Add default replenishable filter unless explicitly showing non-replenishable
if (req.query.showNonReplenishable !== 'true') {
@@ -59,10 +58,9 @@ router.get('/', async (req, res) => {
// Handle search filter
if (req.query.search) {
conditions.push(`(p.title ILIKE $${paramCounter} OR p.SKU ILIKE $${paramCounter} OR p.barcode ILIKE $${paramCounter})`);
conditions.push('(p.title LIKE ? OR p.SKU LIKE ? OR p.barcode LIKE ?)');
const searchTerm = `%${req.query.search}%`;
params.push(searchTerm);
paramCounter++;
params.push(searchTerm, searchTerm, searchTerm);
}
// Handle numeric filters with operators
@@ -86,69 +84,61 @@ router.get('/', async (req, res) => {
if (field) {
const operator = req.query[`${key}_operator`] || '=';
if (operator === 'between') {
// Handle between operator
try {
const [min, max] = JSON.parse(value);
conditions.push(`${field} BETWEEN $${paramCounter} AND $${paramCounter + 1}`);
conditions.push(`${field} BETWEEN ? AND ?`);
params.push(min, max);
paramCounter += 2;
} catch (e) {
console.error(`Invalid between value for ${key}:`, value);
}
} else {
conditions.push(`${field} ${operator} $${paramCounter}`);
// Handle other operators
conditions.push(`${field} ${operator} ?`);
params.push(parseFloat(value));
paramCounter++;
}
}
});
// Handle select filters
if (req.query.vendor) {
conditions.push(`p.vendor = $${paramCounter}`);
conditions.push('p.vendor = ?');
params.push(req.query.vendor);
paramCounter++;
}
if (req.query.brand) {
conditions.push(`p.brand = $${paramCounter}`);
conditions.push('p.brand = ?');
params.push(req.query.brand);
paramCounter++;
}
if (req.query.category) {
conditions.push(`p.categories ILIKE $${paramCounter}`);
conditions.push('p.categories LIKE ?');
params.push(`%${req.query.category}%`);
paramCounter++;
}
if (req.query.stockStatus && req.query.stockStatus !== 'all') {
conditions.push(`pm.stock_status = $${paramCounter}`);
conditions.push('pm.stock_status = ?');
params.push(req.query.stockStatus);
paramCounter++;
}
if (req.query.abcClass) {
conditions.push(`pm.abc_class = $${paramCounter}`);
conditions.push('pm.abc_class = ?');
params.push(req.query.abcClass);
paramCounter++;
}
if (req.query.leadTimeStatus) {
conditions.push(`pm.lead_time_status = $${paramCounter}`);
conditions.push('pm.lead_time_status = ?');
params.push(req.query.leadTimeStatus);
paramCounter++;
}
if (req.query.replenishable !== undefined) {
conditions.push(`p.replenishable = $${paramCounter}`);
params.push(req.query.replenishable === 'true');
paramCounter++;
conditions.push('p.replenishable = ?');
params.push(req.query.replenishable === 'true' ? 1 : 0);
}
if (req.query.managingStock !== undefined) {
conditions.push(`p.managing_stock = $${paramCounter}`);
params.push(req.query.managingStock === 'true');
paramCounter++;
conditions.push('p.managing_stock = ?');
params.push(req.query.managingStock === 'true' ? 1 : 0);
}
// Combine all conditions with AND
@@ -161,17 +151,17 @@ router.get('/', async (req, res) => {
LEFT JOIN product_metrics pm ON p.pid = pm.pid
${whereClause}
`;
const { rows: [countResult] } = await pool.query(countQuery, params);
const total = countResult.total;
const [countResult] = await pool.query(countQuery, params);
const total = countResult[0].total;
// Get available filters
const { rows: categories } = await pool.query(
const [categories] = await pool.query(
'SELECT name FROM categories ORDER BY name'
);
const { rows: vendors } = await pool.query(
'SELECT DISTINCT vendor FROM products WHERE visible = true AND vendor IS NOT NULL AND vendor != \'\' ORDER BY vendor'
const [vendors] = await pool.query(
'SELECT DISTINCT vendor FROM products WHERE visible = true AND vendor IS NOT NULL AND vendor != "" ORDER BY vendor'
);
const { rows: brands } = await pool.query(
const [brands] = await pool.query(
'SELECT DISTINCT COALESCE(brand, \'Unbranded\') as brand FROM products WHERE visible = true ORDER BY brand'
);
@@ -183,7 +173,7 @@ router.get('/', async (req, res) => {
c.cat_id,
c.name,
c.parent_id,
CAST(c.name AS text) as path
CAST(c.name AS CHAR(1000)) as path
FROM categories c
WHERE c.parent_id IS NULL
@@ -193,7 +183,7 @@ router.get('/', async (req, res) => {
c.cat_id,
c.name,
c.parent_id,
cp.path || ' > ' || c.name
CONCAT(cp.path, ' > ', c.name)
FROM categories c
JOIN category_path cp ON c.parent_id = cp.cat_id
),
@@ -220,6 +210,7 @@ router.get('/', async (req, res) => {
FROM products p
),
product_leaf_categories AS (
-- Find categories that aren't parents to other categories for this product
SELECT DISTINCT pc.cat_id
FROM product_categories pc
WHERE NOT EXISTS (
@@ -233,7 +224,7 @@ router.get('/', async (req, res) => {
SELECT
p.*,
COALESCE(p.brand, 'Unbranded') as brand,
string_agg(DISTINCT (c.cat_id || ':' || c.name), ',') as categories,
GROUP_CONCAT(DISTINCT CONCAT(c.cat_id, ':', c.name)) as categories,
pm.daily_sales_avg,
pm.weekly_sales_avg,
pm.monthly_sales_avg,
@@ -256,32 +247,83 @@ router.get('/', async (req, res) => {
pm.last_received_date,
pm.abc_class,
pm.stock_status,
pm.turnover_rate
pm.turnover_rate,
pm.current_lead_time,
pm.target_lead_time,
pm.lead_time_status,
pm.reorder_qty,
pm.overstocked_amt,
COALESCE(pm.days_of_inventory / NULLIF(pt.target_days, 0), 0) as stock_coverage_ratio
FROM products p
LEFT JOIN product_metrics pm ON p.pid = pm.pid
LEFT JOIN product_categories pc ON p.pid = pc.pid
LEFT JOIN categories c ON pc.cat_id = c.cat_id
${whereClause}
GROUP BY p.pid, pm.pid
LEFT JOIN product_thresholds pt ON p.pid = pt.pid
JOIN product_leaf_categories plc ON c.cat_id = plc.cat_id
${whereClause ? 'WHERE ' + whereClause.substring(6) : ''}
GROUP BY p.pid
ORDER BY ${sortColumn} ${sortDirection}
LIMIT $${paramCounter} OFFSET $${paramCounter + 1}
LIMIT ? OFFSET ?
`;
params.push(limit, offset);
const { rows: products } = await pool.query(query, params);
// Add pagination params to the main query params
const queryParams = [...params, limit, offset];
console.log('Query:', query.replace(/\s+/g, ' '));
console.log('Params:', queryParams);
const [rows] = await pool.query(query, queryParams);
// Transform the results
const products = rows.map(row => ({
...row,
categories: row.categories ? row.categories.split(',') : [],
price: parseFloat(row.price),
cost_price: parseFloat(row.cost_price),
landing_cost_price: row.landing_cost_price ? parseFloat(row.landing_cost_price) : null,
stock_quantity: parseInt(row.stock_quantity),
daily_sales_avg: parseFloat(row.daily_sales_avg) || 0,
weekly_sales_avg: parseFloat(row.weekly_sales_avg) || 0,
monthly_sales_avg: parseFloat(row.monthly_sales_avg) || 0,
avg_quantity_per_order: parseFloat(row.avg_quantity_per_order) || 0,
number_of_orders: parseInt(row.number_of_orders) || 0,
first_sale_date: row.first_sale_date || null,
last_sale_date: row.last_sale_date || null,
days_of_inventory: parseFloat(row.days_of_inventory) || 0,
weeks_of_inventory: parseFloat(row.weeks_of_inventory) || 0,
reorder_point: parseFloat(row.reorder_point) || 0,
safety_stock: parseFloat(row.safety_stock) || 0,
avg_margin_percent: parseFloat(row.avg_margin_percent) || 0,
total_revenue: parseFloat(row.total_revenue) || 0,
inventory_value: parseFloat(row.inventory_value) || 0,
cost_of_goods_sold: parseFloat(row.cost_of_goods_sold) || 0,
gross_profit: parseFloat(row.gross_profit) || 0,
gmroi: parseFloat(row.gmroi) || 0,
avg_lead_time_days: parseFloat(row.avg_lead_time_days) || 0,
last_purchase_date: row.last_purchase_date || null,
last_received_date: row.last_received_date || null,
abc_class: row.abc_class || null,
stock_status: row.stock_status || null,
turnover_rate: parseFloat(row.turnover_rate) || 0,
current_lead_time: parseFloat(row.current_lead_time) || 0,
target_lead_time: parseFloat(row.target_lead_time) || 0,
lead_time_status: row.lead_time_status || null,
stock_coverage_ratio: parseFloat(row.stock_coverage_ratio) || 0,
reorder_qty: parseInt(row.reorder_qty) || 0,
overstocked_amt: parseInt(row.overstocked_amt) || 0
}));
res.json({
products,
pagination: {
total,
pages: Math.ceil(total / limit),
currentPage: page,
pages: Math.ceil(total / limit),
limit
},
filters: {
categories: categories.map(c => c.name),
vendors: vendors.map(v => v.vendor),
brands: brands.map(b => b.brand)
categories: categories.map(category => category.name),
vendors: vendors.map(vendor => vendor.vendor),
brands: brands.map(brand => brand.brand)
}
});
} catch (error) {

96
inventory-server/src/routes/purchase-orders.js
View File

@@ -29,46 +29,40 @@ router.get('/', async (req, res) => {
let whereClause = '1=1';
const params = [];
let paramCounter = 1;
if (search) {
whereClause += ` AND (po.po_id ILIKE $${paramCounter} OR po.vendor ILIKE $${paramCounter})`;
params.push(`%${search}%`);
paramCounter++;
whereClause += ' AND (po.po_id LIKE ? OR po.vendor LIKE ?)';
params.push(`%${search}%`, `%${search}%`);
}
if (status && status !== 'all') {
whereClause += ` AND po.status = $${paramCounter}`;
whereClause += ' AND po.status = ?';
params.push(Number(status));
paramCounter++;
}
if (vendor && vendor !== 'all') {
whereClause += ` AND po.vendor = $${paramCounter}`;
whereClause += ' AND po.vendor = ?';
params.push(vendor);
paramCounter++;
}
if (startDate) {
whereClause += ` AND po.date >= $${paramCounter}`;
whereClause += ' AND po.date >= ?';
params.push(startDate);
paramCounter++;
}
if (endDate) {
whereClause += ` AND po.date <= $${paramCounter}`;
whereClause += ' AND po.date <= ?';
params.push(endDate);
paramCounter++;
}
// Get filtered summary metrics
const { rows: [summary] } = await pool.query(`
const [summary] = await pool.query(`
WITH po_totals AS (
SELECT
po_id,
SUM(ordered) as total_ordered,
SUM(received) as total_received,
ROUND(SUM(ordered * cost_price)::numeric, 3) as total_cost
CAST(SUM(ordered * cost_price) AS DECIMAL(15,3)) as total_cost
FROM purchase_orders po
WHERE ${whereClause}
GROUP BY po_id
@@ -78,26 +72,26 @@ router.get('/', async (req, res) => {
SUM(total_ordered) as total_ordered,
SUM(total_received) as total_received,
ROUND(
(SUM(total_received)::numeric / NULLIF(SUM(total_ordered), 0)), 3
SUM(total_received) / NULLIF(SUM(total_ordered), 0), 3
) as fulfillment_rate,
ROUND(SUM(total_cost)::numeric, 3) as total_value,
ROUND(AVG(total_cost)::numeric, 3) as avg_cost
CAST(SUM(total_cost) AS DECIMAL(15,3)) as total_value,
CAST(AVG(total_cost) AS DECIMAL(15,3)) as avg_cost
FROM po_totals
`, params);
// Get total count for pagination
const { rows: [countResult] } = await pool.query(`
const [countResult] = await pool.query(`
SELECT COUNT(DISTINCT po_id) as total
FROM purchase_orders po
WHERE ${whereClause}
`, params);
const total = countResult.total;
const total = countResult[0].total;
const offset = (page - 1) * limit;
const pages = Math.ceil(total / limit);
// Get recent purchase orders
const { rows: orders } = await pool.query(`
const [orders] = await pool.query(`
WITH po_totals AS (
SELECT
po_id,
@@ -107,10 +101,10 @@ router.get('/', async (req, res) => {
receiving_status,
COUNT(DISTINCT pid) as total_items,
SUM(ordered) as total_quantity,
ROUND(SUM(ordered * cost_price)::numeric, 3) as total_cost,
CAST(SUM(ordered * cost_price) AS DECIMAL(15,3)) as total_cost,
SUM(received) as total_received,
ROUND(
(SUM(received)::numeric / NULLIF(SUM(ordered), 0)), 3
SUM(received) / NULLIF(SUM(ordered), 0), 3
) as fulfillment_rate
FROM purchase_orders po
WHERE ${whereClause}
@@ -119,7 +113,7 @@ router.get('/', async (req, res) => {
SELECT
po_id as id,
vendor as vendor_name,
to_char(date, 'YYYY-MM-DD') as order_date,
DATE_FORMAT(date, '%Y-%m-%d') as order_date,
status,
receiving_status,
total_items,
@@ -130,21 +124,21 @@ router.get('/', async (req, res) => {
FROM po_totals
ORDER BY
CASE
WHEN $${paramCounter} = 'order_date' THEN date
WHEN $${paramCounter} = 'vendor_name' THEN vendor
WHEN $${paramCounter} = 'total_cost' THEN total_cost
WHEN $${paramCounter} = 'total_received' THEN total_received
WHEN $${paramCounter} = 'total_items' THEN total_items
WHEN $${paramCounter} = 'total_quantity' THEN total_quantity
WHEN $${paramCounter} = 'fulfillment_rate' THEN fulfillment_rate
WHEN $${paramCounter} = 'status' THEN status
WHEN ? = 'order_date' THEN date
WHEN ? = 'vendor_name' THEN vendor
WHEN ? = 'total_cost' THEN CAST(total_cost AS DECIMAL(15,3))
WHEN ? = 'total_received' THEN CAST(total_received AS DECIMAL(15,3))
WHEN ? = 'total_items' THEN CAST(total_items AS SIGNED)
WHEN ? = 'total_quantity' THEN CAST(total_quantity AS SIGNED)
WHEN ? = 'fulfillment_rate' THEN CAST(fulfillment_rate AS DECIMAL(5,3))
WHEN ? = 'status' THEN status
ELSE date
END ${sortDirection === 'desc' ? 'DESC' : 'ASC'}
LIMIT $${paramCounter + 1} OFFSET $${paramCounter + 2}
`, [...params, sortColumn, Number(limit), offset]);
LIMIT ? OFFSET ?
`, [...params, sortColumn, sortColumn, sortColumn, sortColumn, sortColumn, sortColumn, sortColumn, sortColumn, Number(limit), offset]);
// Get unique vendors for filter options
const { rows: vendors } = await pool.query(`
const [vendors] = await pool.query(`
SELECT DISTINCT vendor
FROM purchase_orders
WHERE vendor IS NOT NULL AND vendor != ''
@@ -152,7 +146,7 @@ router.get('/', async (req, res) => {
`);
// Get unique statuses for filter options
const { rows: statuses } = await pool.query(`
const [statuses] = await pool.query(`
SELECT DISTINCT status
FROM purchase_orders
WHERE status IS NOT NULL
@@ -175,12 +169,12 @@ router.get('/', async (req, res) => {
// Parse summary metrics
const parsedSummary = {
order_count: Number(summary.order_count) || 0,
total_ordered: Number(summary.total_ordered) || 0,
total_received: Number(summary.total_received) || 0,
fulfillment_rate: Number(summary.fulfillment_rate) || 0,
total_value: Number(summary.total_value) || 0,
avg_cost: Number(summary.avg_cost) || 0
order_count: Number(summary[0].order_count) || 0,
total_ordered: Number(summary[0].total_ordered) || 0,
total_received: Number(summary[0].total_received) || 0,
fulfillment_rate: Number(summary[0].fulfillment_rate) || 0,
total_value: Number(summary[0].total_value) || 0,
avg_cost: Number(summary[0].avg_cost) || 0
};
res.json({
@@ -208,7 +202,7 @@ router.get('/vendor-metrics', async (req, res) => {
try {
const pool = req.app.locals.pool;
const { rows: metrics } = await pool.query(`
const [metrics] = await pool.query(`
WITH delivery_metrics AS (
SELECT
vendor,
@@ -219,7 +213,7 @@ router.get('/vendor-metrics', async (req, res) => {
CASE
WHEN status >= ${STATUS.RECEIVING_STARTED} AND receiving_status >= ${RECEIVING_STATUS.PARTIAL_RECEIVED}
AND received_date IS NOT NULL AND date IS NOT NULL
THEN (received_date - date)::integer
THEN DATEDIFF(received_date, date)
ELSE NULL
END as delivery_days
FROM purchase_orders
@@ -232,18 +226,18 @@ router.get('/vendor-metrics', async (req, res) => {
SUM(ordered) as total_ordered,
SUM(received) as total_received,
ROUND(
(SUM(received)::numeric / NULLIF(SUM(ordered), 0)), 3
SUM(received) / NULLIF(SUM(ordered), 0), 3
) as fulfillment_rate,
CAST(ROUND(
SUM(ordered * cost_price) / NULLIF(SUM(ordered), 0), 2
) AS DECIMAL(15,3)) as avg_unit_cost,
CAST(SUM(ordered * cost_price) AS DECIMAL(15,3)) as total_spend,
ROUND(
(SUM(ordered * cost_price)::numeric / NULLIF(SUM(ordered), 0)), 2
) as avg_unit_cost,
ROUND(SUM(ordered * cost_price)::numeric, 3) as total_spend,
ROUND(
AVG(NULLIF(delivery_days, 0))::numeric, 1
AVG(NULLIF(delivery_days, 0)), 1
) as avg_delivery_days
FROM delivery_metrics
GROUP BY vendor
HAVING COUNT(DISTINCT po_id) > 0
HAVING total_orders > 0
ORDER BY total_spend DESC
`);
@@ -257,7 +251,7 @@ router.get('/vendor-metrics', async (req, res) => {
fulfillment_rate: Number(vendor.fulfillment_rate) || 0,
avg_unit_cost: Number(vendor.avg_unit_cost) || 0,
total_spend: Number(vendor.total_spend) || 0,
avg_delivery_days: Number(vendor.avg_delivery_days) || 0
avg_delivery_days: vendor.avg_delivery_days === null ? null : Number(vendor.avg_delivery_days)
}));
res.json(parsedMetrics);

283
inventory-server/src/routes/templates.js
View File

@@ -1,283 +0,0 @@
const express = require('express');
const { getPool } = require('../utils/db');
const dotenv = require('dotenv');
const path = require('path');
dotenv.config({ path: path.join(__dirname, "../../.env") });
const router = express.Router();
// Get all templates
router.get('/', async (req, res) => {
try {
const pool = getPool();
if (!pool) {
throw new Error('Database pool not initialized');
}
const result = await pool.query(`
SELECT * FROM templates
ORDER BY company ASC, product_type ASC
`);
res.json(result.rows);
} catch (error) {
console.error('Error fetching templates:', error);
res.status(500).json({
error: 'Failed to fetch templates',
details: error instanceof Error ? error.message : 'Unknown error'
});
}
});
// Get template by company and product type
router.get('/:company/:productType', async (req, res) => {
try {
const { company, productType } = req.params;
const pool = getPool();
if (!pool) {
throw new Error('Database pool not initialized');
}
const result = await pool.query(`
SELECT * FROM templates
WHERE company = $1 AND product_type = $2
`, [company, productType]);
if (result.rows.length === 0) {
return res.status(404).json({ error: 'Template not found' });
}
res.json(result.rows[0]);
} catch (error) {
console.error('Error fetching template:', error);
res.status(500).json({
error: 'Failed to fetch template',
details: error instanceof Error ? error.message : 'Unknown error'
});
}
});
// Create new template
router.post('/', async (req, res) => {
try {
const {
company,
product_type,
supplier,
msrp,
cost_each,
qty_per_unit,
case_qty,
hts_code,
description,
weight,
length,
width,
height,
tax_cat,
size_cat,
categories,
ship_restrictions
} = req.body;
// Validate required fields
if (!company || !product_type) {
return res.status(400).json({ error: 'Company and Product Type are required' });
}
const pool = getPool();
if (!pool) {
throw new Error('Database pool not initialized');
}
const result = await pool.query(`
INSERT INTO templates (
company,
product_type,
supplier,
msrp,
cost_each,
qty_per_unit,
case_qty,
hts_code,
description,
weight,
length,
width,
height,
tax_cat,
size_cat,
categories,
ship_restrictions
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
RETURNING *
`, [
company,
product_type,
supplier,
msrp,
cost_each,
qty_per_unit,
case_qty,
hts_code,
description,
weight,
length,
width,
height,
tax_cat,
size_cat,
categories,
ship_restrictions
]);
res.status(201).json(result.rows[0]);
} catch (error) {
console.error('Error creating template:', error);
// Check for unique constraint violation
if (error instanceof Error && error.message.includes('unique constraint')) {
return res.status(409).json({
error: 'Template already exists for this company and product type',
details: error.message
});
}
res.status(500).json({
error: 'Failed to create template',
details: error instanceof Error ? error.message : 'Unknown error'
});
}
});
// Update template
router.put('/:id', async (req, res) => {
try {
const { id } = req.params;
const {
company,
product_type,
supplier,
msrp,
cost_each,
qty_per_unit,
case_qty,
hts_code,
description,
weight,
length,
width,
height,
tax_cat,
size_cat,
categories,
ship_restrictions
} = req.body;
// Validate required fields
if (!company || !product_type) {
return res.status(400).json({ error: 'Company and Product Type are required' });
}
const pool = getPool();
if (!pool) {
throw new Error('Database pool not initialized');
}
const result = await pool.query(`
UPDATE templates
SET
company = $1,
product_type = $2,
supplier = $3,
msrp = $4,
cost_each = $5,
qty_per_unit = $6,
case_qty = $7,
hts_code = $8,
description = $9,
weight = $10,
length = $11,
width = $12,
height = $13,
tax_cat = $14,
size_cat = $15,
categories = $16,
ship_restrictions = $17
WHERE id = $18
RETURNING *
`, [
company,
product_type,
supplier,
msrp,
cost_each,
qty_per_unit,
case_qty,
hts_code,
description,
weight,
length,
width,
height,
tax_cat,
size_cat,
categories,
ship_restrictions,
id
]);
if (result.rows.length === 0) {
return res.status(404).json({ error: 'Template not found' });
}
res.json(result.rows[0]);
} catch (error) {
console.error('Error updating template:', error);
// Check for unique constraint violation
if (error instanceof Error && error.message.includes('unique constraint')) {
return res.status(409).json({
error: 'Template already exists for this company and product type',
details: error.message
});
}
res.status(500).json({
error: 'Failed to update template',
details: error instanceof Error ? error.message : 'Unknown error'
});
}
});
// Delete template
router.delete('/:id', async (req, res) => {
try {
const { id } = req.params;
const pool = getPool();
if (!pool) {
throw new Error('Database pool not initialized');
}
const result = await pool.query('DELETE FROM templates WHERE id = $1 RETURNING *', [id]);
if (result.rows.length === 0) {
return res.status(404).json({ error: 'Template not found' });
}
res.json({ message: 'Template deleted successfully' });
} catch (error) {
console.error('Error deleting template:', error);
res.status(500).json({
error: 'Failed to delete template',
details: error instanceof Error ? error.message : 'Unknown error'
});
}
});
// Error handling middleware
router.use((err, req, res, next) => {
console.error('Template route error:', err);
res.status(500).json({
error: 'Internal server error',
details: err.message
});
});
module.exports = router;

0
inventory-server/old/test-connection.js → inventory-server/src/routes/test-connection.js
View File

38
inventory-server/src/routes/vendors.js
View File

@@ -6,7 +6,7 @@ router.get('/', async (req, res) => {
const pool = req.app.locals.pool;
try {
// Get all vendors with metrics
const { rows: vendors } = await pool.query(`
const [vendors] = await pool.query(`
SELECT DISTINCT
p.vendor as name,
COALESCE(vm.active_products, 0) as active_products,
@@ -26,16 +26,16 @@ router.get('/', async (req, res) => {
// Get cost metrics for all vendors
const vendorNames = vendors.map(v => v.name);
const { rows: costMetrics } = await pool.query(`
const [costMetrics] = await pool.query(`
SELECT
vendor,
ROUND((SUM(ordered * cost_price)::numeric / NULLIF(SUM(ordered), 0)), 2) as avg_unit_cost,
ROUND(SUM(ordered * cost_price)::numeric, 3) as total_spend
CAST(ROUND(SUM(ordered * cost_price) / NULLIF(SUM(ordered), 0), 2) AS DECIMAL(15,3)) as avg_unit_cost,
CAST(SUM(ordered * cost_price) AS DECIMAL(15,3)) as total_spend
FROM purchase_orders
WHERE status = 'closed'
AND cost_price IS NOT NULL
AND ordered > 0
AND vendor = ANY($1)
AND vendor IN (?)
GROUP BY vendor
`, [vendorNames]);
@@ -49,26 +49,26 @@ router.get('/', async (req, res) => {
}, {});
// Get overall stats
const { rows: [stats] } = await pool.query(`
const [stats] = await pool.query(`
SELECT
COUNT(DISTINCT p.vendor) as totalVendors,
COUNT(DISTINCT CASE
WHEN COALESCE(vm.total_orders, 0) > 0 AND COALESCE(vm.order_fill_rate, 0) >= 75
THEN p.vendor
END) as activeVendors,
COALESCE(ROUND(AVG(NULLIF(vm.avg_lead_time_days, 0))::numeric, 1), 0) as avgLeadTime,
COALESCE(ROUND(AVG(NULLIF(vm.order_fill_rate, 0))::numeric, 1), 0) as avgFillRate,
COALESCE(ROUND(AVG(NULLIF(vm.on_time_delivery_rate, 0))::numeric, 1), 0) as avgOnTimeDelivery
COALESCE(ROUND(AVG(NULLIF(vm.avg_lead_time_days, 0)), 1), 0) as avgLeadTime,
COALESCE(ROUND(AVG(NULLIF(vm.order_fill_rate, 0)), 1), 0) as avgFillRate,
COALESCE(ROUND(AVG(NULLIF(vm.on_time_delivery_rate, 0)), 1), 0) as avgOnTimeDelivery
FROM products p
LEFT JOIN vendor_metrics vm ON p.vendor = vm.vendor
WHERE p.vendor IS NOT NULL AND p.vendor != ''
`);
// Get overall cost metrics
const { rows: [overallCostMetrics] } = await pool.query(`
const [overallCostMetrics] = await pool.query(`
SELECT
ROUND((SUM(ordered * cost_price)::numeric / NULLIF(SUM(ordered), 0)), 2) as avg_unit_cost,
ROUND(SUM(ordered * cost_price)::numeric, 3) as total_spend
CAST(ROUND(SUM(ordered * cost_price) / NULLIF(SUM(ordered), 0), 2) AS DECIMAL(15,3)) as avg_unit_cost,
CAST(SUM(ordered * cost_price) AS DECIMAL(15,3)) as total_spend
FROM purchase_orders
WHERE status = 'closed'
AND cost_price IS NOT NULL
@@ -90,13 +90,13 @@ router.get('/', async (req, res) => {
total_spend: parseFloat(costMetricsMap[vendor.name]?.total_spend || 0)
})),
stats: {
totalVendors: parseInt(stats.totalvendors),
activeVendors: parseInt(stats.activevendors),
avgLeadTime: parseFloat(stats.avgleadtime),
avgFillRate: parseFloat(stats.avgfillrate),
avgOnTimeDelivery: parseFloat(stats.avgontimedelivery),
avgUnitCost: parseFloat(overallCostMetrics.avg_unit_cost),
totalSpend: parseFloat(overallCostMetrics.total_spend)
totalVendors: parseInt(stats[0].totalVendors),
activeVendors: parseInt(stats[0].activeVendors),
avgLeadTime: parseFloat(stats[0].avgLeadTime),
avgFillRate: parseFloat(stats[0].avgFillRate),
avgOnTimeDelivery: parseFloat(stats[0].avgOnTimeDelivery),
avgUnitCost: parseFloat(overallCostMetrics[0].avg_unit_cost),
totalSpend: parseFloat(overallCostMetrics[0].total_spend)
}
});
} catch (error) {

200
inventory-server/src/server.js
View File

@@ -3,6 +3,7 @@ const cors = require('cors');
const { spawn } = require('child_process');
const path = require('path');
const fs = require('fs');
const mysql = require('mysql2/promise');
const { corsMiddleware, corsErrorHandler } = require('./middleware/cors');
const { initPool } = require('./utils/db');
const productsRouter = require('./routes/products');
@@ -15,13 +16,11 @@ const configRouter = require('./routes/config');
const metricsRouter = require('./routes/metrics');
const vendorsRouter = require('./routes/vendors');
const categoriesRouter = require('./routes/categories');
const importRouter = require('./routes/import');
const aiValidationRouter = require('./routes/ai-validation');
const templatesRouter = require('./routes/templates');
const aiPromptsRouter = require('./routes/ai-prompts');
const testConnectionRouter = require('./routes/test-connection');
// Get the absolute path to the .env file
const envPath = '/var/www/html/inventory/.env';
const envPath = path.resolve(process.cwd(), '.env');
console.log('Current working directory:', process.cwd());
console.log('Looking for .env file at:', envPath);
console.log('.env file exists:', fs.existsSync(envPath));
@@ -34,9 +33,6 @@ try {
DB_HOST: process.env.DB_HOST || 'not set',
DB_USER: process.env.DB_USER || 'not set',
DB_NAME: process.env.DB_NAME || 'not set',
DB_PASSWORD: process.env.DB_PASSWORD ? '[password set]' : 'not set',
DB_PORT: process.env.DB_PORT || 'not set',
DB_SSL: process.env.DB_SSL || 'not set'
});
} catch (error) {
console.error('Error loading .env file:', error);
@@ -66,79 +62,61 @@ app.use((req, res, next) => {
app.use(corsMiddleware);
// Body parser middleware
app.use(express.json({ limit: '10mb' }));
app.use(express.urlencoded({ extended: true, limit: '10mb' }));
app.use(express.json());
app.use(express.urlencoded({ extended: true }));
// Initialize database pool and start server
async function startServer() {
try {
// Initialize database pool
const pool = await initPool({
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
port: process.env.DB_PORT || 5432,
max: process.env.NODE_ENV === 'production' ? 20 : 10,
idleTimeoutMillis: 30000,
connectionTimeoutMillis: 2000,
ssl: process.env.DB_SSL === 'true' ? {
rejectUnauthorized: false
} : false
});
// Initialize database pool
const pool = initPool({
host: process.env.DB_HOST,
user: process.env.DB_USER,
password: process.env.DB_PASSWORD,
database: process.env.DB_NAME,
waitForConnections: true,
connectionLimit: process.env.NODE_ENV === 'production' ? 20 : 10,
queueLimit: 0,
enableKeepAlive: true,
keepAliveInitialDelay: 0
});
// Make pool available to routes
app.locals.pool = pool;
// Make pool available to routes
app.locals.pool = pool;
// Set up routes after pool is initialized
app.use('/api/products', productsRouter);
app.use('/api/dashboard', dashboardRouter);
app.use('/api/orders', ordersRouter);
app.use('/api/csv', csvRouter);
app.use('/api/analytics', analyticsRouter);
app.use('/api/purchase-orders', purchaseOrdersRouter);
app.use('/api/config', configRouter);
app.use('/api/metrics', metricsRouter);
app.use('/api/vendors', vendorsRouter);
app.use('/api/categories', categoriesRouter);
app.use('/api/import', importRouter);
app.use('/api/ai-validation', aiValidationRouter);
app.use('/api/templates', templatesRouter);
app.use('/api/ai-prompts', aiPromptsRouter);
// Routes
app.use('/api/products', productsRouter);
app.use('/api/dashboard', dashboardRouter);
app.use('/api/orders', ordersRouter);
app.use('/api/csv', csvRouter);
app.use('/api/analytics', analyticsRouter);
app.use('/api/purchase-orders', purchaseOrdersRouter);
app.use('/api/config', configRouter);
app.use('/api/metrics', metricsRouter);
app.use('/api/vendors', vendorsRouter);
app.use('/api/categories', categoriesRouter);
app.use('/api', testConnectionRouter);
// Basic health check route
app.get('/health', (req, res) => {
res.json({
status: 'ok',
timestamp: new Date().toISOString(),
environment: process.env.NODE_ENV
});
});
// Basic health check route
app.get('/health', (req, res) => {
res.json({
status: 'ok',
timestamp: new Date().toISOString(),
environment: process.env.NODE_ENV
});
});
// CORS error handler - must be before other error handlers
app.use(corsErrorHandler);
// CORS error handler - must be before other error handlers
app.use(corsErrorHandler);
// Error handling middleware - MUST be after routes and CORS error handler
app.use((err, req, res, next) => {
console.error(`[${new Date().toISOString()}] Error:`, err);
// Send detailed error in development, generic in production
const error = process.env.NODE_ENV === 'production'
? 'An internal server error occurred'
: err.message || err;
res.status(err.status || 500).json({ error });
});
const PORT = process.env.PORT || 3000;
app.listen(PORT, () => {
console.log(`[Server] Running in ${process.env.NODE_ENV || 'development'} mode on port ${PORT}`);
});
} catch (error) {
console.error('Failed to start server:', error);
process.exit(1);
}
}
// Error handling middleware - MUST be after routes and CORS error handler
app.use((err, req, res, next) => {
console.error(`[${new Date().toISOString()}] Error:`, err);
// Send detailed error in development, generic in production
const error = process.env.NODE_ENV === 'production'
? 'An internal server error occurred'
: err.message || err;
res.status(err.status || 500).json({ error });
});
// Handle uncaught exceptions
process.on('uncaughtException', (err) => {
@@ -150,6 +128,17 @@ process.on('unhandledRejection', (reason, promise) => {
console.error(`[${new Date().toISOString()}] Unhandled Rejection at:`, promise, 'reason:', reason);
});
// Test database connection
pool.getConnection()
.then(connection => {
console.log('[Database] Connected successfully');
connection.release();
})
.catch(err => {
console.error('[Database] Error connecting:', err);
process.exit(1);
});
// Initialize client sets for SSE
const importClients = new Set();
const updateClients = new Set();
@@ -200,5 +189,62 @@ const setupSSE = (req, res) => {
}
};
// Start the server
startServer();
// Update the status endpoint to include reset-metrics
app.get('/csv/status', (req, res) => {
res.json({
active: !!currentOperation,
type: currentOperation?.type || null,
progress: currentOperation ? {
status: currentOperation.status,
operation: currentOperation.operation,
current: currentOperation.current,
total: currentOperation.total,
percentage: currentOperation.percentage
} : null
});
});
// Update progress endpoint mapping
app.get('/csv/:type/progress', (req, res) => {
const { type } = req.params;
if (!['import', 'update', 'reset', 'reset-metrics'].includes(type)) {
res.status(400).json({ error: 'Invalid operation type' });
return;
}
setupSSE(req, res);
});
// Update the cancel endpoint to handle reset-metrics
app.post('/csv/cancel', (req, res) => {
const { operation } = req.query;
if (!currentOperation) {
res.status(400).json({ error: 'No operation in progress' });
return;
}
if (operation && operation.toLowerCase() !== currentOperation.type) {
res.status(400).json({ error: 'Operation type mismatch' });
return;
}
try {
// Handle cancellation based on operation type
if (currentOperation.type === 'reset-metrics') {
// Reset metrics doesn't need special cleanup
currentOperation = null;
res.json({ message: 'Reset metrics cancelled' });
} else {
// ... existing cancellation logic for other operations ...
}
} catch (error) {
console.error('Error during cancellation:', error);
res.status(500).json({ error: 'Failed to cancel operation' });
}
});
const PORT = process.env.PORT || 3000;
app.listen(PORT, () => {
console.log(`[Server] Running in ${process.env.NODE_ENV || 'development'} mode on port ${PORT}`);
});

6
inventory-server/src/utils/db.js
View File

@@ -1,9 +1,9 @@
const { Pool } = require('pg');
const mysql = require('mysql2/promise');
let pool;
function initPool(config) {
pool = new Pool(config);
pool = mysql.createPool(config);
return pool;
}
@@ -11,7 +11,7 @@ async function getConnection() {
if (!pool) {
throw new Error('Database pool not initialized');
}
return pool.connect();
return pool.getConnection();
}
module.exports = {

2345
inventory/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

32
inventory/package.json
View File

@@ -13,75 +13,55 @@
"@dnd-kit/core": "^6.3.1",
"@dnd-kit/sortable": "^10.0.0",
"@dnd-kit/utilities": "^3.2.2",
"@emotion/react": "^11.14.0",
"@emotion/styled": "^11.14.0",
"@hookform/resolvers": "^3.10.0",
"@radix-ui/react-accordion": "^1.2.2",
"@radix-ui/react-alert-dialog": "^1.1.4",
"@radix-ui/react-avatar": "^1.1.2",
"@radix-ui/react-checkbox": "^1.1.4",
"@radix-ui/react-collapsible": "^1.1.2",
"@radix-ui/react-dialog": "^1.1.6",
"@radix-ui/react-dialog": "^1.1.4",
"@radix-ui/react-dropdown-menu": "^2.1.4",
"@radix-ui/react-label": "^2.1.1",
"@radix-ui/react-popover": "^1.1.6",
"@radix-ui/react-popover": "^1.1.4",
"@radix-ui/react-progress": "^1.1.1",
"@radix-ui/react-radio-group": "^1.2.3",
"@radix-ui/react-scroll-area": "^1.2.2",
"@radix-ui/react-select": "^2.1.4",
"@radix-ui/react-separator": "^1.1.1",
"@radix-ui/react-slot": "^1.1.2",
"@radix-ui/react-slot": "^1.1.1",
"@radix-ui/react-switch": "^1.1.2",
"@radix-ui/react-tabs": "^1.1.2",
"@radix-ui/react-toast": "^1.2.6",
"@radix-ui/react-toggle": "^1.1.1",
"@radix-ui/react-toggle-group": "^1.1.1",
"@radix-ui/react-tooltip": "^1.1.6",
"@shadcn/ui": "^0.0.4",
"@tabler/icons-react": "^3.28.1",
"@tanstack/react-query": "^5.66.7",
"@tanstack/react-query": "^5.63.0",
"@tanstack/react-table": "^8.20.6",
"@tanstack/react-virtual": "^3.11.2",
"@tanstack/virtual-core": "^3.11.2",
"@types/js-levenshtein": "^1.1.3",
"@types/uuid": "^10.0.0",
"axios": "^1.8.1",
"chart.js": "^4.4.7",
"chartjs-adapter-date-fns": "^3.0.0",
"class-variance-authority": "^0.7.1",
"clsx": "^2.1.1",
"cmdk": "^1.0.0",
"date-fns": "^3.6.0",
"diff": "^7.0.0",
"framer-motion": "^12.4.4",
"js-levenshtein": "^1.1.6",
"lodash": "^4.17.21",
"lucide-react": "^0.469.0",
"motion": "^11.18.0",
"next-themes": "^0.4.4",
"react": "^18.3.1",
"react-chartjs-2": "^5.3.0",
"react-data-grid": "^7.0.0-beta.13",
"react-day-picker": "^8.10.1",
"react-dom": "^18.3.1",
"react-dropzone": "^14.3.5",
"react-hook-form": "^7.54.2",
"react-icons": "^5.4.0",
"react-router-dom": "^7.1.1",
"recharts": "^2.15.0",
"sonner": "^1.7.1",
"tailwind-merge": "^2.6.0",
"tailwindcss-animate": "^1.0.7",
"tanstack": "^1.0.0",
"uuid": "^11.0.5",
"vaul": "^1.1.2",
"xlsx": "^0.18.5",
"zod": "^3.24.2"
"vaul": "^1.1.2"
},
"devDependencies": {
"@eslint/js": "^9.17.0",
"@types/fs-extra": "^11.0.4",
"@types/lodash": "^4.17.15",
"@types/lodash": "^4.17.14",
"@types/node": "^22.10.5",
"@types/react": "^18.3.18",
"@types/react-dom": "^18.3.5",

25
inventory/scripts/copyBuild.ts Normal file
View File

@@ -0,0 +1,25 @@
import fs from 'fs-extra';
import path from 'path';
async function copyBuild() {
const sourcePath = path.resolve(__dirname, '../build');
const targetPath = path.resolve(__dirname, '../../inventory-server/frontend/build');
try {
// Ensure the target directory exists
await fs.ensureDir(path.dirname(targetPath));
// Remove old build if it exists
await fs.remove(targetPath);
// Copy new build
await fs.copy(sourcePath, targetPath);
console.log('Build files copied successfully to server directory!');
} catch (error) {
console.error('Error copying build files:', error);
process.exit(1);
}
}
copyBuild();

122
inventory/src/App.tsx
View File

@@ -1,8 +1,9 @@
import { Routes, Route, useNavigate, Navigate, useLocation } from 'react-router-dom';
import { Routes, Route, useNavigate, Navigate } from 'react-router-dom';
import { MainLayout } from './components/layout/MainLayout';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
import { Products } from './pages/Products';
import { Dashboard } from './pages/Dashboard';
import { Orders } from './pages/Orders';
import { Settings } from './pages/Settings';
import { Analytics } from './pages/Analytics';
import { Toaster } from '@/components/ui/sonner';
@@ -14,124 +15,61 @@ import { RequireAuth } from './components/auth/RequireAuth';
import Forecasting from "@/pages/Forecasting";
import { Vendors } from '@/pages/Vendors';
import { Categories } from '@/pages/Categories';
import { Import } from '@/pages/Import';
import { AuthProvider } from './contexts/AuthContext';
import { Protected } from './components/auth/Protected';
import { FirstAccessiblePage } from './components/auth/FirstAccessiblePage';
const queryClient = new QueryClient();
function App() {
const navigate = useNavigate();
const location = useLocation();
useEffect(() => {
const checkAuth = async () => {
const token = localStorage.getItem('token');
const isLoggedIn = sessionStorage.getItem('isLoggedIn') === 'true';
// If we have a token but aren't logged in yet, verify the token
if (token && !isLoggedIn) {
const token = sessionStorage.getItem('token');
if (token) {
try {
const response = await fetch(`${config.authUrl}/me`, {
const response = await fetch(`${config.authUrl}/protected`, {
headers: {
Authorization: `Bearer ${token}`,
},
});
if (!response.ok) {
localStorage.removeItem('token');
sessionStorage.removeItem('token');
sessionStorage.removeItem('isLoggedIn');
// Only navigate to login if we're not already there
if (!location.pathname.includes('/login')) {
navigate(`/login?redirect=${encodeURIComponent(location.pathname + location.search)}`);
}
} else {
// If token is valid, set the login flag
sessionStorage.setItem('isLoggedIn', 'true');
navigate('/login');
}
} catch (error) {
console.error('Token verification failed:', error);
localStorage.removeItem('token');
sessionStorage.removeItem('token');
sessionStorage.removeItem('isLoggedIn');
// Only navigate to login if we're not already there
if (!location.pathname.includes('/login')) {
navigate(`/login?redirect=${encodeURIComponent(location.pathname + location.search)}`);
}
navigate('/login');
}
}
};
checkAuth();
}, [navigate, location.pathname, location.search]);
}, [navigate]);
return (
<QueryClientProvider client={queryClient}>
<AuthProvider>
<Toaster richColors position="top-center" />
<Routes>
<Route path="/login" element={<Login />} />
<Route element={
<RequireAuth>
<MainLayout />
</RequireAuth>
}>
<Route index element={
<Protected page="dashboard" fallback={<FirstAccessiblePage />}>
<Dashboard />
</Protected>
} />
<Route path="/" element={
<Protected page="dashboard">
<Dashboard />
</Protected>
} />
<Route path="/products" element={
<Protected page="products">
<Products />
</Protected>
} />
<Route path="/import" element={
<Protected page="import">
<Import />
</Protected>
} />
<Route path="/categories" element={
<Protected page="categories">
<Categories />
</Protected>
} />
<Route path="/vendors" element={
<Protected page="vendors">
<Vendors />
</Protected>
} />
<Route path="/purchase-orders" element={
<Protected page="purchase_orders">
<PurchaseOrders />
</Protected>
} />
<Route path="/analytics" element={
<Protected page="analytics">
<Analytics />
</Protected>
} />
<Route path="/settings" element={
<Protected page="settings">
<Settings />
</Protected>
} />
<Route path="/forecasting" element={
<Protected page="forecasting">
<Forecasting />
</Protected>
} />
<Route path="*" element={<Navigate to="/" replace />} />
</Route>
</Routes>
</AuthProvider>
<Toaster richColors position="top-center" />
<Routes>
<Route path="/login" element={<Login />} />
<Route element={
<RequireAuth>
<MainLayout />
</RequireAuth>
}>
<Route path="/" element={<Dashboard />} />
<Route path="/products" element={<Products />} />
<Route path="/categories" element={<Categories />} />
<Route path="/vendors" element={<Vendors />} />
<Route path="/orders" element={<Orders />} />
<Route path="/purchase-orders" element={<PurchaseOrders />} />
<Route path="/analytics" element={<Analytics />} />
<Route path="/settings" element={<Settings />} />
<Route path="/forecasting" element={<Forecasting />} />
<Route path="*" element={<Navigate to="/" replace />} />
</Route>
</Routes>
</QueryClientProvider>
);
}

4
inventory/src/components/analytics/CategoryPerformance.tsx
View File

@@ -99,7 +99,7 @@ export function CategoryPerformance() {
))}
</Pie>
<Tooltip
formatter={(value: number, _: string, props: any) => [
formatter={(value: number, name: string, props: any) => [
`$${value.toLocaleString()}`,
<div key="tooltip">
<div className="font-medium">Category Path:</div>
@@ -143,7 +143,7 @@ export function CategoryPerformance() {
/>
<YAxis tickFormatter={(value) => `${value}%`} />
<Tooltip
formatter={(value: number, _: string, props: any) => [
formatter={(value: number, name: string, props: any) => [
`${value.toFixed(1)}%`,
<div key="tooltip">
<div className="font-medium">Category Path:</div>

2
inventory/src/components/analytics/ProfitAnalysis.tsx
View File

@@ -96,7 +96,7 @@ export function ProfitAnalysis() {
/>
<YAxis tickFormatter={(value) => `${value}%`} />
<Tooltip
formatter={(value: number, _: string, props: any) => [
formatter={(value: number, name: string, props: any) => [
`${value.toFixed(1)}%`,
<div key="tooltip">
<div className="font-medium">Category Path:</div>

44
inventory/src/components/auth/FirstAccessiblePage.tsx
View File

@@ -1,44 +0,0 @@
import { useContext } from "react";
import { Navigate } from "react-router-dom";
import { AuthContext } from "@/contexts/AuthContext";
// Define available pages in order of priority
const PAGES = [
{ path: "/products", permission: "access:products" },
{ path: "/categories", permission: "access:categories" },
{ path: "/vendors", permission: "access:vendors" },
{ path: "/purchase-orders", permission: "access:purchase_orders" },
{ path: "/analytics", permission: "access:analytics" },
{ path: "/forecasting", permission: "access:forecasting" },
{ path: "/import", permission: "access:import" },
{ path: "/settings", permission: "access:settings" },
{ path: "/ai-validation/debug", permission: "access:ai_validation_debug" }
];
export function FirstAccessiblePage() {
const { user } = useContext(AuthContext);
// If user isn't loaded yet, don't render anything
if (!user) {
return null;
}
// Admin users have access to all pages, so this component
// shouldn't be rendering for them (handled by App.tsx)
if (user.is_admin) {
return null;
}
// Find the first page the user has access to
const firstAccessiblePage = PAGES.find(page => {
return user.permissions?.includes(page.permission);
});
// If we found a page, redirect to it
if (firstAccessiblePage) {
return <Navigate to={firstAccessiblePage.path} replace />;
}
// If user has no access to any page, redirect to login
return <Navigate to="/login" replace />;
}

104
inventory/src/components/auth/PERMISSIONS.md
View File

@@ -1,104 +0,0 @@
# Permission System Documentation
This document outlines the simplified permission system implemented in the Inventory Manager application.
## Permission Structure
Permissions follow this naming convention:
- Page access: `access:{page_name}`
- Actions: `{action}:{resource}`
Examples:
- `access:products` - Can access the Products page
- `create:products` - Can create new products
- `edit:users` - Can edit user accounts
## Permission Component
### Protected
The core component that conditionally renders content based on permissions.
```tsx
<Protected
permission="create:products"
fallback={<p>No permission</p>}
>
<button>Create Product</button>
</Protected>
```
Options:
- `permission`: Single permission code (e.g., "create:products")
- `page`: Page name (checks `access:{page}` permission)
- `resource` + `action`: Resource and action (checks `{action}:{resource}` permission)
- `adminOnly`: For admin-only sections
- `fallback`: Content to show if permission check fails
### RequireAuth
Used for basic authentication checks (is user logged in?).
```tsx
<Route element={
<RequireAuth>
<MainLayout />
</RequireAuth>
}>
{/* Protected routes */}
</Route>
```
## Common Permission Codes
| Code | Description |
|------|-------------|
| `access:dashboard` | Access to Dashboard page |
| `access:products` | Access to Products page |
| `create:products` | Create new products |
| `edit:products` | Edit existing products |
| `delete:products` | Delete products |
| `view:users` | View user accounts |
| `edit:users` | Edit user accounts |
| `manage:permissions` | Assign permissions to users |
## Implementation Examples
### Page Protection
In `App.tsx`:
```tsx
<Route path="/products" element={
<Protected page="products" fallback={<Navigate to="/" />}>
<Products />
</Protected>
} />
```
### Component Level Protection
```tsx
<Protected permission="edit:products">
<form>
{/* Form fields */}
<button type="submit">Save Changes</button>
</form>
</Protected>
```
### Button Protection
```tsx
<Button
onClick={handleDelete}
disabled={!hasPermission('delete:products')}
>
Delete
</Button>
// With Protected component
<Protected permission="delete:products" fallback={null}>
<Button onClick={handleDelete}>Delete</Button>
</Protected>
```

82
inventory/src/components/auth/Protected.tsx
View File

@@ -1,82 +0,0 @@
import { ReactNode, useContext } from "react";
import { AuthContext } from "@/contexts/AuthContext";
interface ProtectedProps {
// For specific permission code
permission?: string;
// For page access permission format: access:{page}
page?: string;
// For action permission format: {action}:{resource}
resource?: string;
action?: "view" | "create" | "edit" | "delete" | string;
// For admin-only access
adminOnly?: boolean;
// Content to render if permission check passes
children: ReactNode;
// Optional fallback content
fallback?: ReactNode;
}
/**
* A simplified component that conditionally renders content based on user permissions
*/
export function Protected({
permission,
page,
resource,
action,
adminOnly,
children,
fallback = null
}: ProtectedProps) {
const { user } = useContext(AuthContext);
// If user isn't loaded yet, don't render anything
if (!user) {
return null;
}
// Admin check - admins always have access to everything
if (user.is_admin) {
return <>{children}</>;
}
// Admin-only check
if (adminOnly) {
return <>{fallback}</>;
}
// Check permissions array exists
if (!user.permissions) {
return <>{fallback}</>;
}
// Page access check (access:page)
if (page) {
const pagePermission = `access:${page.toLowerCase()}`;
if (!user.permissions.includes(pagePermission)) {
return <>{fallback}</>;
}
}
// Resource action check (action:resource)
if (resource && action) {
const resourcePermission = `${action}:${resource.toLowerCase()}`;
if (!user.permissions.includes(resourcePermission)) {
return <>{fallback}</>;
}
}
// Single permission check
if (permission && !user.permissions.includes(permission)) {
return <>{fallback}</>;
}
// If all checks pass, render children
return <>{children}</>;
}

37
inventory/src/components/auth/RequireAuth.tsx
View File

@@ -1,45 +1,8 @@
import { Navigate, useLocation } from "react-router-dom"
import { useContext, useEffect, useState } from "react"
import { AuthContext } from "@/contexts/AuthContext"
export function RequireAuth({ children }: { children: React.ReactNode }) {
const isLoggedIn = sessionStorage.getItem("isLoggedIn") === "true"
const { token, user, fetchCurrentUser } = useContext(AuthContext)
const location = useLocation()
const [isLoading, setIsLoading] = useState(!!token && !user)
// This will make sure the user data is loaded the first time
useEffect(() => {
const loadUserData = async () => {
if (token && !user) {
setIsLoading(true)
try {
await fetchCurrentUser()
} catch (error) {
console.error("Failed to fetch user data:", error)
} finally {
setIsLoading(false)
}
}
}
loadUserData()
}, [token, user, fetchCurrentUser])
// Check if token exists but we're not logged in
useEffect(() => {
if (token && !isLoggedIn) {
// Verify the token and fetch user data
fetchCurrentUser().catch(() => {
// Do nothing - the AuthContext will handle errors
})
}
}, [token, isLoggedIn, fetchCurrentUser])
// If still loading user data, show nothing yet
if (isLoading) {
return <div className="p-8 flex justify-center items-center h-screen">Loading...</div>
}
if (!isLoggedIn) {
// Redirect to login with the current path in the redirect parameter

9
inventory/src/components/dashboard/BestSellers.tsx
View File

@@ -33,6 +33,15 @@ interface BestSellerBrand {
growth_rate: string
}
interface BestSellerCategory {
cat_id: number;
name: string;
units_sold: number;
revenue: string;
profit: string;
growth_rate: string;
}
interface BestSellersData {
products: Product[]
brands: BestSellerBrand[]

20
inventory/src/components/dashboard/LowStockAlerts.tsx
View File

@@ -9,8 +9,8 @@ import {
TableRow,
} from "@/components/ui/table"
import { Badge } from "@/components/ui/badge"
import { AlertCircle, AlertTriangle } from "lucide-react"
import config from "@/config"
import { format } from "date-fns"
interface Product {
pid: number;
@@ -24,24 +24,6 @@ interface Product {
lead_time_status: string;
}
// Helper functions
const formatDate = (dateString: string) => {
return format(new Date(dateString), 'MMM dd, yyyy')
}
const getLeadTimeVariant = (status: string) => {
switch (status.toLowerCase()) {
case 'critical':
return 'destructive'
case 'warning':
return 'secondary'
case 'good':
return 'secondary'
default:
return 'secondary'
}
}
export function LowStockAlerts() {
const { data: products } = useQuery<Product[]>({
queryKey: ["low-stock"],

1
inventory/src/components/dashboard/PurchaseMetrics.tsx
View File

@@ -5,6 +5,7 @@ import config from "@/config"
import { formatCurrency } from "@/lib/utils"
import { ClipboardList, AlertCircle, Layers, DollarSign, ShoppingCart } from "lucide-react" // Importing icons
import { useState } from "react"
import { PurchaseOrderStatus, ReceivingStatus } from "@/types/status-codes"
interface PurchaseMetricsData {
activePurchaseOrders: number // Orders that are not canceled, done, or fully received

Some files were not shown because too many files have changed in this diff Show More