Update import scripts, working through categories

This commit is contained in:
2025-02-14 13:30:14 -05:00
parent cc22fd8c35
commit 9623681a15
6 changed files with 928 additions and 1106 deletions

View File

@@ -1,5 +1,6 @@
const mysql = require("mysql2/promise");
const { Client } = require("ssh2");
const { Pool } = require('pg');
const dotenv = require("dotenv");
const path = require("path");
@@ -41,17 +42,41 @@ async function setupSshTunnel(sshConfig) {
async function setupConnections(sshConfig) {
const tunnel = await setupSshTunnel(sshConfig);
// Setup MySQL connection for production
const prodConnection = await mysql.createConnection({
...sshConfig.prodDbConfig,
stream: tunnel.stream,
});
const localConnection = await mysql.createPool({
...sshConfig.localDbConfig,
waitForConnections: true,
connectionLimit: 10,
queueLimit: 0
});
// Setup PostgreSQL connection pool for local
const localPool = new Pool(sshConfig.localDbConfig);
// Test the PostgreSQL connection
try {
const client = await localPool.connect();
await client.query('SELECT NOW()');
client.release();
console.log('PostgreSQL connection successful');
} catch (err) {
console.error('PostgreSQL connection error:', err);
throw err;
}
// Create a wrapper for the PostgreSQL pool to match MySQL interface
const localConnection = {
query: async (text, params) => {
const client = await localPool.connect();
try {
const result = await client.query(text, params);
return [result];
} finally {
client.release();
}
},
end: async () => {
await localPool.end();
}
};
return {
ssh: tunnel.ssh,