1064 lines
37 KiB
TypeScript
1064 lines
37 KiB
TypeScript
import type { Express, Request, Response } from "express";
|
|
import multer from "multer";
|
|
import path from "path";
|
|
import fs from "fs";
|
|
import { createRequire } from "module";
|
|
import AdmZip from "adm-zip";
|
|
import * as BSON from "bson";
|
|
import { db } from "../../db/index";
|
|
import { biDatasets, stagedTables } from "@shared/schema";
|
|
import { sql } from "drizzle-orm";
|
|
|
|
const require = createRequire(import.meta.url);
|
|
const XLSX = require("xlsx");
|
|
|
|
const uploadDir = path.join(process.cwd(), "uploads");
|
|
if (!fs.existsSync(uploadDir)) {
|
|
fs.mkdirSync(uploadDir, { recursive: true });
|
|
}
|
|
|
|
const storage = multer.diskStorage({
|
|
destination: (req, file, cb) => cb(null, uploadDir),
|
|
filename: (req, file, cb) => {
|
|
const uniqueSuffix = Date.now() + "-" + Math.round(Math.random() * 1e9);
|
|
cb(null, uniqueSuffix + "-" + file.originalname);
|
|
},
|
|
});
|
|
|
|
const upload = multer({
|
|
storage,
|
|
limits: { fileSize: 200 * 1024 * 1024 },
|
|
fileFilter: (req, file, cb) => {
|
|
const allowedTypes = [".csv", ".txt", ".json", ".zip", ".sql", ".bson", ".xlsx", ".xls"];
|
|
const ext = path.extname(file.originalname).toLowerCase();
|
|
if (allowedTypes.includes(ext)) {
|
|
cb(null, true);
|
|
} else {
|
|
cb(new Error("Tipo de arquivo não suportado. Use CSV, TXT, JSON, SQL, ZIP ou Excel (.xlsx/.xls)."));
|
|
}
|
|
},
|
|
});
|
|
|
|
function parseCSV(content: string): { headers: string[]; rows: any[] } {
|
|
const lines = content.trim().split("\n");
|
|
if (lines.length === 0) return { headers: [], rows: [] };
|
|
|
|
const headers = lines[0].split(",").map((h) => h.trim().replace(/^["']|["']$/g, ""));
|
|
const rows = lines.slice(1).map((line) => {
|
|
const values: string[] = [];
|
|
let current = "";
|
|
let inQuotes = false;
|
|
|
|
for (const char of line) {
|
|
if (char === '"' || char === "'") {
|
|
inQuotes = !inQuotes;
|
|
} else if (char === "," && !inQuotes) {
|
|
values.push(current.trim());
|
|
current = "";
|
|
} else {
|
|
current += char;
|
|
}
|
|
}
|
|
values.push(current.trim());
|
|
|
|
const row: any = {};
|
|
headers.forEach((h, i) => {
|
|
row[h] = values[i] || "";
|
|
});
|
|
return row;
|
|
});
|
|
|
|
return { headers, rows };
|
|
}
|
|
|
|
function parseExcel(filePath: string): { headers: string[]; rows: any[]; sheets: { name: string; rowCount: number }[] } {
|
|
try {
|
|
const workbook = XLSX.readFile(filePath);
|
|
const sheets: { name: string; rowCount: number }[] = [];
|
|
let allHeaders: string[] = [];
|
|
let allRows: any[] = [];
|
|
|
|
for (const sheetName of workbook.SheetNames) {
|
|
const worksheet = workbook.Sheets[sheetName];
|
|
const jsonData = XLSX.utils.sheet_to_json(worksheet, { header: 1, defval: '' }) as any[][];
|
|
|
|
console.log(`[Excel] Processing sheet "${sheetName}" with ${jsonData.length} rows`);
|
|
|
|
if (jsonData.length > 0) {
|
|
let headerRowIndex = 0;
|
|
for (let i = 0; i < Math.min(jsonData.length, 10); i++) {
|
|
const row = jsonData[i] || [];
|
|
const nonEmptyCells = row.filter((cell: any) => cell !== null && cell !== undefined && String(cell).trim() !== '');
|
|
if (nonEmptyCells.length >= 2) {
|
|
headerRowIndex = i;
|
|
break;
|
|
}
|
|
}
|
|
|
|
const rawHeaders = jsonData[headerRowIndex] || [];
|
|
const headers: string[] = [];
|
|
const headerIndexMap: number[] = [];
|
|
|
|
for (let i = 0; i < rawHeaders.length; i++) {
|
|
const h = String(rawHeaders[i] || '').trim();
|
|
if (h !== '') {
|
|
headers.push(h);
|
|
headerIndexMap.push(i);
|
|
}
|
|
}
|
|
|
|
console.log(`[Excel] Found ${headers.length} headers at row ${headerRowIndex + 1}:`, headers.slice(0, 5));
|
|
|
|
const rows = jsonData.slice(headerRowIndex + 1)
|
|
.filter((row: any[]) => {
|
|
const hasData = row && row.some((cell: any) => cell !== null && cell !== undefined && String(cell).trim() !== '');
|
|
return hasData;
|
|
})
|
|
.map((row: any[]) => {
|
|
const obj: any = {};
|
|
headers.forEach((h: string, idx: number) => {
|
|
const colIndex = headerIndexMap[idx];
|
|
obj[h] = row[colIndex] !== undefined && row[colIndex] !== null ? String(row[colIndex]) : '';
|
|
});
|
|
return obj;
|
|
});
|
|
|
|
sheets.push({ name: sheetName, rowCount: rows.length });
|
|
console.log(`[Excel] Sheet "${sheetName}": ${headers.length} columns, ${rows.length} data rows`);
|
|
|
|
if (allHeaders.length === 0 && headers.length > 0) {
|
|
allHeaders = headers;
|
|
allRows = rows;
|
|
}
|
|
}
|
|
}
|
|
|
|
return { headers: allHeaders, rows: allRows, sheets };
|
|
} catch (e) {
|
|
console.error('Error parsing Excel:', e);
|
|
return { headers: [], rows: [], sheets: [] };
|
|
}
|
|
}
|
|
|
|
function parseJSON(content: string): { headers: string[]; rows: any[]; rootKey?: string } {
|
|
try {
|
|
const json = JSON.parse(content);
|
|
|
|
if (Array.isArray(json) && json.length > 0) {
|
|
const flatRows = json.map((item) => flattenObject(item));
|
|
return { headers: Object.keys(flatRows[0]), rows: flatRows };
|
|
}
|
|
|
|
if (typeof json === 'object' && json !== null) {
|
|
for (const key of Object.keys(json)) {
|
|
if (Array.isArray(json[key]) && json[key].length > 0) {
|
|
const flatRows = json[key].map((item: any) => flattenObject(item));
|
|
return { headers: Object.keys(flatRows[0]), rows: flatRows, rootKey: key };
|
|
}
|
|
}
|
|
const flatObj = flattenObject(json);
|
|
return { headers: Object.keys(flatObj), rows: [flatObj] };
|
|
}
|
|
|
|
return { headers: [], rows: [] };
|
|
} catch {
|
|
return { headers: [], rows: [] };
|
|
}
|
|
}
|
|
|
|
function flattenObject(obj: any, prefix = ''): any {
|
|
const result: any = {};
|
|
|
|
for (const key of Object.keys(obj)) {
|
|
const newKey = prefix ? `${prefix}_${key}` : key;
|
|
const value = obj[key];
|
|
|
|
if (value === null || value === undefined) {
|
|
result[newKey] = '';
|
|
} else if (typeof value === 'object' && !Array.isArray(value)) {
|
|
Object.assign(result, flattenObject(value, newKey));
|
|
} else if (Array.isArray(value)) {
|
|
result[newKey] = JSON.stringify(value);
|
|
} else {
|
|
result[newKey] = String(value);
|
|
}
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
interface ZipFileInfo {
|
|
name: string;
|
|
type: 'sql' | 'mongodb' | 'bson' | 'json' | 'csv' | 'unknown';
|
|
size: number;
|
|
content?: string;
|
|
bsonData?: any[];
|
|
tables?: string[];
|
|
collections?: { name: string; documents: any[] }[];
|
|
documentCount?: number;
|
|
}
|
|
|
|
function parseBsonBuffer(buffer: Buffer): any[] {
|
|
const documents: any[] = [];
|
|
let offset = 0;
|
|
|
|
try {
|
|
while (offset < buffer.length) {
|
|
if (offset + 4 > buffer.length) break;
|
|
|
|
const docSize = buffer.readInt32LE(offset);
|
|
if (docSize <= 0 || offset + docSize > buffer.length) break;
|
|
|
|
const docBuffer = buffer.slice(offset, offset + docSize);
|
|
const doc = BSON.deserialize(docBuffer);
|
|
documents.push(doc);
|
|
offset += docSize;
|
|
}
|
|
} catch (e) {
|
|
try {
|
|
const singleDoc = BSON.deserialize(buffer);
|
|
if (Array.isArray(singleDoc)) {
|
|
return singleDoc;
|
|
}
|
|
return [singleDoc];
|
|
} catch {
|
|
return documents;
|
|
}
|
|
}
|
|
|
|
return documents;
|
|
}
|
|
|
|
function parseZipFile(zipPath: string): { files: ZipFileInfo[]; summary: string } {
|
|
const zip = new AdmZip(zipPath);
|
|
const entries = zip.getEntries();
|
|
const files: ZipFileInfo[] = [];
|
|
|
|
for (const entry of entries) {
|
|
if (entry.isDirectory) continue;
|
|
|
|
const ext = path.extname(entry.entryName).toLowerCase();
|
|
const fileName = path.basename(entry.entryName);
|
|
|
|
if (fileName.endsWith('.metadata.json')) {
|
|
continue;
|
|
}
|
|
|
|
if (ext === '.bson') {
|
|
try {
|
|
const buffer = entry.getData();
|
|
const documents = parseBsonBuffer(buffer);
|
|
const collectionName = path.basename(entry.entryName, '.bson');
|
|
|
|
files.push({
|
|
name: entry.entryName,
|
|
type: 'bson',
|
|
size: entry.header.size,
|
|
bsonData: documents,
|
|
documentCount: documents.length,
|
|
collections: [{ name: collectionName, documents }],
|
|
});
|
|
} catch (e) {
|
|
console.error(`Error parsing BSON file ${entry.entryName}:`, e);
|
|
files.push({
|
|
name: entry.entryName,
|
|
type: 'bson',
|
|
size: entry.header.size,
|
|
documentCount: 0,
|
|
collections: [],
|
|
});
|
|
}
|
|
continue;
|
|
}
|
|
|
|
const content = entry.getData().toString('utf-8');
|
|
|
|
if (ext === '.sql') {
|
|
const tables = extractSqlTables(content);
|
|
files.push({
|
|
name: entry.entryName,
|
|
type: 'sql',
|
|
size: entry.header.size,
|
|
content,
|
|
tables,
|
|
});
|
|
} else if (ext === '.json') {
|
|
try {
|
|
const parsed = JSON.parse(content);
|
|
if (Array.isArray(parsed)) {
|
|
files.push({
|
|
name: entry.entryName,
|
|
type: 'mongodb',
|
|
size: entry.header.size,
|
|
content,
|
|
documentCount: parsed.length,
|
|
collections: [{ name: path.basename(entry.entryName, '.json'), documents: parsed }],
|
|
});
|
|
} else if (typeof parsed === 'object') {
|
|
const collections: { name: string; documents: any[] }[] = [];
|
|
let totalDocs = 0;
|
|
for (const key of Object.keys(parsed)) {
|
|
if (Array.isArray(parsed[key])) {
|
|
collections.push({ name: key, documents: parsed[key] });
|
|
totalDocs += parsed[key].length;
|
|
}
|
|
}
|
|
files.push({
|
|
name: entry.entryName,
|
|
type: 'mongodb',
|
|
size: entry.header.size,
|
|
content,
|
|
documentCount: totalDocs,
|
|
collections,
|
|
});
|
|
}
|
|
} catch {
|
|
files.push({ name: entry.entryName, type: 'unknown', size: entry.header.size });
|
|
}
|
|
} else if (ext === '.csv' || ext === '.txt') {
|
|
files.push({
|
|
name: entry.entryName,
|
|
type: 'csv',
|
|
size: entry.header.size,
|
|
content,
|
|
});
|
|
} else {
|
|
files.push({ name: entry.entryName, type: 'unknown', size: entry.header.size });
|
|
}
|
|
}
|
|
|
|
const sqlCount = files.filter(f => f.type === 'sql').length;
|
|
const mongoCount = files.filter(f => f.type === 'mongodb' || f.type === 'bson').length;
|
|
const csvCount = files.filter(f => f.type === 'csv').length;
|
|
|
|
return {
|
|
files,
|
|
summary: `${sqlCount} SQL, ${mongoCount} MongoDB/JSON/BSON, ${csvCount} CSV/TXT`,
|
|
};
|
|
}
|
|
|
|
function extractSqlTables(content: string): string[] {
|
|
const tables: string[] = [];
|
|
const createTableRegex = /CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?[`"']?(\w+)[`"']?/gi;
|
|
let match;
|
|
while ((match = createTableRegex.exec(content)) !== null) {
|
|
tables.push(match[1]);
|
|
}
|
|
return tables;
|
|
}
|
|
|
|
function parseMySQLCreateTable(content: string, tableName: string): { columns: { name: string; type: string }[] } | null {
|
|
const escapedTable = tableName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
const createRegex = new RegExp(
|
|
`CREATE\\s+TABLE\\s+(?:IF\\s+NOT\\s+EXISTS\\s+)?[\`"']?${escapedTable}[\`"']?\\s*\\(([\\s\\S]*?)\\)\\s*(?:ENGINE|DEFAULT|CHARSET|COLLATE|;|$)`,
|
|
'i'
|
|
);
|
|
|
|
const match = content.match(createRegex);
|
|
if (!match) return null;
|
|
|
|
const columnsSection = match[1];
|
|
const columns: { name: string; type: string }[] = [];
|
|
|
|
const lines = columnsSection.split('\n');
|
|
for (const line of lines) {
|
|
const trimmed = line.trim();
|
|
if (!trimmed || trimmed.startsWith('PRIMARY') || trimmed.startsWith('KEY') ||
|
|
trimmed.startsWith('INDEX') || trimmed.startsWith('UNIQUE') ||
|
|
trimmed.startsWith('FOREIGN') || trimmed.startsWith('CONSTRAINT')) {
|
|
continue;
|
|
}
|
|
|
|
const colMatch = trimmed.match(/^[`"']?(\w+)[`"']?\s+(\w+(?:\([^)]+\))?)/i);
|
|
if (colMatch) {
|
|
const mysqlType = colMatch[2].toUpperCase();
|
|
let pgType = 'TEXT';
|
|
|
|
if (mysqlType.includes('BIGINT')) {
|
|
pgType = 'BIGINT';
|
|
} else if (mysqlType.includes('INT') || mysqlType.includes('TINYINT') || mysqlType.includes('SMALLINT')) {
|
|
pgType = 'INTEGER';
|
|
} else if (mysqlType.includes('DECIMAL') || mysqlType.includes('NUMERIC')) {
|
|
pgType = mysqlType.replace(/UNSIGNED/gi, '').trim();
|
|
} else if (mysqlType.includes('FLOAT') || mysqlType.includes('DOUBLE')) {
|
|
pgType = 'DOUBLE PRECISION';
|
|
} else if (mysqlType.includes('DATETIME') || mysqlType.includes('TIMESTAMP')) {
|
|
pgType = 'TIMESTAMP';
|
|
} else if (mysqlType.includes('DATE')) {
|
|
pgType = 'DATE';
|
|
} else if (mysqlType.includes('TIME')) {
|
|
pgType = 'TIME';
|
|
} else if (mysqlType === 'TINYINT(1)') {
|
|
pgType = 'BOOLEAN';
|
|
} else if (mysqlType.includes('TEXT') || mysqlType.includes('LONGTEXT') || mysqlType.includes('MEDIUMTEXT')) {
|
|
pgType = 'TEXT';
|
|
} else if (mysqlType.includes('VARCHAR') || mysqlType.includes('CHAR')) {
|
|
pgType = 'TEXT';
|
|
} else if (mysqlType.includes('BLOB') || mysqlType.includes('BINARY')) {
|
|
pgType = 'BYTEA';
|
|
} else if (mysqlType.includes('JSON')) {
|
|
pgType = 'JSONB';
|
|
}
|
|
|
|
columns.push({ name: colMatch[1], type: pgType });
|
|
}
|
|
}
|
|
|
|
return columns.length > 0 ? { columns } : null;
|
|
}
|
|
|
|
function parseSqlInserts(content: string, tableName: string): { headers: string[]; rows: any[] } {
|
|
let headers: string[] = [];
|
|
const rows: any[] = [];
|
|
|
|
const escapedTable = tableName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
const insertRegex = new RegExp(
|
|
`INSERT\\s+INTO\\s+[\`"']?${escapedTable}[\`"']?\\s*\\(([^)]+)\\)\\s*VALUES\\s*\\(([\\s\\S]*?)\\)(?:\\s*ON\\s+DUPLICATE|;)`,
|
|
'gi'
|
|
);
|
|
|
|
let match;
|
|
while ((match = insertRegex.exec(content)) !== null) {
|
|
if (headers.length === 0) {
|
|
headers = match[1]
|
|
.split(',')
|
|
.map(h => h.trim().replace(/[`"'\s]/g, ''))
|
|
.filter(h => h.length > 0);
|
|
}
|
|
|
|
const valuesStr = match[2];
|
|
const values = parseValueGroup(valuesStr);
|
|
|
|
if (values.length > 0 && headers.length > 0) {
|
|
const row: any = {};
|
|
headers.forEach((h, i) => {
|
|
let val = values[i] || '';
|
|
if (val === 'NULL' || val === 'null') val = '';
|
|
row[h] = val;
|
|
});
|
|
rows.push(row);
|
|
}
|
|
}
|
|
|
|
if (rows.length === 0) {
|
|
const simpleRegex = new RegExp(
|
|
`INSERT\\s+INTO\\s+[\`"']?${escapedTable}[\`"']?\\s*\\(([^)]+)\\)\\s*VALUES\\s*([^;]+);`,
|
|
'gi'
|
|
);
|
|
|
|
while ((match = simpleRegex.exec(content)) !== null) {
|
|
if (headers.length === 0) {
|
|
headers = match[1]
|
|
.split(',')
|
|
.map(h => h.trim().replace(/[`"'\s]/g, ''))
|
|
.filter(h => h.length > 0);
|
|
}
|
|
|
|
const valuesSection = match[2];
|
|
const valueGroups = valuesSection.match(/\(([^)]+)\)/g);
|
|
|
|
if (valueGroups) {
|
|
for (const group of valueGroups) {
|
|
const values = parseValueGroup(group.slice(1, -1));
|
|
const row: any = {};
|
|
headers.forEach((h, i) => {
|
|
let val = values[i] || '';
|
|
if (val === 'NULL' || val === 'null') val = '';
|
|
row[h] = val;
|
|
});
|
|
rows.push(row);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return { headers, rows };
|
|
}
|
|
|
|
function parseValueGroup(str: string): string[] {
|
|
const values: string[] = [];
|
|
let current = '';
|
|
let inString = false;
|
|
let stringChar = '';
|
|
let i = 0;
|
|
|
|
while (i < str.length) {
|
|
const char = str[i];
|
|
|
|
if (!inString && (char === "'" || char === '"')) {
|
|
inString = true;
|
|
stringChar = char;
|
|
i++;
|
|
continue;
|
|
}
|
|
|
|
if (inString && char === stringChar) {
|
|
if (str[i + 1] === stringChar) {
|
|
current += char;
|
|
i += 2;
|
|
continue;
|
|
}
|
|
inString = false;
|
|
i++;
|
|
continue;
|
|
}
|
|
|
|
if (!inString && char === ',') {
|
|
values.push(current.trim());
|
|
current = '';
|
|
i++;
|
|
continue;
|
|
}
|
|
|
|
current += char;
|
|
i++;
|
|
}
|
|
|
|
values.push(current.trim());
|
|
return values;
|
|
}
|
|
|
|
export function registerUploadRoutes(app: Express): void {
|
|
app.post("/api/bi/upload", upload.single("file"), async (req: Request, res: Response) => {
|
|
try {
|
|
if (!req.isAuthenticated()) {
|
|
return res.status(401).json({ error: "Not authenticated" });
|
|
}
|
|
if (!req.file) {
|
|
return res.status(400).json({ error: "Nenhum arquivo enviado" });
|
|
}
|
|
|
|
const ext = path.extname(req.file.originalname).toLowerCase();
|
|
|
|
if (ext === ".zip") {
|
|
const zipResult = parseZipFile(req.file.path);
|
|
return res.json({
|
|
success: true,
|
|
filename: req.file.originalname,
|
|
filepath: req.file.path,
|
|
filesize: req.file.size,
|
|
fileType: "zip",
|
|
isZip: true,
|
|
files: zipResult.files.map(f => ({
|
|
name: f.name,
|
|
type: f.type,
|
|
size: f.size,
|
|
tables: f.tables,
|
|
collections: f.collections?.map(c => ({ name: c.name, count: c.documents.length })),
|
|
})),
|
|
summary: zipResult.summary,
|
|
});
|
|
}
|
|
|
|
const content = fs.readFileSync(req.file.path, "utf-8");
|
|
|
|
let headers: string[] = [];
|
|
let rows: any[] = [];
|
|
|
|
if (ext === ".csv" || ext === ".txt") {
|
|
const parsed = parseCSV(content);
|
|
headers = parsed.headers;
|
|
rows = parsed.rows;
|
|
} else if (ext === ".json") {
|
|
const parsed = parseJSON(content);
|
|
headers = parsed.headers;
|
|
rows = parsed.rows;
|
|
} else if (ext === ".sql") {
|
|
const tables = extractSqlTables(content);
|
|
return res.json({
|
|
success: true,
|
|
filename: req.file.originalname,
|
|
filepath: req.file.path,
|
|
filesize: req.file.size,
|
|
fileType: "sql",
|
|
isSql: true,
|
|
tables,
|
|
tableCount: tables.length,
|
|
});
|
|
} else if (ext === ".xlsx" || ext === ".xls") {
|
|
const parsed = parseExcel(req.file.path);
|
|
return res.json({
|
|
success: true,
|
|
filename: req.file.originalname,
|
|
filepath: req.file.path,
|
|
filesize: req.file.size,
|
|
fileType: ext,
|
|
isExcel: true,
|
|
headers: parsed.headers,
|
|
rowCount: parsed.rows.length,
|
|
preview: parsed.rows.slice(0, 10),
|
|
sheets: parsed.sheets,
|
|
});
|
|
}
|
|
|
|
res.json({
|
|
success: true,
|
|
filename: req.file.originalname,
|
|
filepath: req.file.path,
|
|
filesize: req.file.size,
|
|
fileType: ext,
|
|
headers,
|
|
rowCount: rows.length,
|
|
preview: rows.slice(0, 10),
|
|
});
|
|
} catch (error: any) {
|
|
console.error("Upload error:", error);
|
|
res.status(500).json({ error: error.message || "Falha no upload" });
|
|
}
|
|
});
|
|
|
|
app.post("/api/bi/upload/zip-import", upload.single("file"), async (req: Request, res: Response) => {
|
|
try {
|
|
if (!req.isAuthenticated()) {
|
|
return res.status(401).json({ error: "Not authenticated" });
|
|
}
|
|
|
|
const { filepath, selectedFiles, targetPrefix } = req.body;
|
|
|
|
if (!filepath || !fs.existsSync(filepath)) {
|
|
return res.status(400).json({ error: "Arquivo ZIP não encontrado" });
|
|
}
|
|
|
|
const zipResult = parseZipFile(filepath);
|
|
const results: any[] = [];
|
|
const prefix = (targetPrefix || "import").replace(/[^a-zA-Z0-9_]/g, "_").toLowerCase();
|
|
|
|
for (const file of zipResult.files) {
|
|
if (selectedFiles && !selectedFiles.includes(file.name)) continue;
|
|
|
|
if (file.type === "sql" && file.content && file.tables) {
|
|
for (const tableName of file.tables) {
|
|
const parsed = parseSqlInserts(file.content, tableName);
|
|
if (parsed.rows.length > 0) {
|
|
const safeTableName = `staged_${prefix}_${tableName}`.replace(/[^a-zA-Z0-9_]/g, "_").toLowerCase();
|
|
|
|
const columns = parsed.headers.map(h => `"${h.replace(/[^a-zA-Z0-9_]/g, "_")}" TEXT`);
|
|
await db.execute(sql.raw(`DROP TABLE IF EXISTS "${safeTableName}"`));
|
|
await db.execute(sql.raw(`CREATE TABLE "${safeTableName}" (id SERIAL PRIMARY KEY, ${columns.join(", ")})`));
|
|
|
|
let importedCount = 0;
|
|
for (const row of parsed.rows) {
|
|
const values = parsed.headers.map(h => `'${String(row[h] || "").replace(/'/g, "''")}'`);
|
|
try {
|
|
await db.execute(sql.raw(`INSERT INTO "${safeTableName}" (${parsed.headers.map(h => `"${h.replace(/[^a-zA-Z0-9_]/g, "_")}"`).join(", ")}) VALUES (${values.join(", ")})`));
|
|
importedCount++;
|
|
} catch {}
|
|
}
|
|
|
|
const [staged] = await db.insert(stagedTables).values({
|
|
userId: req.user!.id,
|
|
name: tableName,
|
|
sourceType: "sql",
|
|
sourceFile: file.name,
|
|
tableName: safeTableName,
|
|
columns: JSON.stringify(parsed.headers),
|
|
rowCount: importedCount,
|
|
status: "ready",
|
|
}).returning();
|
|
|
|
results.push({ table: tableName, staged: safeTableName, rows: importedCount, stagedId: staged.id });
|
|
}
|
|
}
|
|
} else if (file.type === "mongodb" && file.collections) {
|
|
for (const collection of file.collections) {
|
|
if (collection.documents.length === 0) continue;
|
|
|
|
const flatDocs = collection.documents.map((doc: any) => flattenObject(doc));
|
|
if (flatDocs.length === 0) continue;
|
|
|
|
const allKeys = new Set<string>();
|
|
for (const doc of flatDocs) {
|
|
for (const key of Object.keys(doc)) {
|
|
allKeys.add(key);
|
|
}
|
|
}
|
|
const headers = Array.from(allKeys);
|
|
|
|
const safeTableName = `staged_${prefix}_${collection.name}`.replace(/[^a-zA-Z0-9_]/g, "_").toLowerCase();
|
|
|
|
const columns = headers.map(h => `"${h.replace(/[^a-zA-Z0-9_]/g, "_")}" TEXT`);
|
|
await db.execute(sql.raw(`DROP TABLE IF EXISTS "${safeTableName}"`));
|
|
await db.execute(sql.raw(`CREATE TABLE "${safeTableName}" (id SERIAL PRIMARY KEY, ${columns.join(", ")})`));
|
|
|
|
let importedCount = 0;
|
|
for (const doc of flatDocs) {
|
|
const values = headers.map(h => `'${String(doc[h] ?? "").replace(/'/g, "''")}'`);
|
|
try {
|
|
await db.execute(sql.raw(`INSERT INTO "${safeTableName}" (${headers.map(h => `"${h.replace(/[^a-zA-Z0-9_]/g, "_")}"`).join(", ")}) VALUES (${values.join(", ")})`));
|
|
importedCount++;
|
|
} catch {}
|
|
}
|
|
|
|
const [staged] = await db.insert(stagedTables).values({
|
|
userId: req.user!.id,
|
|
name: collection.name,
|
|
sourceType: "mongodb",
|
|
sourceFile: file.name,
|
|
tableName: safeTableName,
|
|
columns: JSON.stringify(headers),
|
|
rowCount: importedCount,
|
|
status: "ready",
|
|
}).returning();
|
|
|
|
results.push({ collection: collection.name, staged: safeTableName, rows: importedCount, stagedId: staged.id });
|
|
}
|
|
} else if (file.type === "bson" && file.collections) {
|
|
for (const collection of file.collections) {
|
|
if (!collection.documents || collection.documents.length === 0) continue;
|
|
|
|
const flatDocs = collection.documents.map((doc: any) => flattenObject(doc));
|
|
if (flatDocs.length === 0) continue;
|
|
|
|
const allKeys = new Set<string>();
|
|
for (const doc of flatDocs) {
|
|
for (const key of Object.keys(doc)) {
|
|
allKeys.add(key);
|
|
}
|
|
}
|
|
const headers = Array.from(allKeys);
|
|
|
|
const safeTableName = `staged_${prefix}_${collection.name}`.replace(/[^a-zA-Z0-9_]/g, "_").toLowerCase();
|
|
|
|
const columns = headers.map(h => `"${h.replace(/[^a-zA-Z0-9_]/g, "_")}" TEXT`);
|
|
await db.execute(sql.raw(`DROP TABLE IF EXISTS "${safeTableName}"`));
|
|
await db.execute(sql.raw(`CREATE TABLE "${safeTableName}" (id SERIAL PRIMARY KEY, ${columns.join(", ")})`));
|
|
|
|
let importedCount = 0;
|
|
for (const doc of flatDocs) {
|
|
const values = headers.map(h => `'${String(doc[h] ?? "").replace(/'/g, "''")}'`);
|
|
try {
|
|
await db.execute(sql.raw(`INSERT INTO "${safeTableName}" (${headers.map(h => `"${h.replace(/[^a-zA-Z0-9_]/g, "_")}"`).join(", ")}) VALUES (${values.join(", ")})`));
|
|
importedCount++;
|
|
} catch {}
|
|
}
|
|
|
|
const [staged] = await db.insert(stagedTables).values({
|
|
userId: req.user!.id,
|
|
name: collection.name,
|
|
sourceType: "mongodb",
|
|
sourceFile: file.name,
|
|
tableName: safeTableName,
|
|
columns: JSON.stringify(headers),
|
|
rowCount: importedCount,
|
|
status: "ready",
|
|
}).returning();
|
|
|
|
results.push({ collection: collection.name, staged: safeTableName, rows: importedCount, stagedId: staged.id });
|
|
}
|
|
} else if (file.type === "csv" && file.content) {
|
|
const parsed = parseCSV(file.content);
|
|
if (parsed.rows.length > 0) {
|
|
const baseName = path.basename(file.name, path.extname(file.name));
|
|
const safeTableName = `staged_${prefix}_${baseName}`.replace(/[^a-zA-Z0-9_]/g, "_").toLowerCase();
|
|
|
|
const columns = parsed.headers.map(h => `"${h.replace(/[^a-zA-Z0-9_]/g, "_")}" TEXT`);
|
|
await db.execute(sql.raw(`DROP TABLE IF EXISTS "${safeTableName}"`));
|
|
await db.execute(sql.raw(`CREATE TABLE "${safeTableName}" (id SERIAL PRIMARY KEY, ${columns.join(", ")})`));
|
|
|
|
let importedCount = 0;
|
|
for (const row of parsed.rows) {
|
|
const values = parsed.headers.map(h => `'${String(row[h] || "").replace(/'/g, "''")}'`);
|
|
try {
|
|
await db.execute(sql.raw(`INSERT INTO "${safeTableName}" (${parsed.headers.map(h => `"${h.replace(/[^a-zA-Z0-9_]/g, "_")}"`).join(", ")}) VALUES (${values.join(", ")})`));
|
|
importedCount++;
|
|
} catch {}
|
|
}
|
|
|
|
const [staged] = await db.insert(stagedTables).values({
|
|
userId: req.user!.id,
|
|
name: baseName,
|
|
sourceType: "csv",
|
|
sourceFile: file.name,
|
|
tableName: safeTableName,
|
|
columns: JSON.stringify(parsed.headers),
|
|
rowCount: importedCount,
|
|
status: "ready",
|
|
}).returning();
|
|
|
|
results.push({ file: baseName, staged: safeTableName, rows: importedCount, stagedId: staged.id });
|
|
}
|
|
}
|
|
}
|
|
|
|
try {
|
|
fs.unlinkSync(filepath);
|
|
} catch {}
|
|
|
|
res.json({
|
|
success: true,
|
|
imported: results,
|
|
totalTables: results.length,
|
|
totalRows: results.reduce((sum, r) => sum + r.rows, 0),
|
|
});
|
|
} catch (error: any) {
|
|
console.error("ZIP import error:", error);
|
|
res.status(500).json({ error: error.message || "Falha na importação do ZIP" });
|
|
}
|
|
});
|
|
|
|
app.post("/api/bi/upload/sql-import", async (req: Request, res: Response) => {
|
|
try {
|
|
if (!req.isAuthenticated()) {
|
|
return res.status(401).json({ error: "Not authenticated" });
|
|
}
|
|
|
|
const { filepath, selectedTables, targetPrefix } = req.body;
|
|
|
|
if (!filepath || !selectedTables || !Array.isArray(selectedTables) || selectedTables.length === 0) {
|
|
return res.status(400).json({ error: "Dados inválidos" });
|
|
}
|
|
|
|
if (!fs.existsSync(filepath)) {
|
|
return res.status(400).json({ error: "Arquivo não encontrado" });
|
|
}
|
|
|
|
const content = fs.readFileSync(filepath, "utf-8");
|
|
const safePrefix = (targetPrefix || "sql").replace(/[^a-zA-Z0-9_]/g, "_").toLowerCase();
|
|
|
|
const results: { table: string; pgTable: string; rows: number; datasetId?: number; structureOnly?: boolean }[] = [];
|
|
|
|
for (const tableName of selectedTables) {
|
|
const parsed = parseSqlInserts(content, tableName);
|
|
const safeTableName = `uploaded_${safePrefix}_${tableName.replace(/[^a-zA-Z0-9_]/g, "_").toLowerCase()}`;
|
|
|
|
if (parsed.headers.length > 0 && parsed.rows.length > 0) {
|
|
const columns = parsed.headers.map((h: string) => {
|
|
const safeName = h.replace(/[^a-zA-Z0-9_]/g, "_").toLowerCase();
|
|
return `"${safeName}" TEXT`;
|
|
});
|
|
|
|
await db.execute(sql.raw(`DROP TABLE IF EXISTS "${safeTableName}"`));
|
|
await db.execute(sql.raw(`CREATE TABLE "${safeTableName}" (id SERIAL PRIMARY KEY, ${columns.join(", ")})`));
|
|
|
|
const columnNames = parsed.headers.map((h: string) =>
|
|
`"${h.replace(/[^a-zA-Z0-9_]/g, "_").toLowerCase()}"`
|
|
).join(", ");
|
|
|
|
let importedCount = 0;
|
|
|
|
for (const row of parsed.rows) {
|
|
const values = parsed.headers.map((h: string) => {
|
|
const val = row[h] || "";
|
|
const cleanVal = String(val).replace(/'/g, "''").replace(/\\/g, "\\\\");
|
|
return `'${cleanVal}'`;
|
|
}).join(", ");
|
|
|
|
try {
|
|
await db.execute(sql.raw(`INSERT INTO "${safeTableName}" (${columnNames}) VALUES (${values})`));
|
|
importedCount++;
|
|
} catch (err) {
|
|
console.error(`Row insert error for ${tableName}:`, err);
|
|
}
|
|
}
|
|
|
|
if (importedCount > 0) {
|
|
const [dataset] = await db.insert(biDatasets).values({
|
|
userId: req.user!.id,
|
|
name: tableName,
|
|
description: `Importado de SQL: ${importedCount} registros`,
|
|
queryType: "table",
|
|
tableName: safeTableName,
|
|
columns: JSON.stringify(parsed.headers.map((h: string) => ({
|
|
originalName: h,
|
|
name: h.replace(/[^a-zA-Z0-9_]/g, "_").toLowerCase(),
|
|
type: "text",
|
|
}))),
|
|
}).returning();
|
|
|
|
results.push({ table: tableName, pgTable: safeTableName, rows: importedCount, datasetId: dataset.id });
|
|
}
|
|
} else {
|
|
const structure = parseMySQLCreateTable(content, tableName);
|
|
if (structure && structure.columns.length > 0) {
|
|
const hasIdColumn = structure.columns.some(col => col.name.toLowerCase() === 'id');
|
|
const filteredColumns = hasIdColumn
|
|
? structure.columns.filter(col => col.name.toLowerCase() !== 'id')
|
|
: structure.columns;
|
|
|
|
const columns = filteredColumns.map((col) => {
|
|
const safeName = col.name.replace(/[^a-zA-Z0-9_]/g, "_").toLowerCase();
|
|
return `"${safeName}" ${col.type}`;
|
|
});
|
|
|
|
await db.execute(sql.raw(`DROP TABLE IF EXISTS "${safeTableName}"`));
|
|
const createSql = hasIdColumn
|
|
? `CREATE TABLE "${safeTableName}" (id SERIAL PRIMARY KEY, ${columns.join(", ")})`
|
|
: `CREATE TABLE "${safeTableName}" (id SERIAL PRIMARY KEY, ${columns.join(", ")})`
|
|
await db.execute(sql.raw(createSql));
|
|
|
|
const [dataset] = await db.insert(biDatasets).values({
|
|
userId: req.user!.id,
|
|
name: tableName,
|
|
description: `Estrutura importada de SQL (tabela vazia)`,
|
|
queryType: "table",
|
|
tableName: safeTableName,
|
|
columns: JSON.stringify(structure.columns.map((col) => ({
|
|
originalName: col.name,
|
|
name: col.name.replace(/[^a-zA-Z0-9_]/g, "_").toLowerCase(),
|
|
type: col.type.toLowerCase(),
|
|
}))),
|
|
}).returning();
|
|
|
|
results.push({ table: tableName, pgTable: safeTableName, rows: 0, datasetId: dataset.id, structureOnly: true });
|
|
}
|
|
}
|
|
}
|
|
|
|
try {
|
|
fs.unlinkSync(filepath);
|
|
} catch {}
|
|
|
|
res.json({
|
|
success: true,
|
|
imported: results,
|
|
totalTables: results.length,
|
|
totalRows: results.reduce((sum, r) => sum + r.rows, 0),
|
|
});
|
|
} catch (error: any) {
|
|
console.error("SQL import error:", error);
|
|
res.status(500).json({ error: error.message || "Falha na importação do SQL" });
|
|
}
|
|
});
|
|
|
|
app.post("/api/bi/upload/import", async (req: Request, res: Response) => {
|
|
try {
|
|
if (!req.isAuthenticated()) {
|
|
return res.status(401).json({ error: "Not authenticated" });
|
|
}
|
|
|
|
const { filepath, tableName, headers, fileType } = req.body;
|
|
|
|
if (!filepath || !tableName || !headers || !Array.isArray(headers)) {
|
|
return res.status(400).json({ error: "Dados inválidos" });
|
|
}
|
|
|
|
if (!fs.existsSync(filepath)) {
|
|
return res.status(400).json({ error: "Arquivo não encontrado" });
|
|
}
|
|
|
|
const safeTableName = tableName.replace(/[^a-zA-Z0-9_]/g, "_").toLowerCase();
|
|
|
|
let rows: any[] = [];
|
|
const ext = fileType || path.extname(filepath).toLowerCase();
|
|
|
|
if (ext === ".xlsx" || ext === ".xls") {
|
|
const parsed = parseExcel(filepath);
|
|
rows = parsed.rows;
|
|
console.log(`[Excel Import] Parsed ${rows.length} rows with ${parsed.headers.length} columns`);
|
|
} else if (ext === ".json") {
|
|
const content = fs.readFileSync(filepath, "utf-8");
|
|
const parsed = parseJSON(content);
|
|
rows = parsed.rows;
|
|
} else {
|
|
const content = fs.readFileSync(filepath, "utf-8");
|
|
const parsed = parseCSV(content);
|
|
rows = parsed.rows;
|
|
}
|
|
|
|
if (rows.length === 0) {
|
|
return res.status(400).json({ error: "Arquivo vazio ou formato inválido" });
|
|
}
|
|
|
|
const validHeaders = headers.filter((h: any) => h.name && h.name.trim() !== "");
|
|
|
|
if (validHeaders.length === 0) {
|
|
return res.status(400).json({ error: "Nenhuma coluna válida encontrada" });
|
|
}
|
|
|
|
const columns = validHeaders.map((h: any) => {
|
|
const safeName = h.name.replace(/[^a-zA-Z0-9_]/g, "_").toLowerCase() || `col_${Date.now()}`;
|
|
const type = h.type === "integer" ? "INTEGER" : h.type === "decimal" ? "DECIMAL(15,2)" : "TEXT";
|
|
return `"${safeName}" ${type}`;
|
|
});
|
|
|
|
const fullTableName = `uploaded_${safeTableName}`;
|
|
await db.execute(sql.raw(`DROP TABLE IF EXISTS "${fullTableName}"`));
|
|
await db.execute(sql.raw(`CREATE TABLE "${fullTableName}" (id SERIAL PRIMARY KEY, ${columns.join(", ")})`));
|
|
|
|
const columnNames = validHeaders.map((h: any) =>
|
|
`"${h.name.replace(/[^a-zA-Z0-9_]/g, "_").toLowerCase()}"`
|
|
).join(", ");
|
|
|
|
let importedCount = 0;
|
|
const batchSize = 100;
|
|
|
|
for (let i = 0; i < rows.length; i += batchSize) {
|
|
const batch = rows.slice(i, i + batchSize);
|
|
|
|
for (const row of batch) {
|
|
const values = validHeaders.map((h: any) => {
|
|
const val = row[h.originalName] || row[h.name] || "";
|
|
if (h.type === "integer") {
|
|
const num = parseInt(String(val).replace(/[^\d-]/g, ""));
|
|
return isNaN(num) ? 0 : num;
|
|
}
|
|
if (h.type === "decimal") {
|
|
const num = parseFloat(String(val).replace(/[^\d.-]/g, ""));
|
|
return isNaN(num) ? 0 : num;
|
|
}
|
|
return String(val).replace(/'/g, "''");
|
|
});
|
|
|
|
const valuesList = values.map((v, idx) => {
|
|
const h = validHeaders[idx];
|
|
if (h.type === "integer" || h.type === "decimal") {
|
|
return v;
|
|
}
|
|
return `'${v}'`;
|
|
}).join(", ");
|
|
|
|
try {
|
|
await db.execute(sql.raw(`INSERT INTO "${fullTableName}" (${columnNames}) VALUES (${valuesList})`));
|
|
importedCount++;
|
|
} catch (err) {
|
|
console.error("Row insert error:", err);
|
|
}
|
|
}
|
|
}
|
|
|
|
const [dataset] = await db.insert(biDatasets).values({
|
|
userId: req.user!.id,
|
|
name: `Dados: ${tableName}`,
|
|
description: `Importado de arquivo: ${importedCount} registros`,
|
|
queryType: "table",
|
|
tableName: fullTableName,
|
|
columns: JSON.stringify(validHeaders),
|
|
}).returning();
|
|
|
|
try {
|
|
fs.unlinkSync(filepath);
|
|
} catch {}
|
|
|
|
res.json({
|
|
success: true,
|
|
tableName: fullTableName,
|
|
rowsImported: importedCount,
|
|
datasetId: dataset.id,
|
|
});
|
|
} catch (error: any) {
|
|
console.error("Import error:", error);
|
|
res.status(500).json({ error: error.message || "Falha na importação" });
|
|
}
|
|
});
|
|
|
|
app.get("/api/bi/uploaded-files", async (req: Request, res: Response) => {
|
|
try {
|
|
if (!req.isAuthenticated()) {
|
|
return res.status(401).json({ error: "Not authenticated" });
|
|
}
|
|
|
|
const result = await db.execute(sql`
|
|
SELECT table_name
|
|
FROM information_schema.tables
|
|
WHERE table_schema = 'public' AND table_name LIKE 'uploaded_%'
|
|
ORDER BY table_name
|
|
`);
|
|
|
|
res.json(result.rows.map((r: any) => r.table_name));
|
|
} catch (error) {
|
|
console.error("Get uploaded files error:", error);
|
|
res.status(500).json({ error: "Failed to get uploaded files" });
|
|
}
|
|
});
|
|
}
|