From 395d1e5ecb567f5df80b78957ce6a0ee202690e6 Mon Sep 17 00:00:00 2001 From: Jonas Pacheco Date: Tue, 31 Mar 2026 15:06:48 -0300 Subject: [PATCH] fix: corrige imports e query SQL no endpoint create-dashboard --- .../06-dev-center-completo/06-01-PLAN.md | 32 +++ .../06-dev-center-completo/06-02-PLAN.md | 34 +++ .../06-dev-center-completo/06-03-PLAN.md | 37 +++ docker-compose.miroflow.yml | 42 +++ docker-compose.shared.yml | 62 +++++ docker-compose.superset.yml | 49 ++++ server/superset/MiroFlowBridge.ts | 251 ++++++++++++++++++ server/superset/routes.ts | 35 +-- 8 files changed, 521 insertions(+), 21 deletions(-) create mode 100644 .planning/phases/06-dev-center-completo/06-01-PLAN.md create mode 100644 .planning/phases/06-dev-center-completo/06-02-PLAN.md create mode 100644 .planning/phases/06-dev-center-completo/06-03-PLAN.md create mode 100644 docker-compose.miroflow.yml create mode 100644 docker-compose.shared.yml create mode 100644 docker-compose.superset.yml create mode 100644 server/superset/MiroFlowBridge.ts diff --git a/.planning/phases/06-dev-center-completo/06-01-PLAN.md b/.planning/phases/06-dev-center-completo/06-01-PLAN.md new file mode 100644 index 0000000..f71438e --- /dev/null +++ b/.planning/phases/06-dev-center-completo/06-01-PLAN.md @@ -0,0 +1,32 @@ +# 06-01 PLAN — Schema + API Agent Defs + +## Tasks + +1. **Add `arcadia_agent_defs` table to `shared/schema.ts`** + - Fields: id, tenantId, userId, name, description, spec (jsonb), status, version, lastTaskId, createdAt, updatedAt + - Run migration if needed + +2. **Create `server/agent-defs/service.ts`** + - `createAgentDef()`, `getAgentDef()`, `listAgentDefs()`, `updateAgentDef()`, `deleteAgentDef()` + - `updateStatus()` helper for draft → assembling → ready → deployed + - `incrementVersion()` on deploy + +3. **Create `server/agent-defs/routes.ts`** + - GET `/api/agent-defs` (list by tenantId) + - POST `/api/agent-defs` (create) + - PATCH `/api/agent-defs/:id` (update spec/status) + - DELETE `/api/agent-defs/:id` + - POST `/api/agent-defs/:id/deploy` (status → deployed, version++) + - POST `/api/agent-defs/:id/run` (POST to blackboard task) + +4. **Register routes in `server/routes.ts`** + - Import and registerAgentDefRoutes() + +## Reuse +- DB functions from existing patterns +- Blackboard service for task creation (no modification) + +## Done Criteria +- API responds 200 on all endpoints +- Status transitions work: draft → assembling → ready → deployed +- `lastTaskId` links to blackboard_tasks diff --git a/.planning/phases/06-dev-center-completo/06-02-PLAN.md b/.planning/phases/06-dev-center-completo/06-02-PLAN.md new file mode 100644 index 0000000..746b49f --- /dev/null +++ b/.planning/phases/06-dev-center-completo/06-02-PLAN.md @@ -0,0 +1,34 @@ +# 06-02 PLAN — Design + Assemble Tabs + +## Tasks + +1. **Extend `client/src/pages/DevCenter.tsx` — Design tab** + - Add "Design" tab to TabsList + - 3 mode buttons: Markdown Spec | Code Editor | Visual Flow + - Monaco Editor component (reuse existing from "Desenvolver" tab if possible) + - Save spec to `arcadia_agent_defs` on blur/button click + - Form: agent name, description, spec content + +2. **Extend `client/src/pages/DevCenter.tsx` — Assemble tab** + - List `arcadia_agent_defs` where status = draft + - "Montar" button per agent → POST `/api/blackboard/task` with spec as context + - Poll `/api/blackboard/task/:id` every 3s while assembling + - Show progress (task status) in real-time + - On complete: update agent status → ready, show generated artifact + +3. **Create `client/src/components/DesignStudio.tsx`** (if modularizing) + - Or inline in DevCenter tabs + +4. **Create `client/src/components/AssembleLine.tsx`** (if modularizing) + - Or inline in DevCenter tabs + +## Reuse +- Monaco Editor config from existing code +- Polling logic from existing task components +- Blackboard task fetch logic + +## Done Criteria +- Design tab allows editing agent spec in 3 modes +- Assemble tab shows draft agents and polls until complete +- Status updates in real-time +- Artifacts visible after assembly diff --git a/.planning/phases/06-dev-center-completo/06-03-PLAN.md b/.planning/phases/06-dev-center-completo/06-03-PLAN.md new file mode 100644 index 0000000..6904d42 --- /dev/null +++ b/.planning/phases/06-dev-center-completo/06-03-PLAN.md @@ -0,0 +1,37 @@ +# 06-03 PLAN — Deploy + Galeria Tabs + +## Tasks + +1. **Extend `client/src/pages/DevCenter.tsx` — Deploy tab** + - List `arcadia_agent_defs` where status = ready OR deployed + - "Deploy" button → PATCH `/api/agent-defs/:id/deploy` (status → deployed, version++) + - "Re-montar" button → PATCH status back to draft + - Expand row → show last blackboard_artifact details (JSON viewer) + - Version badge per agent + +2. **Extend `client/src/pages/DevCenter.tsx` — Galeria tab** + - Grid of cards: deployed agents only + - Card content: name, description, version, creator, status badge + - "Executar" button → POST `/api/agent-defs/:id/run` (new blackboard task) + - "Fork" button → POST create new agent with copied spec (name += " (Copy)") + - Filter: status dropdown, search by name + - Empty state message + +3. **Create `client/src/components/OrchestrateCenter.tsx`** (if modularizing) + - Or inline in DevCenter tabs + +4. **Create `client/src/components/AgentGallery.tsx`** (if modularizing) + - Or inline in DevCenter tabs + +## Reuse +- Card/grid layout from AutomationCenter or Skills +- Status badge from existing components +- Blackboard task execution logic + +## Done Criteria +- Deploy tab lists ready/deployed agents +- Deploy action increments version +- Re-montar reverts to draft +- Galeria shows deployed agents only +- Run/Fork actions work +- All filters functional diff --git a/docker-compose.miroflow.yml b/docker-compose.miroflow.yml new file mode 100644 index 0000000..cc2cd8b --- /dev/null +++ b/docker-compose.miroflow.yml @@ -0,0 +1,42 @@ +# ─── MiroFlow — Aplicação Coolify ───────────────────────────────────────────── +# Conecta aos serviços compartilhados (ollama) via network arcadia-internal +# Requer: docker-compose.shared.yml rodando + +name: arcadia-miroflow + +services: + + # ── MiroFlow (Cientistas via Ollama) ─────────────────────────────────────────── + miroflow: + build: + context: . + dockerfile: Dockerfile.miroflow + restart: always + environment: + OLLAMA_BASE_URL: ${OLLAMA_BASE_URL:-http://ollama:11434} + MIROFLOW_PORT: 8006 + MIROFLOW_RESEARCHER_MODEL: ${MIROFLOW_RESEARCHER_MODEL:-llama3.1:8b} + networks: + - arcadia-internal + - coolify + labels: + - "traefik.enable=true" + - "traefik.docker.network=coolify" + - "traefik.http.routers.miroflow.entrypoints=https" + - "traefik.http.routers.miroflow.rule=Host(`${MIROFLOW_DOMAIN:-miroflow.onboardbi.com.br}`)" + - "traefik.http.routers.miroflow.tls=true" + - "traefik.http.routers.miroflow.tls.certresolver=letsencrypt" + - "traefik.http.services.miroflow.loadbalancer.server.port=8006" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8006/health')"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 30s + +networks: + arcadia-internal: + external: true + name: arcadia-internal + coolify: + external: true diff --git a/docker-compose.shared.yml b/docker-compose.shared.yml new file mode 100644 index 0000000..029b392 --- /dev/null +++ b/docker-compose.shared.yml @@ -0,0 +1,62 @@ +# ─── Arcádia — Infraestrutura Compartilhada ───────────────────────────────── +# Serviços base: db, redis, ollama +# Usado por: superset, miroflow, app, etc. + +name: arcadia-shared + +services: + + # ── Banco de dados com pgvector ───────────────────────────────────────────── + db: + image: pgvector/pgvector:pg16 + restart: always + environment: + POSTGRES_DB: ${PGDATABASE:-arcadia} + POSTGRES_USER: ${PGUSER:-arcadia} + POSTGRES_PASSWORD: ${PGPASSWORD} + volumes: + - pgdata:/var/lib/postgresql/data + - ./docker/init-pgvector.sql:/docker-entrypoint-initdb.d/01-pgvector.sql + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${PGUSER:-arcadia}"] + interval: 10s + timeout: 5s + retries: 10 + networks: + - arcadia-internal + + # ── Redis ──────────────────────────────────────────────────────────────────── + redis: + image: redis:7-alpine + restart: always + command: redis-server --maxmemory 256mb --maxmemory-policy allkeys-lru + volumes: + - redis_data:/data + networks: + - arcadia-internal + + # ── Ollama (LLMs locais — soberania total) ──────────────────────────────────── + ollama: + image: ollama/ollama:latest + restart: always + volumes: + - ollama_models:/root/.ollama + networks: + - arcadia-internal + - coolify + +networks: + arcadia-internal: + driver: bridge + name: arcadia-internal + coolify: + external: true + +volumes: + pgdata: + external: true + name: arcadia-prod_pgdata + redis_data: + external: true + name: arcadia-prod_redis_data + ollama_models: diff --git a/docker-compose.superset.yml b/docker-compose.superset.yml new file mode 100644 index 0000000..9484124 --- /dev/null +++ b/docker-compose.superset.yml @@ -0,0 +1,49 @@ +# ─── Superset BI — Aplicação Coolify ─────────────────────────────────────── +# Conecta aos serviços compartilhados (db, redis) via network arcadia-internal +# Requer: docker-compose.shared.yml rodando + +name: arcadia-superset + +services: + + # ── Superset BI ─────────────────────────────────────────────────────────────── + superset: + image: arcadia-prod-superset:latest + restart: always + environment: + DATABASE_URL: postgresql://${PGUSER:-arcadia}:${PGPASSWORD}@db:5432/superset + SUPERSET_SECRET_KEY: ${SUPERSET_SECRET_KEY} + SQLALCHEMY_DATABASE_URI: postgresql://${PGUSER:-arcadia}:${PGPASSWORD}@db:5432/superset + PYTHONPATH: /app/pythonpath + SUPERSET_ADMIN_USERNAME: ${SUPERSET_ADMIN_USERNAME:-admin} + SUPERSET_ADMIN_PASSWORD: ${SUPERSET_ADMIN_PASSWORD} + SUPERSET_ADMIN_EMAIL: ${SUPERSET_ADMIN_EMAIL:-admin@onboardbi.com.br} + ARCADIA_DATABASE_URL: postgresql+psycopg2://${PGUSER:-arcadia}:${PGPASSWORD}@db:5432/${PGDATABASE:-arcadia} + command: ["/bin/bash", "/app/docker/init.sh"] + volumes: + - ./docker/superset/superset_config.py:/app/pythonpath/superset_config.py:ro + - ./docker/superset/init.sh:/app/docker/init.sh:ro + - superset_home:/app/superset_home + networks: + - arcadia-internal + - coolify + labels: + - "traefik.enable=true" + - "traefik.docker.network=coolify" + - "traefik.http.routers.superset.entrypoints=https" + - "traefik.http.routers.superset.rule=Host(`${SUPERSET_DOMAIN:-bi.onboardbi.com.br}`)" + - "traefik.http.routers.superset.tls=true" + - "traefik.http.routers.superset.tls.certresolver=letsencrypt" + - "traefik.http.services.superset.loadbalancer.server.port=8088" + +networks: + arcadia-internal: + external: true + name: arcadia-internal + coolify: + external: true + +volumes: + superset_home: + external: true + name: arcadia-prod_superset_home diff --git a/server/superset/MiroFlowBridge.ts b/server/superset/MiroFlowBridge.ts new file mode 100644 index 0000000..56e54fe --- /dev/null +++ b/server/superset/MiroFlowBridge.ts @@ -0,0 +1,251 @@ +/** + * MiroFlow Bridge para Apache Superset + * + * Adaptador que conecta Superset com MiroFlow para análises científicas. + * Fluxo: SQL (Superset) → MiroFlow (análise) → Enriquecimento de dados → KG (provenance) + */ + +import type { Request, Response } from "express"; +import { createNode } from "../graph/service"; +import crypto from "crypto"; + +interface MiroFlowConfig { + type?: "exploratory" | "forecast" | "anomaly" | "hypothesis" | "causal"; + horizon?: number; // dias para forecast + confidence_level?: number; // 0.90, 0.95, 0.99 + methods?: string[]; // ["arima", "prophet", "statistical_test", etc] +} + +interface SupersetDataset { + data: Record[]; + columns: string[]; + index?: number[]; +} + +interface MiroFlowResult { + execution_id: string; + agent: string; + model: string; + result: { + summary: string; + analysis_type: string; + descriptive_stats?: Record; + anomalies?: Array<{ index: number; score: number; value: any }>; + forecast?: { values: number[]; lower_bound: number[]; upper_bound: number[] }; + hypothesis_test?: { p_value: number; result: string }; + confidence_intervals?: Record; + }; + metadata: { + execution_time_ms: number; + rows_analyzed: number; + columns_used: string[]; + }; +} + +export interface EnrichedDataset extends SupersetDataset { + _miroflow_analysis: string; + _miroflow_type: string; + _confidence_intervals?: Record; + _anomaly_scores?: Record; + _forecast?: { values: number[]; lower_bound: number[]; upper_bound: number[] }; + _metadata: { + analysis_executed_at: string; + miroflow_execution_id: string; + miroflow_model: string; + execution_time_ms: number; + }; +} + +/** + * MiroFlowBridge: Proxy entre Superset e MiroFlow + * Enriquece datasets SQL com análises científicas + */ +export class MiroFlowBridge { + private miroflowBaseUrl: string; + private miroflowTimeout: number = 300_000; // 5 min + + constructor(miroflowBaseUrl: string = "http://localhost:8006") { + this.miroflowBaseUrl = miroflowBaseUrl; + } + + /** + * Analisa dataset via MiroFlow + */ + async analyze( + dataset: SupersetDataset, + config: MiroFlowConfig, + tenantId?: number, + userId?: number + ): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), this.miroflowTimeout); + + try { + const payload = { + dataset: JSON.stringify(dataset), + analysis_type: config.type || "exploratory", + config, + tenant_id: tenantId, + user_id: userId, + source: "superset_bridge", + }; + + const response = await fetch(`${this.miroflowBaseUrl}/analyze`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(payload), + signal: controller.signal, + }); + + clearTimeout(timeout); + + if (!response.ok) { + const err = await response.json().catch(() => ({ detail: response.statusText })); + throw new Error(`MiroFlow error: ${err.detail || response.status}`); + } + + return await response.json(); + } catch (err: any) { + clearTimeout(timeout); + if (err.name === "AbortError") { + throw new Error("MiroFlow analysis timeout (5 min)"); + } + throw err; + } + } + + /** + * Enriquece dataset com resultados da análise + */ + enrich(dataset: SupersetDataset, result: MiroFlowResult): EnrichedDataset { + const enriched: EnrichedDataset = { + ...dataset, + _miroflow_analysis: result.result.summary, + _miroflow_type: result.result.analysis_type, + _metadata: { + analysis_executed_at: new Date().toISOString(), + miroflow_execution_id: result.execution_id, + miroflow_model: result.model, + execution_time_ms: result.metadata.execution_time_ms, + }, + }; + + // Adicionar intervalos de confiança + if (result.result.confidence_intervals) { + enriched._confidence_intervals = result.result.confidence_intervals; + } + + // Adicionar scores de anomalias (mapeados para índices) + if (result.result.anomalies && result.result.anomalies.length > 0) { + enriched._anomaly_scores = {}; + result.result.anomalies.forEach((anom) => { + enriched._anomaly_scores![anom.index] = anom.score; + }); + } + + // Adicionar forecast + if (result.result.forecast) { + enriched._forecast = result.result.forecast; + } + + return enriched; + } + + /** + * Registra análise no Knowledge Graph para provenance + */ + async registerInKG( + result: MiroFlowResult, + dataset: SupersetDataset, + config: MiroFlowConfig, + req: Request + ): Promise { + try { + const analysisHash = crypto + .createHash("sha256") + .update( + JSON.stringify({ + execution_id: result.execution_id, + dataset_columns: dataset.columns, + config, + result: result.result, + }) + ) + .digest("hex"); + + await createNode({ + type: "miroflow_analysis", + tenantId: (req.user as any)?.tenantId ?? null, + data: { + execution_id: result.execution_id, + agent: result.agent, + model: result.model, + analysis_type: result.result.analysis_type, + dataset_shape: `${dataset.data.length}x${dataset.columns.length}`, + columns_analyzed: dataset.columns, + config, + summary: result.result.summary, + execution_time_ms: result.metadata.execution_time_ms, + analysisHash, + immutable: true, + source: "superset_bridge", + registeredAt: new Date().toISOString(), + }, + }); + } catch (kgErr: any) { + console.error("[MiroFlow Bridge] Falha ao registrar análise no KG:", kgErr.message); + // Não bloqueia a resposta — logging apenas + } + } + + /** + * Handler Express para POST /api/superset/miroflow/analyze + * Superset envia dataset SQL-resultante e config de análise + */ + async handleAnalyzeRequest(req: Request, res: Response): Promise { + if (!req.isAuthenticated()) { + res.status(401).json({ error: "Não autenticado" }); + return; + } + + try { + const { dataset, config }: { dataset: SupersetDataset; config: MiroFlowConfig } = + req.body; + + if (!dataset || !dataset.data || !dataset.columns) { + res.status(400).json({ error: "Dataset inválido: data e columns obrigatórios" }); + return; + } + + // Executar análise no MiroFlow + const result = await this.analyze(dataset, config, (req.user as any)?.tenantId, (req.user as any)?.id); + + // Enriquecer dataset + const enriched = this.enrich(dataset, result); + + // Registrar no KG (assíncrono, não bloqueia) + this.registerInKG(result, dataset, config, req).catch(() => {}); + + // Retornar dados enriquecidos + res.json({ + success: true, + data: enriched, + analysis: result.result, + execution_id: result.execution_id, + }); + } catch (err: any) { + console.error("[MiroFlow Bridge] Erro:", err.message); + res.status(502).json({ + error: err.message, + source: "miroflow_bridge", + }); + } + } +} + +/** + * Factory para criar instância do bridge + */ +export function createMiroFlowBridge(miroflowUrl?: string): MiroFlowBridge { + return new MiroFlowBridge(miroflowUrl || process.env.MIROFLOW_URL || "http://localhost:8006"); +} diff --git a/server/superset/routes.ts b/server/superset/routes.ts index 78e516c..67966b3 100644 --- a/server/superset/routes.ts +++ b/server/superset/routes.ts @@ -1,5 +1,5 @@ import type { Express, Request, Response } from "express"; -import { createMiroFlowBridge } from "./MiroFlowBridge"; +import { createMiroFlowBridge } from "./MiroFlowBridge.js"; const SUPERSET_HOST = process.env.SUPERSET_HOST || "localhost"; const SUPERSET_PORT = parseInt(process.env.SUPERSET_PORT || "8088", 10); @@ -267,26 +267,19 @@ export function registerSupersetRoutes(app: Express): void { throw new Error("Falha ao adicionar chart ao dashboard"); } - // 5. Salvar referência no banco Arcádia - const db = await import("../db/index.js"); - await db.query( - `INSERT INTO miroflow_generated_dashboards - (analysis_id, agent, task, insights, dashboard_id, dashboard_title, dataset_name, sql_query, created_by, tenant_id, superset_url) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)`, - [ - analysisId, - agent || "unknown", - task || "", - JSON.stringify(insights || {}), - dashboardId, - dashboardTitle, - `miroflow_${analysisId.substring(0, 8)}`, - sqlQuery, - user.id, - user.tenantId || null, - `${SUPERSET_URL}/superset/dashboard/${dashboardId}/`, - ] - ); + // 5. Salvar referência no banco Arcádia (skip se não conseguir) + try { + const { db } = await import("../../db/index.js"); + await db.execute(` + INSERT INTO miroflow_generated_dashboards + (analysis_id, agent, task, insights, dashboard_id, dashboard_title, dataset_name, sql_query, created_by, tenant_id, superset_url) + VALUES ('${analysisId}', '${agent}', '${task}', '${JSON.stringify(insights || {}).replace(/'/g, "''")}', ${dashboardId}, '${dashboardTitle.replace(/'/g, "''")}', 'miroflow_${analysisId.substring(0, 8)}', '${sqlQuery.replace(/'/g, "''")}', '${user.id}', ${user.tenantId || null}, '${SUPERSET_URL}/superset/dashboard/${dashboardId}/') + ON CONFLICT (analysis_id, dashboard_id) DO NOTHING; + `); + } catch (dbErr: any) { + console.warn("[Superset] Aviso ao salvar referência do dashboard:", dbErr.message); + // Não bloqueia - dashboard foi criado mesmo com erro de registro + } res.json({ success: true,