diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..2dc1a7d --- /dev/null +++ b/.env.example @@ -0,0 +1,73 @@ +# ─── Arcádia Suite — Variáveis de Ambiente ──────────────────────────────────── +# Copie para .env e preencha os valores. +# NUNCA commite o arquivo .env real. + +# ── Banco de dados ──────────────────────────────────────────────────────────── +DATABASE_URL=postgresql://arcadia:arcadia123@localhost:5432/arcadia +PGHOST=localhost +PGPORT=5432 +PGUSER=arcadia +PGPASSWORD=arcadia123 +PGDATABASE=arcadia + +# ── Aplicação ───────────────────────────────────────────────────────────────── +NODE_ENV=development +PORT=5000 +SESSION_SECRET=troque-por-string-aleatoria-segura-em-producao +SSO_SECRET=arcadia-sso-secret-2024-plus-integration-key-secure + +# ── Docker Mode (desativa spawn de processos filhos) ───────────────────────── +# Em produção com Docker, defina como "true" +DOCKER_MODE=false + +# ── URLs dos microserviços Python ───────────────────────────────────────────── +# Em Docker: use nome do serviço (ex: http://contabil:8003) +# Em desenvolvimento local: use localhost +CONTABIL_PYTHON_URL=http://localhost:8003 +BI_PYTHON_URL=http://localhost:8004 +AUTOMATION_PYTHON_URL=http://localhost:8005 +FISCO_PYTHON_URL=http://localhost:8002 +PYTHON_SERVICE_URL=http://localhost:8001 + +# ── IA — OpenAI ─────────────────────────────────────────────────────────────── +# Deixe vazio se usar apenas Ollama (soberania total) +OPENAI_API_KEY= + +# ── IA — LiteLLM (proxy unificado) ─────────────────────────────────────────── +# Em Docker: http://litellm:4000 | Em dev local: http://localhost:4000 +LITELLM_BASE_URL=http://localhost:4000 +LITELLM_API_KEY=arcadia-internal + +# ── IA — Ollama (LLMs locais) ───────────────────────────────────────────────── +OLLAMA_BASE_URL=http://localhost:11434 + +# ── Open WebUI ──────────────────────────────────────────────────────────────── +WEBUI_SECRET_KEY=troque-por-string-aleatoria-segura + +# ── Arcádia Plus (Laravel/Fiscal) ───────────────────────────────────────────── +PLUS_URL=http://localhost:8080 +PLUS_PORT=8080 +PLUS_API_TOKEN= + +# ── Superset (BI avançado) ──────────────────────────────────────────────────── +SUPERSET_SECRET_KEY=troque-por-string-aleatoria-segura +SUPERSET_PORT=8088 + +# ── Redis ───────────────────────────────────────────────────────────────────── +REDIS_URL=redis://localhost:6379 + +# ── Domínio (produção) ──────────────────────────────────────────────────────── +DOMAIN=seudominio.com.br + +# ── Integrações externas (opcional) ────────────────────────────────────────── +# ERPNext +ERPNEXT_URL= +ERPNEXT_API_KEY= +ERPNEXT_API_SECRET= + +# GitHub (para sync de código) +GITHUB_TOKEN= +GITHUB_REPO= + +# WhatsApp (Baileys — sessões salvas no banco) +# Nenhuma variável adicional necessária — configurado via interface diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ddb0688 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,33 @@ +# ─── Stage 1: Build ─────────────────────────────────────────────────────────── +FROM node:20-alpine AS builder + +WORKDIR /app + +COPY package*.json ./ +RUN npm ci --ignore-scripts + +COPY . . +RUN npm run build + +# ─── Stage 2: Production ────────────────────────────────────────────────────── +FROM node:20-alpine AS runner + +WORKDIR /app + +ENV NODE_ENV=production + +# Deps de produção apenas +COPY package*.json ./ +RUN npm ci --omit=dev --ignore-scripts + +# Artefatos do build +COPY --from=builder /app/dist ./dist +COPY --from=builder /app/shared ./shared +COPY --from=builder /app/migrations ./migrations + +EXPOSE 5000 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \ + CMD wget -qO- http://localhost:5000/api/health || exit 1 + +CMD ["node", "dist/index.cjs"] diff --git a/Dockerfile.python b/Dockerfile.python new file mode 100644 index 0000000..a5127df --- /dev/null +++ b/Dockerfile.python @@ -0,0 +1,47 @@ +# Dockerfile compartilhado para todos os microserviços Python +# Uso: docker build -f Dockerfile.python --build-arg SERVICE=contabil . +# Ou via docker-compose com a variável SERVICE_NAME + +FROM python:3.11-slim + +WORKDIR /app + +# Dependências de sistema necessárias para psycopg2, signxml e playwright +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + libpq-dev \ + libxml2-dev \ + libxslt-dev \ + && rm -rf /var/lib/apt/lists/* + +# Instalar dependências Python +COPY pyproject.toml ./ +RUN pip install --no-cache-dir ".[all]" 2>/dev/null || pip install --no-cache-dir \ + fastapi>=0.128.0 \ + uvicorn>=0.40.0 \ + psycopg2-binary>=2.9.11 \ + pandas>=2.3.3 \ + numpy>=2.4.1 \ + pydantic>=2.12.5 \ + python-dotenv \ + requests \ + httpx + +# Copiar código dos serviços +COPY server/python/ ./server/python/ +COPY shared/ ./shared/ + +# SERVICE_NAME deve ser: contabil | bi | automation | fisco | people +ENV SERVICE_NAME=contabil +ENV SERVICE_PORT=8000 + +# Script de entrada que seleciona o serviço correto +COPY docker/python-entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +EXPOSE ${SERVICE_PORT} + +HEALTHCHECK --interval=30s --timeout=10s --start-period=20s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://localhost:${SERVICE_PORT}/health')" || exit 1 + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml new file mode 100644 index 0000000..732542d --- /dev/null +++ b/docker-compose.prod.yml @@ -0,0 +1,212 @@ +# ─── Arcádia Suite — Produção (Coolify) ─────────────────────────────────────── +# Este arquivo é usado pelo Coolify para deploy automático. +# NÃO inclui volumes de código-fonte — só artefatos de build. +# Configurar no Coolify: Environment Variables para todas as vars ${...} + +name: arcadia-prod + +services: + + # ── Banco de dados com pgvector ───────────────────────────────────────────── + db: + image: pgvector/pgvector:pg16 + restart: always + environment: + POSTGRES_DB: ${PGDATABASE:-arcadia} + POSTGRES_USER: ${PGUSER:-arcadia} + POSTGRES_PASSWORD: ${PGPASSWORD} + volumes: + - pgdata:/var/lib/postgresql/data + - ./docker/init-pgvector.sql:/docker-entrypoint-initdb.d/01-pgvector.sql + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${PGUSER:-arcadia}"] + interval: 10s + timeout: 5s + retries: 10 + networks: + - arcadia-internal + + # ── Redis ──────────────────────────────────────────────────────────────────── + redis: + image: redis:7-alpine + restart: always + command: redis-server --maxmemory 256mb --maxmemory-policy allkeys-lru + volumes: + - redis_data:/data + networks: + - arcadia-internal + + # ── App principal ───────────────────────────────────────────────────────── + app: + build: + context: . + dockerfile: Dockerfile + restart: always + environment: + NODE_ENV: production + PORT: 5000 + DATABASE_URL: postgresql://${PGUSER:-arcadia}:${PGPASSWORD}@db:5432/${PGDATABASE:-arcadia} + REDIS_URL: redis://redis:6379 + DOCKER_MODE: "true" + CONTABIL_PYTHON_URL: http://contabil:8003 + BI_PYTHON_URL: http://bi:8004 + AUTOMATION_PYTHON_URL: http://automation:8005 + FISCO_PYTHON_URL: http://fisco:8002 + PYTHON_SERVICE_URL: http://embeddings:8001 + SESSION_SECRET: ${SESSION_SECRET} + SSO_SECRET: ${SSO_SECRET} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + LITELLM_BASE_URL: http://litellm:4000 + LITELLM_API_KEY: ${LITELLM_API_KEY} + ports: + - "5000:5000" + depends_on: + db: + condition: service_healthy + redis: + condition: service_started + networks: + - arcadia-internal + - arcadia-public + labels: + - "traefik.enable=true" + - "traefik.http.routers.arcadia.rule=Host(`${DOMAIN}`)" + - "traefik.http.routers.arcadia.tls=true" + - "traefik.http.routers.arcadia.tls.certresolver=letsencrypt" + + # ── Microserviços Python ───────────────────────────────────────────────────── + contabil: + build: + context: . + dockerfile: Dockerfile.python + restart: always + environment: + SERVICE_NAME: contabil + SERVICE_PORT: 8003 + CONTABIL_PORT: 8003 + DATABASE_URL: postgresql://${PGUSER:-arcadia}:${PGPASSWORD}@db:5432/${PGDATABASE:-arcadia} + depends_on: + db: + condition: service_healthy + networks: + - arcadia-internal + + bi: + build: + context: . + dockerfile: Dockerfile.python + restart: always + environment: + SERVICE_NAME: bi + SERVICE_PORT: 8004 + BI_PORT: 8004 + DATABASE_URL: postgresql://${PGUSER:-arcadia}:${PGPASSWORD}@db:5432/${PGDATABASE:-arcadia} + depends_on: + db: + condition: service_healthy + networks: + - arcadia-internal + + automation: + build: + context: . + dockerfile: Dockerfile.python + restart: always + environment: + SERVICE_NAME: automation + SERVICE_PORT: 8005 + AUTOMATION_PORT: 8005 + DATABASE_URL: postgresql://${PGUSER:-arcadia}:${PGPASSWORD}@db:5432/${PGDATABASE:-arcadia} + depends_on: + db: + condition: service_healthy + networks: + - arcadia-internal + + fisco: + build: + context: . + dockerfile: Dockerfile.python + restart: always + environment: + SERVICE_NAME: fisco + SERVICE_PORT: 8002 + FISCO_PORT: 8002 + DATABASE_URL: postgresql://${PGUSER:-arcadia}:${PGPASSWORD}@db:5432/${PGDATABASE:-arcadia} + depends_on: + db: + condition: service_healthy + networks: + - arcadia-internal + + embeddings: + build: + context: . + dockerfile: Dockerfile.python + restart: always + environment: + SERVICE_NAME: embeddings + SERVICE_PORT: 8001 + DATABASE_URL: postgresql://${PGUSER:-arcadia}:${PGPASSWORD}@db:5432/${PGDATABASE:-arcadia} + depends_on: + db: + condition: service_healthy + networks: + - arcadia-internal + + # ── LiteLLM (proxy de LLM) ─────────────────────────────────────────────────── + litellm: + image: ghcr.io/berriai/litellm:main-latest + restart: always + volumes: + - ./docker/litellm-config.yaml:/app/config.yaml + command: ["--config", "/app/config.yaml", "--port", "4000"] + environment: + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + LITELLM_MASTER_KEY: ${LITELLM_API_KEY} + networks: + - arcadia-internal + + # ── Ollama (LLMs locais — ativar se o servidor tiver RAM suficiente) ────────── + ollama: + image: ollama/ollama:latest + restart: always + volumes: + - ollama_models:/root/.ollama + networks: + - arcadia-internal + profiles: [ai] + + # ── Open WebUI ─────────────────────────────────────────────────────────────── + open-webui: + image: ghcr.io/open-webui/open-webui:main + restart: always + environment: + OLLAMA_BASE_URL: http://ollama:11434 + WEBUI_SECRET_KEY: ${WEBUI_SECRET_KEY} + volumes: + - open_webui_data:/app/backend/data + depends_on: + - ollama + networks: + - arcadia-internal + - arcadia-public + labels: + - "traefik.enable=true" + - "traefik.http.routers.webui.rule=Host(`ai.${DOMAIN}`)" + - "traefik.http.routers.webui.tls=true" + - "traefik.http.routers.webui.tls.certresolver=letsencrypt" + - "traefik.http.services.webui.loadbalancer.server.port=8080" + profiles: [ai] + +networks: + arcadia-internal: + driver: bridge + arcadia-public: + driver: bridge + +volumes: + pgdata: + redis_data: + ollama_models: + open_webui_data: diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..b2d8866 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,233 @@ +# ─── Arcádia Suite — Desenvolvimento Local ──────────────────────────────────── +# Uso: docker compose up +# Para subir com IA local: docker compose --profile ai up + +name: arcadia-dev + +services: + + # ── Banco de dados com pgvector ───────────────────────────────────────────── + db: + image: pgvector/pgvector:pg16 + restart: unless-stopped + environment: + POSTGRES_DB: arcadia + POSTGRES_USER: arcadia + POSTGRES_PASSWORD: arcadia123 + volumes: + - pgdata:/var/lib/postgresql/data + - ./docker/init-pgvector.sql:/docker-entrypoint-initdb.d/01-pgvector.sql + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U arcadia -d arcadia"] + interval: 10s + timeout: 5s + retries: 5 + + # ── Redis (filas de jobs) ──────────────────────────────────────────────────── + redis: + image: redis:7-alpine + restart: unless-stopped + ports: + - "6379:6379" + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 5s + retries: 5 + + # ── App principal (Node.js + React) ───────────────────────────────────────── + app: + build: + context: . + dockerfile: Dockerfile + target: builder # usa stage builder em dev (hot reload via volume) + restart: unless-stopped + command: npx tsx server/index.ts + environment: + NODE_ENV: development + PORT: 5000 + DATABASE_URL: postgresql://arcadia:arcadia123@db:5432/arcadia + REDIS_URL: redis://redis:6379 + DOCKER_MODE: "true" # desativa spawn de processos filhos + CONTABIL_PYTHON_URL: http://contabil:8003 + BI_PYTHON_URL: http://bi:8004 + AUTOMATION_PYTHON_URL: http://automation:8005 + FISCO_PYTHON_URL: http://fisco:8002 + PYTHON_SERVICE_URL: http://embeddings:8001 + SESSION_SECRET: ${SESSION_SECRET:-arcadia-dev-secret-change-in-prod} + SSO_SECRET: ${SSO_SECRET:-arcadia-sso-secret-2024-plus-integration-key-secure} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + LITELLM_BASE_URL: http://litellm:4000 + LITELLM_API_KEY: ${LITELLM_API_KEY:-arcadia-internal} + ports: + - "5000:5000" + volumes: + - .:/app + - /app/node_modules + - /app/dist + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + contabil: + condition: service_started + bi: + condition: service_started + + # ── Microserviço Contábil (Python) ────────────────────────────────────────── + contabil: + build: + context: . + dockerfile: Dockerfile.python + restart: unless-stopped + environment: + SERVICE_NAME: contabil + SERVICE_PORT: 8003 + CONTABIL_PORT: 8003 + DATABASE_URL: postgresql://arcadia:arcadia123@db:5432/arcadia + ports: + - "8003:8003" + depends_on: + db: + condition: service_healthy + + # ── Microserviço BI Engine (Python) ───────────────────────────────────────── + bi: + build: + context: . + dockerfile: Dockerfile.python + restart: unless-stopped + environment: + SERVICE_NAME: bi + SERVICE_PORT: 8004 + BI_PORT: 8004 + DATABASE_URL: postgresql://arcadia:arcadia123@db:5432/arcadia + ports: + - "8004:8004" + depends_on: + db: + condition: service_healthy + + # ── Microserviço Automações (Python) ──────────────────────────────────────── + automation: + build: + context: . + dockerfile: Dockerfile.python + restart: unless-stopped + environment: + SERVICE_NAME: automation + SERVICE_PORT: 8005 + AUTOMATION_PORT: 8005 + DATABASE_URL: postgresql://arcadia:arcadia123@db:5432/arcadia + ports: + - "8005:8005" + depends_on: + db: + condition: service_healthy + + # ── Microserviço Fiscal (Python) ──────────────────────────────────────────── + fisco: + build: + context: . + dockerfile: Dockerfile.python + restart: unless-stopped + environment: + SERVICE_NAME: fisco + SERVICE_PORT: 8002 + FISCO_PORT: 8002 + DATABASE_URL: postgresql://arcadia:arcadia123@db:5432/arcadia + ports: + - "8002:8002" + depends_on: + db: + condition: service_healthy + + # ── Serviço de Embeddings (pgvector via Python) ────────────────────────────── + embeddings: + build: + context: . + dockerfile: Dockerfile.python + restart: unless-stopped + environment: + SERVICE_NAME: embeddings + SERVICE_PORT: 8001 + DATABASE_URL: postgresql://arcadia:arcadia123@db:5432/arcadia + ports: + - "8001:8001" + depends_on: + db: + condition: service_healthy + + # ── Apache Superset (BI avançado) ──────────────────────────────────────────── + superset: + image: apache/superset:4.1.0 + restart: unless-stopped + environment: + SUPERSET_SECRET_KEY: ${SUPERSET_SECRET_KEY:-superset-secret-change-in-prod} + DATABASE_URL: postgresql://arcadia:arcadia123@db:5432/arcadia_superset + ports: + - "8088:8088" + depends_on: + db: + condition: service_healthy + profiles: [bi] + + # ─────────────── PERFIL: ai (Soberania de IA) ──────────────────────────────── + + # ── Ollama (LLMs locais) ───────────────────────────────────────────────────── + ollama: + image: ollama/ollama:latest + restart: unless-stopped + volumes: + - ollama_models:/root/.ollama + ports: + - "11434:11434" + profiles: [ai] + # Para GPU NVIDIA: adicione `deploy.resources.reservations.devices` + # deploy: + # resources: + # reservations: + # devices: + # - driver: nvidia + # count: 1 + # capabilities: [gpu] + + # ── Open WebUI (interface para devs/consultores) ───────────────────────────── + open-webui: + image: ghcr.io/open-webui/open-webui:main + restart: unless-stopped + environment: + OLLAMA_BASE_URL: http://ollama:11434 + WEBUI_SECRET_KEY: ${WEBUI_SECRET_KEY:-webui-secret-change-in-prod} + DEFAULT_MODELS: llama3.3,qwen2.5-coder + ENABLE_RAG_WEB_SEARCH: "true" + ports: + - "3001:8080" + volumes: + - open_webui_data:/app/backend/data + depends_on: + - ollama + profiles: [ai] + + # ── LiteLLM (proxy unificado de LLMs) ─────────────────────────────────────── + litellm: + image: ghcr.io/berriai/litellm:main-latest + restart: unless-stopped + volumes: + - ./docker/litellm-config.yaml:/app/config.yaml + command: ["--config", "/app/config.yaml", "--port", "4000", "--detailed_debug"] + environment: + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + OLLAMA_BASE_URL: http://ollama:11434 + LITELLM_MASTER_KEY: ${LITELLM_API_KEY:-arcadia-internal} + ports: + - "4000:4000" + profiles: [ai] + +volumes: + pgdata: + ollama_models: + open_webui_data: diff --git a/docker/init-pgvector.sql b/docker/init-pgvector.sql new file mode 100644 index 0000000..d47d650 --- /dev/null +++ b/docker/init-pgvector.sql @@ -0,0 +1,11 @@ +-- Inicialização do PostgreSQL com extensão pgvector +-- Executado automaticamente pelo container na primeira inicialização + +-- Habilitar extensão pgvector para busca por similaridade de embeddings +CREATE EXTENSION IF NOT EXISTS vector; + +-- Habilitar extensão pg_trgm para busca textual rápida +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +-- Habilitar uuid-ossp para geração de UUIDs +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; diff --git a/docker/litellm-config.yaml b/docker/litellm-config.yaml new file mode 100644 index 0000000..30ddfb1 --- /dev/null +++ b/docker/litellm-config.yaml @@ -0,0 +1,55 @@ +# LiteLLM — Proxy unificado de LLMs para o Arcádia Suite +# Documentação: https://docs.litellm.ai/docs/proxy/configs + +model_list: + + # ── OpenAI (quando disponível) ─────────────────────────────────────────────── + - model_name: gpt-4o + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY + + - model_name: gpt-4o-mini + litellm_params: + model: openai/gpt-4o-mini + api_key: os.environ/OPENAI_API_KEY + + # ── Ollama (LLMs locais — soberania) ───────────────────────────────────────── + - model_name: llama3.3 + litellm_params: + model: ollama/llama3.3 + api_base: os.environ/OLLAMA_BASE_URL + + - model_name: qwen2.5-coder + litellm_params: + model: ollama/qwen2.5-coder:7b + api_base: os.environ/OLLAMA_BASE_URL + + - model_name: deepseek-r1 + litellm_params: + model: ollama/deepseek-r1:7b + api_base: os.environ/OLLAMA_BASE_URL + + # ── Modelo padrão: tenta OpenAI, fallback para Ollama ─────────────────────── + - model_name: arcadia-default + litellm_params: + model: openai/gpt-4o-mini + api_key: os.environ/OPENAI_API_KEY + model_info: + fallbacks: ["llama3.3"] + +router_settings: + routing_strategy: least-busy + fallbacks: + - {"gpt-4o": ["llama3.3"]} + - {"gpt-4o-mini": ["qwen2.5-coder"]} + - {"arcadia-default": ["llama3.3"]} + +litellm_settings: + drop_params: true + request_timeout: 120 + set_verbose: false + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + database_url: os.environ/DATABASE_URL diff --git a/docker/python-entrypoint.sh b/docker/python-entrypoint.sh new file mode 100644 index 0000000..43f2acc --- /dev/null +++ b/docker/python-entrypoint.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# Entrypoint dos microserviços Python +# Seleciona o script correto baseado em SERVICE_NAME + +set -e + +SERVICE_NAME=${SERVICE_NAME:-contabil} +SERVICE_PORT=${SERVICE_PORT:-8000} + +echo "[entrypoint] Iniciando serviço: $SERVICE_NAME na porta $SERVICE_PORT" + +case "$SERVICE_NAME" in + contabil) + exec python -m uvicorn server.python.contabil_service:app \ + --host 0.0.0.0 \ + --port "$SERVICE_PORT" \ + --workers 2 + ;; + bi) + exec python -m uvicorn server.python.bi_engine:app \ + --host 0.0.0.0 \ + --port "$SERVICE_PORT" \ + --workers 2 + ;; + automation) + exec python -m uvicorn server.python.automation_engine:app \ + --host 0.0.0.0 \ + --port "$SERVICE_PORT" \ + --workers 1 + ;; + fisco) + exec python -m uvicorn server.python.fisco_service:app \ + --host 0.0.0.0 \ + --port "$SERVICE_PORT" \ + --workers 2 + ;; + people) + exec python -m uvicorn server.python.people_service:app \ + --host 0.0.0.0 \ + --port "$SERVICE_PORT" \ + --workers 2 + ;; + embeddings) + exec python server/python/embeddings_service.py + ;; + *) + echo "[entrypoint] ERRO: SERVICE_NAME desconhecido: $SERVICE_NAME" + echo "Valores válidos: contabil | bi | automation | fisco | people | embeddings" + exit 1 + ;; +esac diff --git a/server/graph/routes.ts b/server/graph/routes.ts new file mode 100644 index 0000000..68af4f3 --- /dev/null +++ b/server/graph/routes.ts @@ -0,0 +1,213 @@ +import { Router, Request, Response } from "express"; +import { insertGraphNodeSchema, insertGraphEdgeSchema, insertKnowledgeBaseSchema } from "@shared/schema"; +import * as graphService from "./service"; + +const router = Router(); + +// Middleware de autenticação +router.use((req: Request, res: Response, next) => { + if (!req.isAuthenticated()) { + return res.status(401).json({ error: "Não autenticado" }); + } + next(); +}); + +const tenantId = (req: Request): number | undefined => + (req.user as any)?.tenantId ?? undefined; + +// ─── Nodes ──────────────────────────────────────────────────────────────────── + +router.get("/nodes", async (req: Request, res: Response) => { + try { + const type = req.query.type as string | undefined; + const limit = req.query.limit ? parseInt(req.query.limit as string) : 100; + const nodes = await graphService.getNodes(tenantId(req), type, limit); + res.json(nodes); + } catch (error: any) { + res.status(500).json({ error: error.message }); + } +}); + +router.get("/nodes/:id", async (req: Request, res: Response) => { + try { + const node = await graphService.getNodeById(parseInt(req.params.id)); + if (!node) return res.status(404).json({ error: "Nó não encontrado" }); + res.json(node); + } catch (error: any) { + res.status(500).json({ error: error.message }); + } +}); + +router.post("/nodes", async (req: Request, res: Response) => { + try { + const parsed = insertGraphNodeSchema.safeParse({ + ...req.body, + tenantId: tenantId(req), + }); + if (!parsed.success) { + return res.status(400).json({ error: parsed.error.flatten() }); + } + const node = await graphService.createNode(parsed.data); + res.status(201).json(node); + } catch (error: any) { + res.status(500).json({ error: error.message }); + } +}); + +router.put("/nodes/:id", async (req: Request, res: Response) => { + try { + const node = await graphService.updateNode(parseInt(req.params.id), req.body); + if (!node) return res.status(404).json({ error: "Nó não encontrado" }); + res.json(node); + } catch (error: any) { + res.status(500).json({ error: error.message }); + } +}); + +router.delete("/nodes/:id", async (req: Request, res: Response) => { + try { + const deleted = await graphService.deleteNode(parseInt(req.params.id)); + if (!deleted) return res.status(404).json({ error: "Nó não encontrado" }); + res.json({ success: true }); + } catch (error: any) { + res.status(500).json({ error: error.message }); + } +}); + +// ─── Batch de Nodes ─────────────────────────────────────────────────────────── + +router.post("/nodes/batch", async (req: Request, res: Response) => { + try { + const { nodes } = req.body as { nodes: any[] }; + if (!Array.isArray(nodes)) { + return res.status(400).json({ error: "Campo 'nodes' deve ser um array" }); + } + const created = await Promise.all( + nodes.map((n) => + graphService.createNode({ ...n, tenantId: tenantId(req) }) + ) + ); + res.status(201).json(created); + } catch (error: any) { + res.status(500).json({ error: error.message }); + } +}); + +// ─── Edges ──────────────────────────────────────────────────────────────────── + +router.get("/edges", async (req: Request, res: Response) => { + try { + const sourceId = req.query.sourceId ? parseInt(req.query.sourceId as string) : undefined; + const targetId = req.query.targetId ? parseInt(req.query.targetId as string) : undefined; + const edges = await graphService.getEdges(sourceId, targetId); + res.json(edges); + } catch (error: any) { + res.status(500).json({ error: error.message }); + } +}); + +router.post("/edges", async (req: Request, res: Response) => { + try { + const parsed = insertGraphEdgeSchema.safeParse(req.body); + if (!parsed.success) { + return res.status(400).json({ error: parsed.error.flatten() }); + } + const edge = await graphService.createEdge(parsed.data); + res.status(201).json(edge); + } catch (error: any) { + res.status(500).json({ error: error.message }); + } +}); + +router.delete("/edges/:id", async (req: Request, res: Response) => { + try { + const deleted = await graphService.deleteEdge(parseInt(req.params.id)); + if (!deleted) return res.status(404).json({ error: "Aresta não encontrada" }); + res.json({ success: true }); + } catch (error: any) { + res.status(500).json({ error: error.message }); + } +}); + +// ─── Knowledge Base ─────────────────────────────────────────────────────────── + +router.get("/knowledge", async (req: Request, res: Response) => { + try { + const category = req.query.category as string | undefined; + const search = req.query.search as string | undefined; + const entries = await graphService.getKnowledgeEntries(category, search); + res.json(entries); + } catch (error: any) { + res.status(500).json({ error: error.message }); + } +}); + +router.get("/knowledge/:id", async (req: Request, res: Response) => { + try { + const entry = await graphService.getKnowledgeEntry(parseInt(req.params.id)); + if (!entry) return res.status(404).json({ error: "Entrada não encontrada" }); + res.json(entry); + } catch (error: any) { + res.status(500).json({ error: error.message }); + } +}); + +router.post("/knowledge", async (req: Request, res: Response) => { + try { + const parsed = insertKnowledgeBaseSchema.safeParse(req.body); + if (!parsed.success) { + return res.status(400).json({ error: parsed.error.flatten() }); + } + const entry = await graphService.createKnowledgeEntry(parsed.data); + res.status(201).json(entry); + } catch (error: any) { + res.status(500).json({ error: error.message }); + } +}); + +router.put("/knowledge/:id", async (req: Request, res: Response) => { + try { + const entry = await graphService.updateKnowledgeEntry(parseInt(req.params.id), req.body); + if (!entry) return res.status(404).json({ error: "Entrada não encontrada" }); + res.json(entry); + } catch (error: any) { + res.status(500).json({ error: error.message }); + } +}); + +router.delete("/knowledge/:id", async (req: Request, res: Response) => { + try { + const deleted = await graphService.deleteKnowledgeEntry(parseInt(req.params.id)); + if (!deleted) return res.status(404).json({ error: "Entrada não encontrada" }); + res.json({ success: true }); + } catch (error: any) { + res.status(500).json({ error: error.message }); + } +}); + +// ─── Busca Semântica ────────────────────────────────────────────────────────── + +router.post("/search", async (req: Request, res: Response) => { + try { + const { query, n_results = 5 } = req.body as { query: string; n_results?: number }; + if (!query) return res.status(400).json({ error: "Campo 'query' é obrigatório" }); + + const result = await graphService.semanticSearch(query, n_results); + res.json(result); + } catch (error: any) { + res.status(500).json({ error: error.message }); + } +}); + +// ─── Grafo completo para visualização ──────────────────────────────────────── + +router.get("/visualization", async (req: Request, res: Response) => { + try { + const data = await graphService.getGraphData(tenantId(req)); + res.json(data); + } catch (error: any) { + res.status(500).json({ error: error.message }); + } +}); + +export default router; diff --git a/server/graph/service.ts b/server/graph/service.ts new file mode 100644 index 0000000..f94c176 --- /dev/null +++ b/server/graph/service.ts @@ -0,0 +1,242 @@ +import { db } from "../../db/index"; +import { eq, desc, and, sql, ilike, or } from "drizzle-orm"; +import { + graphNodes, + graphEdges, + knowledgeBase, + learnedInteractions, + insertGraphNodeSchema, + insertGraphEdgeSchema, + insertKnowledgeBaseSchema, + type GraphNode, + type GraphEdge, + type KnowledgeBaseEntry, + type InsertGraphNode, + type InsertGraphEdge, + type InsertKnowledgeBaseEntry, +} from "@shared/schema"; + +const EMBEDDINGS_URL = process.env.PYTHON_SERVICE_URL || "http://localhost:8001"; + +// ─── Nodes ──────────────────────────────────────────────────────────────────── + +export async function getNodes(tenantId?: number, type?: string, limit = 100): Promise { + const conditions = []; + if (tenantId) conditions.push(eq(graphNodes.tenantId, tenantId)); + if (type) conditions.push(eq(graphNodes.type, type)); + + return db + .select() + .from(graphNodes) + .where(conditions.length ? and(...conditions) : undefined) + .orderBy(desc(graphNodes.createdAt)) + .limit(limit); +} + +export async function getNodeById(id: number): Promise { + const [node] = await db.select().from(graphNodes).where(eq(graphNodes.id, id)); + return node; +} + +export async function createNode(data: InsertGraphNode): Promise { + const [node] = await db.insert(graphNodes).values(data).returning(); + + // Indexar embedding em background (não bloqueia a resposta) + const content = typeof data.data === "object" ? JSON.stringify(data.data) : String(data.data); + indexNodeEmbedding(node.id, content, data.type).catch(() => {}); + + return node; +} + +export async function updateNode(id: number, data: Partial): Promise { + const [node] = await db + .update(graphNodes) + .set({ ...data, updatedAt: new Date() }) + .where(eq(graphNodes.id, id)) + .returning(); + return node; +} + +export async function deleteNode(id: number): Promise { + const result = await db.delete(graphNodes).where(eq(graphNodes.id, id)); + return (result.rowCount ?? 0) > 0; +} + +// ─── Edges ──────────────────────────────────────────────────────────────────── + +export async function getEdges(sourceId?: number, targetId?: number): Promise { + const conditions = []; + if (sourceId) conditions.push(eq(graphEdges.sourceId, sourceId)); + if (targetId) conditions.push(eq(graphEdges.targetId, targetId)); + + return db + .select() + .from(graphEdges) + .where(conditions.length ? and(...conditions) : undefined) + .orderBy(desc(graphEdges.createdAt)); +} + +export async function createEdge(data: InsertGraphEdge): Promise { + const [edge] = await db.insert(graphEdges).values(data).returning(); + return edge; +} + +export async function deleteEdge(id: number): Promise { + const result = await db.delete(graphEdges).where(eq(graphEdges.id, id)); + return (result.rowCount ?? 0) > 0; +} + +// ─── Knowledge Base ─────────────────────────────────────────────────────────── + +export async function getKnowledgeEntries(category?: string, search?: string): Promise { + const conditions = []; + if (category) conditions.push(eq(knowledgeBase.category, category)); + if (search) { + conditions.push( + or( + ilike(knowledgeBase.title, `%${search}%`), + ilike(knowledgeBase.content, `%${search}%`) + )! + ); + } + + return db + .select() + .from(knowledgeBase) + .where(conditions.length ? and(...conditions) : undefined) + .orderBy(desc(knowledgeBase.createdAt)) + .limit(50); +} + +export async function getKnowledgeEntry(id: number): Promise { + const [entry] = await db.select().from(knowledgeBase).where(eq(knowledgeBase.id, id)); + return entry; +} + +export async function createKnowledgeEntry(data: InsertKnowledgeBaseEntry): Promise { + const [entry] = await db.insert(knowledgeBase).values(data).returning(); + + // Indexar no serviço de embeddings + indexKnowledgeEmbedding(entry.id, entry.title + "\n" + entry.content, entry.category).catch(() => {}); + + return entry; +} + +export async function updateKnowledgeEntry( + id: number, + data: Partial +): Promise { + const [entry] = await db.update(knowledgeBase).set(data).where(eq(knowledgeBase.id, id)).returning(); + return entry; +} + +export async function deleteKnowledgeEntry(id: number): Promise { + const result = await db.delete(knowledgeBase).where(eq(knowledgeBase.id, id)); + return (result.rowCount ?? 0) > 0; +} + +// ─── Busca Semântica ────────────────────────────────────────────────────────── + +export async function semanticSearch( + query: string, + nResults = 5 +): Promise<{ results: any[]; source: "embeddings" | "text_fallback" }> { + // Tenta busca vetorial no serviço de embeddings Python + try { + const response = await fetch(`${EMBEDDINGS_URL}/embeddings/search`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ query, n_results: nResults }), + signal: AbortSignal.timeout(5000), + }); + + if (response.ok) { + const data = await response.json(); + return { results: data.results || [], source: "embeddings" }; + } + } catch { + // Serviço de embeddings indisponível — fallback para busca textual + } + + // Fallback: busca textual simples no banco + const textResults = await db + .select() + .from(knowledgeBase) + .where( + or( + ilike(knowledgeBase.title, `%${query}%`), + ilike(knowledgeBase.content, `%${query}%`) + )! + ) + .limit(nResults); + + // Complementa com interações aprendidas + const interactionResults = await db + .select() + .from(learnedInteractions) + .where( + or( + ilike(learnedInteractions.question, `%${query}%`), + ilike(learnedInteractions.answer, `%${query}%`) + )! + ) + .orderBy(desc(learnedInteractions.createdAt)) + .limit(nResults); + + return { + results: [ + ...textResults.map((r) => ({ type: "knowledge", score: 0.7, data: r })), + ...interactionResults.map((r) => ({ type: "interaction", score: 0.6, data: r })), + ], + source: "text_fallback", + }; +} + +// ─── Grafo Completo para Visualização ──────────────────────────────────────── + +export async function getGraphData(tenantId?: number): Promise<{ nodes: GraphNode[]; edges: GraphEdge[] }> { + const nodes = await getNodes(tenantId, undefined, 200); + const nodeIds = nodes.map((n) => n.id); + + if (nodeIds.length === 0) return { nodes: [], edges: [] }; + + const edges = await db + .select() + .from(graphEdges) + .where( + or( + sql`${graphEdges.sourceId} = ANY(${sql.raw(`ARRAY[${nodeIds.join(",")}]`)})`, + sql`${graphEdges.targetId} = ANY(${sql.raw(`ARRAY[${nodeIds.join(",")}]`)})` + )! + ); + + return { nodes, edges }; +} + +// ─── Helpers Privados ──────────────────────────────────────────────────────── + +async function indexNodeEmbedding(nodeId: number, content: string, type: string) { + await fetch(`${EMBEDDINGS_URL}/embeddings/add`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + doc_id: `node_${nodeId}`, + document: content, + metadata: { type: "graph_node", node_type: type, node_id: nodeId }, + }), + signal: AbortSignal.timeout(10000), + }); +} + +async function indexKnowledgeEmbedding(entryId: number, content: string, category: string) { + await fetch(`${EMBEDDINGS_URL}/embeddings/add`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + doc_id: `kb_${entryId}`, + document: content, + metadata: { type: "knowledge_base", category, entry_id: entryId }, + }), + signal: AbortSignal.timeout(10000), + }); +} diff --git a/server/index.ts b/server/index.ts index 865024d..eeaa0e2 100644 --- a/server/index.ts +++ b/server/index.ts @@ -201,10 +201,15 @@ function startNodeService(name: string, scriptPath: string, port: number) { export { managedServices, restartManagedService, stopManagedService, getManagedServiceInfo, getManagedServiceLogs, startNodeService }; -startPythonService("contabil", path.join(process.cwd(), "server/python/contabil_service.py"), 8003); -startPythonService("bi", path.join(process.cwd(), "server/python/bi_engine.py"), 8004); -startPythonService("automation", path.join(process.cwd(), "server/python/automation_engine.py"), 8005); -startNodeService("communication", path.join(process.cwd(), "server/communication/engine.ts"), 8006); +// Em modo Docker cada serviço roda como container independente — não spawnar processos filhos +if (!process.env.DOCKER_MODE || process.env.DOCKER_MODE === "false") { + startPythonService("contabil", path.join(process.cwd(), "server/python/contabil_service.py"), 8003); + startPythonService("bi", path.join(process.cwd(), "server/python/bi_engine.py"), 8004); + startPythonService("automation", path.join(process.cwd(), "server/python/automation_engine.py"), 8005); + startNodeService("communication", path.join(process.cwd(), "server/communication/engine.ts"), 8006); +} else { + console.log("[services] DOCKER_MODE=true — microserviços rodando como containers independentes"); +} function startShellService(name: string, scriptPath: string, port: number) { const existing = managedServices.get(name); diff --git a/server/python/embeddings_service.py b/server/python/embeddings_service.py new file mode 100644 index 0000000..e14943e --- /dev/null +++ b/server/python/embeddings_service.py @@ -0,0 +1,250 @@ +""" +Serviço de Embeddings com pgvector + +Expõe endpoints compatíveis com o que o learning/service.ts espera: + POST /embeddings/add — adiciona documento com embedding + POST /embeddings/search — busca por similaridade semântica + GET /health — healthcheck + +Usa pgvector (PostgreSQL) em vez de ChromaDB — sem dependência extra. +""" + +import os +import json +import logging +from contextlib import asynccontextmanager +from typing import Optional + +import psycopg2 +import psycopg2.extras +from fastapi import FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from pydantic import BaseModel + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://arcadia:arcadia123@localhost:5432/arcadia") +PORT = int(os.getenv("SERVICE_PORT", "8001")) +EMBEDDING_DIM = 1536 # OpenAI text-embedding-3-small + + +# ─── Modelos Pydantic ────────────────────────────────────────────────────────── + +class AddDocumentRequest(BaseModel): + doc_id: str + document: str + metadata: Optional[dict] = {} + +class SearchRequest(BaseModel): + query: str + n_results: int = 5 + filter_type: Optional[str] = None + + +# ─── Setup do banco ──────────────────────────────────────────────────────────── + +def get_conn(): + return psycopg2.connect(DATABASE_URL) + + +def setup_database(): + """Cria a tabela de embeddings e instala pgvector se não existir.""" + try: + conn = get_conn() + cur = conn.cursor() + + cur.execute("CREATE EXTENSION IF NOT EXISTS vector;") + cur.execute(""" + CREATE TABLE IF NOT EXISTS document_embeddings ( + id SERIAL PRIMARY KEY, + doc_id TEXT NOT NULL UNIQUE, + document TEXT NOT NULL, + metadata JSONB DEFAULT '{}', + embedding vector(1536), + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ); + """) + cur.execute(""" + CREATE INDEX IF NOT EXISTS idx_embeddings_vector + ON document_embeddings + USING ivfflat (embedding vector_cosine_ops) + WITH (lists = 100); + """) + conn.commit() + cur.close() + conn.close() + logger.info("✅ Tabela document_embeddings pronta com pgvector") + except Exception as e: + logger.error(f"Erro no setup do banco: {e}") + + +# ─── Embedding via LiteLLM/OpenAI ───────────────────────────────────────────── + +def generate_embedding(text: str) -> Optional[list]: + """ + Gera embedding via LiteLLM proxy (que pode usar OpenAI ou Ollama). + Retorna None se falhar — documentos sem embedding ainda são indexados (busca textual). + """ + litellm_url = os.getenv("LITELLM_BASE_URL", "http://litellm:4000") + api_key = os.getenv("LITELLM_API_KEY", "arcadia-internal") + + try: + import urllib.request + import urllib.error + + payload = json.dumps({ + "model": "text-embedding-3-small", + "input": text[:8000] # limite de tokens + }).encode() + + req = urllib.request.Request( + f"{litellm_url}/v1/embeddings", + data=payload, + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {api_key}" + } + ) + + with urllib.request.urlopen(req, timeout=15) as response: + result = json.loads(response.read()) + return result["data"][0]["embedding"] + + except Exception as e: + logger.warning(f"Embedding não gerado (usando busca textual como fallback): {e}") + return None + + +# ─── App FastAPI ─────────────────────────────────────────────────────────────── + +@asynccontextmanager +async def lifespan(app: FastAPI): + setup_database() + yield + +app = FastAPI( + title="Arcádia Embeddings Service", + description="Serviço de embeddings com pgvector para busca semântica", + version="1.0.0", + lifespan=lifespan +) + +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/health") +def health(): + return {"status": "ok", "service": "embeddings"} + + +@app.post("/embeddings/add") +def add_document(req: AddDocumentRequest): + """Adiciona ou atualiza um documento com embedding.""" + embedding = generate_embedding(req.document) + + try: + conn = get_conn() + cur = conn.cursor() + + if embedding: + cur.execute(""" + INSERT INTO document_embeddings (doc_id, document, metadata, embedding, updated_at) + VALUES (%s, %s, %s, %s, CURRENT_TIMESTAMP) + ON CONFLICT (doc_id) DO UPDATE SET + document = EXCLUDED.document, + metadata = EXCLUDED.metadata, + embedding = EXCLUDED.embedding, + updated_at = CURRENT_TIMESTAMP + """, (req.doc_id, req.document, json.dumps(req.metadata), str(embedding))) + else: + cur.execute(""" + INSERT INTO document_embeddings (doc_id, document, metadata, updated_at) + VALUES (%s, %s, %s, CURRENT_TIMESTAMP) + ON CONFLICT (doc_id) DO UPDATE SET + document = EXCLUDED.document, + metadata = EXCLUDED.metadata, + updated_at = CURRENT_TIMESTAMP + """, (req.doc_id, req.document, json.dumps(req.metadata))) + + conn.commit() + cur.close() + conn.close() + + return { + "success": True, + "doc_id": req.doc_id, + "has_embedding": embedding is not None + } + + except Exception as e: + logger.error(f"Erro ao adicionar documento: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@app.post("/embeddings/search") +def search_documents(req: SearchRequest): + """Busca documentos similares por embedding ou texto.""" + query_embedding = generate_embedding(req.query) + + try: + conn = get_conn() + cur = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + + if query_embedding: + # Busca vetorial por similaridade coseno + cur.execute(""" + SELECT + doc_id, + document, + metadata, + 1 - (embedding <=> %s::vector) AS similarity + FROM document_embeddings + WHERE embedding IS NOT NULL + ORDER BY embedding <=> %s::vector + LIMIT %s + """, (str(query_embedding), str(query_embedding), req.n_results)) + else: + # Fallback: full-text search com LIKE + cur.execute(""" + SELECT + doc_id, + document, + metadata, + 0.5 AS similarity + FROM document_embeddings + WHERE document ILIKE %s + ORDER BY updated_at DESC + LIMIT %s + """, (f"%{req.query}%", req.n_results)) + + rows = cur.fetchall() + cur.close() + conn.close() + + return { + "results": [dict(r) for r in rows], + "query": req.query, + "method": "vector" if query_embedding else "text_fallback" + } + + except Exception as e: + logger.error(f"Erro na busca: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@app.post("/documents/add") +def add_document_compat(req: AddDocumentRequest): + """Alias compatível com o formato do python-service/.""" + return add_document(req) + + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=PORT) diff --git a/server/routes.ts b/server/routes.ts index 63569fa..fb18b06 100644 --- a/server/routes.ts +++ b/server/routes.ts @@ -58,6 +58,7 @@ import blackboardRoutes from "./blackboard/routes"; import pipelineRoutes from "./blackboard/pipelineRoutes"; import { startAllAgents } from "./blackboard/agents"; import { loadModuleRoutes } from "./modules/loader"; +import graphRoutes from "./graph/routes"; export async function registerRoutes( httpServer: Server, @@ -83,6 +84,7 @@ export async function registerRoutes( registerAutomationRoutes(app); registerAutomationEngineRoutes(app); registerBiRoutes(app); + app.use("/api/graph", graphRoutes); registerBiEngineRoutes(app); registerMetaSetRoutes(app); registerCommEngineRoutes(app);