ekuster/visualizador_instanciados
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Visualizando todo grafo com anzograph

Browse Source
This commit is contained in:
Oxy8
2026-03-10 17:21:47 -03:00
parent a0c5bec19f
commit 5badcd8d6f
17 changed files with 482 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
backend_go/config.go
View File

@@ -18,6 +18,11 @@ type Config struct {
MaxNodeLimit int
MaxEdgeLimit int
EdgeBatchSize int
FreeOSMemoryAfterSnapshot bool
LogSnapshotTimings bool
SparqlHost string
SparqlEndpoint string
SparqlUser string
@@ -45,6 +50,9 @@ func LoadConfig() (Config, error) {
DefaultEdgeLimit: envInt("DEFAULT_EDGE_LIMIT", 2_000_000),
MaxNodeLimit: envInt("MAX_NODE_LIMIT", 10_000_000),
MaxEdgeLimit: envInt("MAX_EDGE_LIMIT", 20_000_000),
EdgeBatchSize: envInt("EDGE_BATCH_SIZE", 100_000),
FreeOSMemoryAfterSnapshot: envBool("FREE_OS_MEMORY_AFTER_SNAPSHOT", false),
LogSnapshotTimings: envBool("LOG_SNAPSHOT_TIMINGS", false),
SparqlHost: envString("SPARQL_HOST", "http://anzograph:8080"),
SparqlEndpoint: envString("SPARQL_ENDPOINT", ""),
@@ -96,6 +104,12 @@ func LoadConfig() (Config, error) {
if cfg.DefaultEdgeLimit > cfg.MaxEdgeLimit {
return Config{}, fmt.Errorf("DEFAULT_EDGE_LIMIT must be <= MAX_EDGE_LIMIT")
}
if cfg.EdgeBatchSize < 1 {
return Config{}, fmt.Errorf("EDGE_BATCH_SIZE must be >= 1")
}
if cfg.EdgeBatchSize > cfg.MaxEdgeLimit {
return Config{}, fmt.Errorf("EDGE_BATCH_SIZE must be <= MAX_EDGE_LIMIT")
}
return cfg, nil
}

126
backend_go/graph_export.go
View File

@@ -5,89 +5,87 @@ type termKey struct {
key string
}
type termMeta struct {
termType string
iri string
type graphAccumulator struct {
includeBNodes bool
nodeLimit int
nodeIDByKey map[termKey]uint32
nodes []Node
edges []Edge
preds *PredicateDict
}
func graphFromSparqlBindings(
bindings []map[string]sparqlTerm,
nodeLimit int,
includeBNodes bool,
) (nodes []Node, edges []Edge) {
nodeIDByKey := map[termKey]uint32{}
nodeMeta := make([]termMeta, 0, min(nodeLimit, 4096))
func newGraphAccumulator(nodeLimit int, includeBNodes bool, edgeCapHint int, preds *PredicateDict) *graphAccumulator {
if preds == nil {
preds = NewPredicateDict(nil)
}
return &graphAccumulator{
includeBNodes: includeBNodes,
nodeLimit: nodeLimit,
nodeIDByKey: make(map[termKey]uint32),
nodes: make([]Node, 0, min(nodeLimit, 4096)),
edges: make([]Edge, 0, min(edgeCapHint, 4096)),
preds: preds,
}
}
getOrAdd := func(term sparqlTerm) (uint32, bool) {
if term.Type == "" || term.Value == "" {
return 0, false
}
if term.Type == "literal" {
return 0, false
}
var key termKey
var meta termMeta
if term.Type == "bnode" {
if !includeBNodes {
return 0, false
}
key = termKey{termType: "bnode", key: term.Value}
meta = termMeta{termType: "bnode", iri: "_:" + term.Value}
} else {
key = termKey{termType: "uri", key: term.Value}
meta = termMeta{termType: "uri", iri: term.Value}
}
if existing, ok := nodeIDByKey[key]; ok {
return existing, true
}
if len(nodeMeta) >= nodeLimit {
return 0, false
}
nid := uint32(len(nodeMeta))
nodeIDByKey[key] = nid
nodeMeta = append(nodeMeta, meta)
return nid, true
func (g *graphAccumulator) getOrAddNode(term sparqlTerm) (uint32, bool) {
if term.Type == "" || term.Value == "" {
return 0, false
}
if term.Type == "literal" {
return 0, false
}
var key termKey
var node Node
if term.Type == "bnode" {
if !g.includeBNodes {
return 0, false
}
key = termKey{termType: "bnode", key: term.Value}
node = Node{ID: 0, TermType: "bnode", IRI: "_:" + term.Value, Label: nil, X: 0, Y: 0}
} else {
key = termKey{termType: "uri", key: term.Value}
node = Node{ID: 0, TermType: "uri", IRI: term.Value, Label: nil, X: 0, Y: 0}
}
if existing, ok := g.nodeIDByKey[key]; ok {
return existing, true
}
if len(g.nodes) >= g.nodeLimit {
return 0, false
}
nid := uint32(len(g.nodes))
g.nodeIDByKey[key] = nid
node.ID = nid
g.nodes = append(g.nodes, node)
return nid, true
}
func (g *graphAccumulator) addBindings(bindings []map[string]sparqlTerm) {
for _, b := range bindings {
sTerm := b["s"]
oTerm := b["o"]
pTerm := b["p"]
sid, okS := getOrAdd(sTerm)
oid, okO := getOrAdd(oTerm)
sid, okS := g.getOrAddNode(sTerm)
oid, okO := g.getOrAddNode(oTerm)
if !okS || !okO {
continue
}
pred := pTerm.Value
if pred == "" {
predID, ok := g.preds.GetOrAdd(pTerm.Value)
if !ok {
continue
}
edges = append(edges, Edge{
Source: sid,
Target: oid,
Predicate: pred,
g.edges = append(g.edges, Edge{
Source: sid,
Target: oid,
PredicateID: predID,
})
}
nodes = make([]Node, len(nodeMeta))
for i, m := range nodeMeta {
nodes[i] = Node{
ID: uint32(i),
TermType: m.termType,
IRI: m.iri,
Label: nil,
X: 0,
Y: 0,
}
}
return nodes, edges
}
func min(a, b int) int {

35
backend_go/graph_queries/default.go
View File

@@ -2,7 +2,7 @@ package graph_queries
import "fmt"
func defaultEdgeQuery(edgeLimit int, includeBNodes bool) string {
func defaultEdgeQuery(limit int, offset int, includeBNodes bool) string {
bnodeFilter := ""
if !includeBNodes {
bnodeFilter = "FILTER(!isBlank(?s) && !isBlank(?o))"
@@ -28,7 +28,38 @@ WHERE {
FILTER(!isLiteral(?o))
%s
}
ORDER BY ?s ?p ?o
LIMIT %d
`, bnodeFilter, edgeLimit)
OFFSET %d
`, bnodeFilter, limit, offset)
}
func defaultPredicateQuery(includeBNodes bool) string {
bnodeFilter := ""
if !includeBNodes {
bnodeFilter = "FILTER(!isBlank(?s) && !isBlank(?o))"
}
return fmt.Sprintf(`
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
PREFIX owl: <http://www.w3.org/2002/07/owl#>
SELECT DISTINCT ?p
WHERE {
{
VALUES ?p { rdf:type }
?s ?p ?o .
?o rdf:type owl:Class .
}
UNION
{
VALUES ?p { rdfs:subClassOf }
?s ?p ?o .
}
FILTER(!isLiteral(?o))
%s
}
ORDER BY ?p
`, bnodeFilter)
}

25
backend_go/graph_queries/hierarchy.go
View File

@@ -2,7 +2,7 @@ package graph_queries
import "fmt"
func hierarchyEdgeQuery(edgeLimit int, includeBNodes bool) string {
func hierarchyEdgeQuery(limit int, offset int, includeBNodes bool) string {
bnodeFilter := ""
if !includeBNodes {
bnodeFilter = "FILTER(!isBlank(?s) && !isBlank(?o))"
@@ -18,7 +18,28 @@ WHERE {
FILTER(!isLiteral(?o))
%s
}
ORDER BY ?s ?p ?o
LIMIT %d
`, bnodeFilter, edgeLimit)
OFFSET %d
`, bnodeFilter, limit, offset)
}
func hierarchyPredicateQuery(includeBNodes bool) string {
bnodeFilter := ""
if !includeBNodes {
bnodeFilter = "FILTER(!isBlank(?s) && !isBlank(?o))"
}
return fmt.Sprintf(`
PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
SELECT DISTINCT ?p
WHERE {
VALUES ?p { rdfs:subClassOf }
?s ?p ?o .
FILTER(!isLiteral(?o))
%s
}
ORDER BY ?p
`, bnodeFilter)
}

7
backend_go/graph_queries/registry.go
View File

@@ -3,9 +3,9 @@ package graph_queries
const DefaultID = "default"
var definitions = []Definition{
{Meta: Meta{ID: DefaultID, Label: "Default"}, EdgeQuery: defaultEdgeQuery},
{Meta: Meta{ID: "hierarchy", Label: "Hierarchy"}, EdgeQuery: hierarchyEdgeQuery},
{Meta: Meta{ID: "types", Label: "Types"}, EdgeQuery: typesOnlyEdgeQuery},
{Meta: Meta{ID: DefaultID, Label: "Default"}, EdgeQuery: defaultEdgeQuery, PredicateQuery: defaultPredicateQuery},
{Meta: Meta{ID: "hierarchy", Label: "Hierarchy"}, EdgeQuery: hierarchyEdgeQuery, PredicateQuery: hierarchyPredicateQuery},
{Meta: Meta{ID: "types", Label: "Types"}, EdgeQuery: typesOnlyEdgeQuery, PredicateQuery: typesOnlyPredicateQuery},
}
func List() []Meta {
@@ -24,4 +24,3 @@ func Get(id string) (Definition, bool) {
}
return Definition{}, false
}

4
backend_go/graph_queries/types.go
View File

@@ -7,6 +7,6 @@ type Meta struct {
type Definition struct {
Meta Meta
EdgeQuery func(edgeLimit int, includeBNodes bool) string
EdgeQuery func(limit int, offset int, includeBNodes bool) string
PredicateQuery func(includeBNodes bool) string
}

27
backend_go/graph_queries/types_only.go
View File

@@ -2,7 +2,7 @@ package graph_queries
import "fmt"
func typesOnlyEdgeQuery(edgeLimit int, includeBNodes bool) string {
func typesOnlyEdgeQuery(limit int, offset int, includeBNodes bool) string {
bnodeFilter := ""
if !includeBNodes {
bnodeFilter = "FILTER(!isBlank(?s) && !isBlank(?o))"
@@ -20,7 +20,30 @@ WHERE {
FILTER(!isLiteral(?o))
%s
}
ORDER BY ?s ?p ?o
LIMIT %d
`, bnodeFilter, edgeLimit)
OFFSET %d
`, bnodeFilter, limit, offset)
}
func typesOnlyPredicateQuery(includeBNodes bool) string {
bnodeFilter := ""
if !includeBNodes {
bnodeFilter = "FILTER(!isBlank(?s) && !isBlank(?o))"
}
return fmt.Sprintf(`
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX owl: <http://www.w3.org/2002/07/owl#>
SELECT DISTINCT ?p
WHERE {
VALUES ?p { rdf:type }
?s ?p ?o .
?o rdf:type owl:Class .
FILTER(!isLiteral(?o))
%s
}
ORDER BY ?p
`, bnodeFilter)
}

151
backend_go/graph_snapshot.go
View File

@@ -4,13 +4,19 @@ import (
"context"
"encoding/json"
"fmt"
"log"
"runtime"
"runtime/debug"
"sort"
"strings"
"time"
graphqueries "visualizador_instanciados/backend_go/graph_queries"
)
const rdfsLabelIRI = "http://www.w3.org/2000/01/rdf-schema#label"
const (
rdfsLabelIRI = "http://www.w3.org/2000/01/rdf-schema#label"
)
func fetchGraphSnapshot(
ctx context.Context,
@@ -20,22 +26,148 @@ func fetchGraphSnapshot(
edgeLimit int,
graphQueryID string,
) (GraphResponse, error) {
start := time.Now()
logStats := func(stage string) {
if !cfg.LogSnapshotTimings {
return
}
var ms runtime.MemStats
runtime.ReadMemStats(&ms)
log.Printf(
"[snapshot] %s graph_query_id=%s node_limit=%d edge_limit=%d elapsed=%s alloc=%dMB heap_inuse=%dMB sys=%dMB numgc=%d",
stage,
graphQueryID,
nodeLimit,
edgeLimit,
time.Since(start).Truncate(time.Millisecond),
ms.Alloc/1024/1024,
ms.HeapInuse/1024/1024,
ms.Sys/1024/1024,
ms.NumGC,
)
}
def, ok := graphqueries.Get(graphQueryID)
if !ok {
return GraphResponse{}, fmt.Errorf("unknown graph_query_id: %s", graphQueryID)
}
edgesQ := def.EdgeQuery(edgeLimit, cfg.IncludeBNodes)
raw, err := sparql.Query(ctx, edgesQ)
// Build predicate dictionary (predicate IRI -> uint32 ID) before fetching edges.
preds, err := func() (*PredicateDict, error) {
logStats("predicates_query_start")
predQ := def.PredicateQuery(cfg.IncludeBNodes)
t0 := time.Now()
rawPred, err := sparql.Query(ctx, predQ)
if err != nil {
return nil, fmt.Errorf("predicates query failed: %w", err)
}
if cfg.LogSnapshotTimings {
log.Printf("[snapshot] predicates_query_returned bytes=%d query_time=%s", len(rawPred), time.Since(t0).Truncate(time.Millisecond))
}
var predRes sparqlResponse
t1 := time.Now()
if err := json.Unmarshal(rawPred, &predRes); err != nil {
return nil, fmt.Errorf("predicates unmarshal failed: %w", err)
}
if cfg.LogSnapshotTimings {
log.Printf("[snapshot] predicates_unmarshal_done bindings=%d unmarshal_time=%s", len(predRes.Results.Bindings), time.Since(t1).Truncate(time.Millisecond))
}
predicateIRIs := make([]string, 0, len(predRes.Results.Bindings))
for _, b := range predRes.Results.Bindings {
pTerm, ok := b["p"]
if !ok || pTerm.Type != "uri" || pTerm.Value == "" {
continue
}
predicateIRIs = append(predicateIRIs, pTerm.Value)
}
logStats("predicates_dict_built")
return NewPredicateDict(predicateIRIs), nil
}()
if err != nil {
return GraphResponse{}, err
}
var res sparqlResponse
if err := json.Unmarshal(raw, &res); err != nil {
return GraphResponse{}, fmt.Errorf("failed to parse SPARQL JSON: %w", err)
}
// Fetch edges in batches to avoid decoding a single huge SPARQL JSON response.
logStats("edges_batched_start")
batchSize := cfg.EdgeBatchSize
acc := newGraphAccumulator(nodeLimit, cfg.IncludeBNodes, min(edgeLimit, batchSize), preds)
nodes, edges := graphFromSparqlBindings(res.Results.Bindings, nodeLimit, cfg.IncludeBNodes)
totalBindings := 0
convAllT0 := time.Now()
for batch, offset := 0, 0; offset < edgeLimit; batch, offset = batch+1, offset+batchSize {
limit := batchSize
remaining := edgeLimit - offset
if remaining < limit {
limit = remaining
}
logStats(fmt.Sprintf("edges_batch_start batch=%d offset=%d limit=%d", batch, offset, limit))
bindings, err := func() ([]map[string]sparqlTerm, error) {
edgesQ := def.EdgeQuery(limit, offset, cfg.IncludeBNodes)
t0 := time.Now()
raw, err := sparql.Query(ctx, edgesQ)
if err != nil {
return nil, fmt.Errorf("edges query failed: %w", err)
}
if cfg.LogSnapshotTimings {
log.Printf("[snapshot] edges_batch_query_returned batch=%d offset=%d limit=%d bytes=%d query_time=%s", batch, offset, limit, len(raw), time.Since(t0).Truncate(time.Millisecond))
}
var res sparqlResponse
t1 := time.Now()
if err := json.Unmarshal(raw, &res); err != nil {
return nil, fmt.Errorf("edges unmarshal failed: %w", err)
}
if cfg.LogSnapshotTimings {
log.Printf("[snapshot] edges_batch_unmarshal_done batch=%d bindings=%d unmarshal_time=%s", batch, len(res.Results.Bindings), time.Since(t1).Truncate(time.Millisecond))
}
return res.Results.Bindings, nil
}()
if err != nil {
return GraphResponse{}, fmt.Errorf("edges batch=%d offset=%d limit=%d: %w", batch, offset, limit, err)
}
got := len(bindings)
totalBindings += got
if got == 0 {
bindings = nil
logStats(fmt.Sprintf("edges_batch_done_empty batch=%d offset=%d", batch, offset))
break
}
convT0 := time.Now()
acc.addBindings(bindings)
if cfg.LogSnapshotTimings {
log.Printf(
"[snapshot] edges_batch_convert_done batch=%d got_bindings=%d total_bindings=%d nodes=%d edges=%d convert_time=%s",
batch,
got,
totalBindings,
len(acc.nodes),
len(acc.edges),
time.Since(convT0).Truncate(time.Millisecond),
)
}
// Make the batch eligible for GC.
bindings = nil
logStats(fmt.Sprintf("edges_batch_done batch=%d offset=%d", batch, offset))
if cfg.FreeOSMemoryAfterSnapshot {
debug.FreeOSMemory()
logStats(fmt.Sprintf("edges_batch_free_os_memory_done batch=%d offset=%d", batch, offset))
}
if got < limit {
break
}
}
if cfg.LogSnapshotTimings {
log.Printf("[snapshot] convert_batches_done total_bindings=%d total_time=%s", totalBindings, time.Since(convAllT0).Truncate(time.Millisecond))
}
logStats("edges_batched_done")
nodes := acc.nodes
edges := acc.edges
// Layout: invert edges for hierarchy (target -> source).
hierEdges := make([][2]int, 0, len(edges))
@@ -82,7 +214,7 @@ func fetchGraphSnapshot(
if len(iris) > 0 {
labelByIRI, err := fetchRDFSLabels(ctx, sparql, iris, 500)
if err != nil {
return GraphResponse{}, err
return GraphResponse{}, fmt.Errorf("fetch rdfs:label failed: %w", err)
}
for i := range nodes {
if nodes[i].TermType != "uri" {
@@ -103,6 +235,7 @@ func fetchGraphSnapshot(
SparqlEndpoint: cfg.EffectiveSparqlEndpoint(),
IncludeBNodes: cfg.IncludeBNodes,
GraphQueryID: graphQueryID,
Predicates: preds.IRIs(),
NodeLimit: nodeLimit,
EdgeLimit: edgeLimit,
Nodes: len(nodes),

3
backend_go/models.go
View File

@@ -20,7 +20,7 @@ type Node struct {
type Edge struct {
Source uint32 `json:"source"`
Target uint32 `json:"target"`
Predicate string `json:"predicate"`
PredicateID uint32 `json:"predicate_id"`
}
type GraphMeta struct {
@@ -29,6 +29,7 @@ type GraphMeta struct {
SparqlEndpoint string `json:"sparql_endpoint"`
IncludeBNodes bool `json:"include_bnodes"`
GraphQueryID string `json:"graph_query_id"`
Predicates []string `json:"predicates,omitempty"` // index = predicate_id
NodeLimit int `json:"node_limit"`
EdgeLimit int `json:"edge_limit"`
Nodes int `json:"nodes"`

40
backend_go/predicate_dict.go Normal file
View File

@@ -0,0 +1,40 @@
package main
type PredicateDict struct {
idByIRI map[string]uint32
iriByID []string
}
func NewPredicateDict(predicates []string) *PredicateDict {
idByIRI := make(map[string]uint32, len(predicates))
iriByID := make([]string, 0, len(predicates))
for _, iri := range predicates {
if iri == "" {
continue
}
if _, ok := idByIRI[iri]; ok {
continue
}
id := uint32(len(iriByID))
idByIRI[iri] = id
iriByID = append(iriByID, iri)
}
return &PredicateDict{idByIRI: idByIRI, iriByID: iriByID}
}
func (d *PredicateDict) GetOrAdd(iri string) (uint32, bool) {
if iri == "" {
return 0, false
}
if id, ok := d.idByIRI[iri]; ok {
return id, true
}
id := uint32(len(d.iriByID))
d.idByIRI[iri] = id
d.iriByID = append(d.iriByID, iri)
return id, true
}
func (d *PredicateDict) IRIs() []string {
return d.iriByID
}

23
backend_go/server.go
View File

@@ -2,6 +2,7 @@ package main
import (
"fmt"
"log"
"net/http"
"strconv"
"strings"
@@ -75,11 +76,12 @@ func (s *APIServer) handleStats(w http.ResponseWriter, r *http.Request) {
}
ctx := r.Context()
snap, err := s.snapshots.Get(ctx, s.cfg.DefaultNodeLimit, s.cfg.DefaultEdgeLimit, graphqueries.DefaultID)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
snap, err := s.snapshots.Get(ctx, s.cfg.DefaultNodeLimit, s.cfg.DefaultEdgeLimit, graphqueries.DefaultID)
if err != nil {
log.Printf("handleStats: snapshot error: %v", err)
writeError(w, http.StatusInternalServerError, err.Error())
return
}
endpoint := snap.Meta.SparqlEndpoint
writeJSON(w, http.StatusOK, StatsResponse{
@@ -141,11 +143,12 @@ func (s *APIServer) handleGraph(w http.ResponseWriter, r *http.Request) {
return
}
snap, err := s.snapshots.Get(r.Context(), nodeLimit, edgeLimit, graphQueryID)
if err != nil {
if _, ok := err.(*CycleError); ok {
writeError(w, http.StatusUnprocessableEntity, err.Error())
return
snap, err := s.snapshots.Get(r.Context(), nodeLimit, edgeLimit, graphQueryID)
if err != nil {
log.Printf("handleGraph: snapshot error graph_query_id=%s node_limit=%d edge_limit=%d err=%v", graphQueryID, nodeLimit, edgeLimit, err)
if _, ok := err.(*CycleError); ok {
writeError(w, http.StatusUnprocessableEntity, err.Error())
return
}
writeError(w, http.StatusInternalServerError, err.Error())
return