backend: support external SPARQL and named-graph snapshots

backend_go/graph_export.go

@@ -5,10 +5,17 @@ type termKey struct {
 	key      string
 }
 
+type edgeKey struct {
+	source      uint32
+	target      uint32
+	predicateID uint32
+}
+
 type graphAccumulator struct {
 	includeBNodes bool
 	nodeLimit     int
 	nodeIDByKey   map[termKey]uint32
+	seenEdges     map[edgeKey]struct{}
 	nodes         []Node
 	edges         []Edge
 	preds         *PredicateDict
@@ -22,6 +29,7 @@ func newGraphAccumulator(nodeLimit int, includeBNodes bool, edgeCapHint int, pre
 		includeBNodes: includeBNodes,
 		nodeLimit:     nodeLimit,
 		nodeIDByKey:   make(map[termKey]uint32),
+		seenEdges:     make(map[edgeKey]struct{}, min(edgeCapHint, 4096)),
 		nodes:         make([]Node, 0, min(nodeLimit, 4096)),
 		edges:         make([]Edge, 0, min(edgeCapHint, 4096)),
 		preds:         preds,
@@ -63,29 +71,29 @@ func (g *graphAccumulator) getOrAddNode(term sparqlTerm) (uint32, bool) {
 	return nid, true
 }
 
-func (g *graphAccumulator) addBindings(bindings []map[string]sparqlTerm) {
-	for _, b := range bindings {
-		sTerm := b["s"]
-		oTerm := b["o"]
-		pTerm := b["p"]
-
-		sid, okS := g.getOrAddNode(sTerm)
-		oid, okO := g.getOrAddNode(oTerm)
-		if !okS || !okO {
-			continue
-		}
-
-		predID, ok := g.preds.GetOrAdd(pTerm.Value)
-		if !ok {
-			continue
-		}
-
-		g.edges = append(g.edges, Edge{
-			Source:      sid,
-			Target:      oid,
-			PredicateID: predID,
-		})
+func (g *graphAccumulator) addTripleBinding(binding sparqlTripleBinding) {
+	sid, okS := g.getOrAddNode(binding.S)
+	oid, okO := g.getOrAddNode(binding.O)
+	if !okS || !okO {
+		return
 	}
+
+	predID, ok := g.preds.GetOrAdd(binding.P.Value)
+	if !ok {
+		return
+	}
+
+	key := edgeKey{source: sid, target: oid, predicateID: predID}
+	if _, seen := g.seenEdges[key]; seen {
+		return
+	}
+	g.seenEdges[key] = struct{}{}
+
+	g.edges = append(g.edges, Edge{
+		Source:      sid,
+		Target:      oid,
+		PredicateID: predID,
+	})
 }
 
 func min(a, b int) int {