graph: stream snapshots over Arrow transport
This commit is contained in:
Oxy8
@@ -11,12 +11,10 @@ FROM golang:${GO_VERSION}-alpine AS go-builder
|
||||
|
||||
WORKDIR /src/backend_go
|
||||
|
||||
COPY backend_go/go.mod /src/backend_go/go.mod
|
||||
|
||||
RUN go mod download
|
||||
|
||||
COPY backend_go /src/backend_go
|
||||
|
||||
RUN go mod tidy
|
||||
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/backend ./
|
||||
|
||||
FROM debian:bookworm-slim
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
module visualizador_instanciados/backend_go
|
||||
|
||||
go 1.22
|
||||
|
||||
require github.com/apache/arrow/go/v17 v17.0.0
|
||||
|
||||
359
backend_go/graph_arrow.go
Normal file
359
backend_go/graph_arrow.go
Normal file
@@ -0,0 +1,359 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/apache/arrow/go/v17/arrow"
|
||||
"github.com/apache/arrow/go/v17/arrow/array"
|
||||
"github.com/apache/arrow/go/v17/arrow/ipc"
|
||||
"github.com/apache/arrow/go/v17/arrow/memory"
|
||||
)
|
||||
|
||||
const graphTransportVersion = "1"
|
||||
|
||||
const (
|
||||
graphArrowMetaBackendField = iota
|
||||
graphArrowMetaGraphQueryIDField
|
||||
graphArrowMetaNodeLimitField
|
||||
graphArrowMetaEdgeLimitField
|
||||
graphArrowMetaNodesField
|
||||
graphArrowMetaEdgesField
|
||||
graphArrowMetaRouteLineSegmentsField
|
||||
graphArrowNodeIDField
|
||||
graphArrowNodeXField
|
||||
graphArrowNodeYField
|
||||
graphArrowNodeIRIField
|
||||
graphArrowNodeLabelField
|
||||
graphArrowEdgeSourceField
|
||||
graphArrowEdgeTargetField
|
||||
graphArrowRouteX1Field
|
||||
graphArrowRouteY1Field
|
||||
graphArrowRouteX2Field
|
||||
graphArrowRouteY2Field
|
||||
)
|
||||
|
||||
var graphArrowSchema = arrow.NewSchema([]arrow.Field{
|
||||
{Name: "meta_backend", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "meta_graph_query_id", Type: arrow.BinaryTypes.String, Nullable: true},
|
||||
{Name: "meta_node_limit", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "meta_edge_limit", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "meta_nodes", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "meta_edges", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "meta_route_line_segments", Type: arrow.PrimitiveTypes.Int32, Nullable: true},
|
||||
{Name: "node_id", Type: arrow.ListOf(arrow.PrimitiveTypes.Uint32), Nullable: true},
|
||||
{Name: "node_x", Type: arrow.ListOf(arrow.PrimitiveTypes.Float32), Nullable: true},
|
||||
{Name: "node_y", Type: arrow.ListOf(arrow.PrimitiveTypes.Float32), Nullable: true},
|
||||
{Name: "node_iri", Type: arrow.ListOf(arrow.BinaryTypes.String), Nullable: true},
|
||||
{Name: "node_label", Type: arrow.ListOf(arrow.BinaryTypes.String), Nullable: true},
|
||||
{Name: "edge_source", Type: arrow.ListOf(arrow.PrimitiveTypes.Uint32), Nullable: true},
|
||||
{Name: "edge_target", Type: arrow.ListOf(arrow.PrimitiveTypes.Uint32), Nullable: true},
|
||||
{Name: "route_x1", Type: arrow.ListOf(arrow.PrimitiveTypes.Float32), Nullable: true},
|
||||
{Name: "route_y1", Type: arrow.ListOf(arrow.PrimitiveTypes.Float32), Nullable: true},
|
||||
{Name: "route_x2", Type: arrow.ListOf(arrow.PrimitiveTypes.Float32), Nullable: true},
|
||||
{Name: "route_y2", Type: arrow.ListOf(arrow.PrimitiveTypes.Float32), Nullable: true},
|
||||
}, nil)
|
||||
|
||||
type graphArrowRouteLines struct {
|
||||
X1 []float32
|
||||
Y1 []float32
|
||||
X2 []float32
|
||||
Y2 []float32
|
||||
}
|
||||
|
||||
func writeGraphArrow(w http.ResponseWriter, snap GraphResponse) error {
|
||||
start := time.Now()
|
||||
routes := flattenGraphRouteLines(snap.RouteSegments)
|
||||
meta := snap.Meta
|
||||
if meta == nil {
|
||||
meta = &GraphMeta{
|
||||
GraphQueryID: "default",
|
||||
NodeLimit: len(snap.Nodes),
|
||||
EdgeLimit: len(snap.Edges),
|
||||
Nodes: len(snap.Nodes),
|
||||
Edges: len(snap.Edges),
|
||||
}
|
||||
}
|
||||
|
||||
alloc := memory.NewGoAllocator()
|
||||
w.Header().Set("Content-Type", "application/vnd.apache.arrow.stream")
|
||||
w.Header().Set("X-Graph-Transport-Version", graphTransportVersion)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
writer := ipc.NewWriter(w, ipc.WithSchema(graphArrowSchema))
|
||||
closed := false
|
||||
defer func() {
|
||||
if !closed {
|
||||
_ = writer.Close()
|
||||
}
|
||||
}()
|
||||
|
||||
if err := writeGraphArrowMetaBatch(writer, alloc, meta, len(routes.X1)); err != nil {
|
||||
return fmt.Errorf("write meta batch failed: %w", err)
|
||||
}
|
||||
if err := writeGraphArrowNodeBatch(writer, alloc, snap.Nodes); err != nil {
|
||||
return fmt.Errorf("write node batch failed: %w", err)
|
||||
}
|
||||
if err := writeGraphArrowEdgeBatch(writer, alloc, snap.Edges); err != nil {
|
||||
return fmt.Errorf("write edge batch failed: %w", err)
|
||||
}
|
||||
if err := writeGraphArrowRouteBatch(writer, alloc, routes); err != nil {
|
||||
return fmt.Errorf("write route batch failed: %w", err)
|
||||
}
|
||||
if err := writer.Close(); err != nil {
|
||||
return fmt.Errorf("close arrow writer failed: %w", err)
|
||||
}
|
||||
closed = true
|
||||
|
||||
log.Printf(
|
||||
"[graph-arrow] encode_done graph_query_id=%s nodes=%d edges=%d route_line_segments=%d encode_time=%s",
|
||||
meta.GraphQueryID,
|
||||
len(snap.Nodes),
|
||||
len(snap.Edges),
|
||||
len(routes.X1),
|
||||
time.Since(start).Truncate(time.Millisecond),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
func writeGraphArrowMetaBatch(writer *ipc.Writer, alloc memory.Allocator, meta *GraphMeta, routeLineSegments int) error {
|
||||
builder := array.NewRecordBuilder(alloc, graphArrowSchema)
|
||||
defer builder.Release()
|
||||
|
||||
appendStringBuilderValue(builder.Field(graphArrowMetaBackendField), meta.Backend)
|
||||
appendStringBuilderValue(builder.Field(graphArrowMetaGraphQueryIDField), meta.GraphQueryID)
|
||||
appendInt32BuilderValue(builder.Field(graphArrowMetaNodeLimitField), int32(meta.NodeLimit))
|
||||
appendInt32BuilderValue(builder.Field(graphArrowMetaEdgeLimitField), int32(meta.EdgeLimit))
|
||||
appendInt32BuilderValue(builder.Field(graphArrowMetaNodesField), int32(meta.Nodes))
|
||||
appendInt32BuilderValue(builder.Field(graphArrowMetaEdgesField), int32(meta.Edges))
|
||||
appendInt32BuilderValue(builder.Field(graphArrowMetaRouteLineSegmentsField), int32(routeLineSegments))
|
||||
|
||||
appendNullTopLevel(builder.Field(graphArrowNodeIDField))
|
||||
appendNullTopLevel(builder.Field(graphArrowNodeXField))
|
||||
appendNullTopLevel(builder.Field(graphArrowNodeYField))
|
||||
appendNullTopLevel(builder.Field(graphArrowNodeIRIField))
|
||||
appendNullTopLevel(builder.Field(graphArrowNodeLabelField))
|
||||
appendNullTopLevel(builder.Field(graphArrowEdgeSourceField))
|
||||
appendNullTopLevel(builder.Field(graphArrowEdgeTargetField))
|
||||
appendNullTopLevel(builder.Field(graphArrowRouteX1Field))
|
||||
appendNullTopLevel(builder.Field(graphArrowRouteY1Field))
|
||||
appendNullTopLevel(builder.Field(graphArrowRouteX2Field))
|
||||
appendNullTopLevel(builder.Field(graphArrowRouteY2Field))
|
||||
|
||||
return writeGraphArrowRecord(writer, builder)
|
||||
}
|
||||
|
||||
func writeGraphArrowNodeBatch(writer *ipc.Writer, alloc memory.Allocator, nodes []Node) error {
|
||||
builder := array.NewRecordBuilder(alloc, graphArrowSchema)
|
||||
defer builder.Release()
|
||||
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaBackendField))
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaGraphQueryIDField))
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaNodeLimitField))
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaEdgeLimitField))
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaNodesField))
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaEdgesField))
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaRouteLineSegmentsField))
|
||||
|
||||
appendUint32List(builder.Field(graphArrowNodeIDField), func(valueBuilder *array.Uint32Builder) {
|
||||
for _, node := range nodes {
|
||||
valueBuilder.Append(node.ID)
|
||||
}
|
||||
})
|
||||
appendFloat32List(builder.Field(graphArrowNodeXField), func(valueBuilder *array.Float32Builder) {
|
||||
for _, node := range nodes {
|
||||
valueBuilder.Append(float32(node.X))
|
||||
}
|
||||
})
|
||||
appendFloat32List(builder.Field(graphArrowNodeYField), func(valueBuilder *array.Float32Builder) {
|
||||
for _, node := range nodes {
|
||||
valueBuilder.Append(float32(node.Y))
|
||||
}
|
||||
})
|
||||
appendStringList(builder.Field(graphArrowNodeIRIField), func(valueBuilder *array.StringBuilder) {
|
||||
for _, node := range nodes {
|
||||
valueBuilder.Append(node.IRI)
|
||||
}
|
||||
})
|
||||
appendNullableStringList(builder.Field(graphArrowNodeLabelField), func(valueBuilder *array.StringBuilder) {
|
||||
for _, node := range nodes {
|
||||
if node.Label == nil {
|
||||
valueBuilder.AppendNull()
|
||||
continue
|
||||
}
|
||||
valueBuilder.Append(*node.Label)
|
||||
}
|
||||
})
|
||||
|
||||
appendNullTopLevel(builder.Field(graphArrowEdgeSourceField))
|
||||
appendNullTopLevel(builder.Field(graphArrowEdgeTargetField))
|
||||
appendNullTopLevel(builder.Field(graphArrowRouteX1Field))
|
||||
appendNullTopLevel(builder.Field(graphArrowRouteY1Field))
|
||||
appendNullTopLevel(builder.Field(graphArrowRouteX2Field))
|
||||
appendNullTopLevel(builder.Field(graphArrowRouteY2Field))
|
||||
|
||||
return writeGraphArrowRecord(writer, builder)
|
||||
}
|
||||
|
||||
func writeGraphArrowEdgeBatch(writer *ipc.Writer, alloc memory.Allocator, edges []Edge) error {
|
||||
builder := array.NewRecordBuilder(alloc, graphArrowSchema)
|
||||
defer builder.Release()
|
||||
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaBackendField))
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaGraphQueryIDField))
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaNodeLimitField))
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaEdgeLimitField))
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaNodesField))
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaEdgesField))
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaRouteLineSegmentsField))
|
||||
appendNullTopLevel(builder.Field(graphArrowNodeIDField))
|
||||
appendNullTopLevel(builder.Field(graphArrowNodeXField))
|
||||
appendNullTopLevel(builder.Field(graphArrowNodeYField))
|
||||
appendNullTopLevel(builder.Field(graphArrowNodeIRIField))
|
||||
appendNullTopLevel(builder.Field(graphArrowNodeLabelField))
|
||||
|
||||
appendUint32List(builder.Field(graphArrowEdgeSourceField), func(valueBuilder *array.Uint32Builder) {
|
||||
for _, edge := range edges {
|
||||
valueBuilder.Append(edge.Source)
|
||||
}
|
||||
})
|
||||
appendUint32List(builder.Field(graphArrowEdgeTargetField), func(valueBuilder *array.Uint32Builder) {
|
||||
for _, edge := range edges {
|
||||
valueBuilder.Append(edge.Target)
|
||||
}
|
||||
})
|
||||
|
||||
appendNullTopLevel(builder.Field(graphArrowRouteX1Field))
|
||||
appendNullTopLevel(builder.Field(graphArrowRouteY1Field))
|
||||
appendNullTopLevel(builder.Field(graphArrowRouteX2Field))
|
||||
appendNullTopLevel(builder.Field(graphArrowRouteY2Field))
|
||||
|
||||
return writeGraphArrowRecord(writer, builder)
|
||||
}
|
||||
|
||||
func writeGraphArrowRouteBatch(writer *ipc.Writer, alloc memory.Allocator, routes graphArrowRouteLines) error {
|
||||
builder := array.NewRecordBuilder(alloc, graphArrowSchema)
|
||||
defer builder.Release()
|
||||
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaBackendField))
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaGraphQueryIDField))
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaNodeLimitField))
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaEdgeLimitField))
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaNodesField))
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaEdgesField))
|
||||
appendNullTopLevel(builder.Field(graphArrowMetaRouteLineSegmentsField))
|
||||
appendNullTopLevel(builder.Field(graphArrowNodeIDField))
|
||||
appendNullTopLevel(builder.Field(graphArrowNodeXField))
|
||||
appendNullTopLevel(builder.Field(graphArrowNodeYField))
|
||||
appendNullTopLevel(builder.Field(graphArrowNodeIRIField))
|
||||
appendNullTopLevel(builder.Field(graphArrowNodeLabelField))
|
||||
appendNullTopLevel(builder.Field(graphArrowEdgeSourceField))
|
||||
appendNullTopLevel(builder.Field(graphArrowEdgeTargetField))
|
||||
|
||||
appendFloat32List(builder.Field(graphArrowRouteX1Field), func(valueBuilder *array.Float32Builder) {
|
||||
for _, value := range routes.X1 {
|
||||
valueBuilder.Append(value)
|
||||
}
|
||||
})
|
||||
appendFloat32List(builder.Field(graphArrowRouteY1Field), func(valueBuilder *array.Float32Builder) {
|
||||
for _, value := range routes.Y1 {
|
||||
valueBuilder.Append(value)
|
||||
}
|
||||
})
|
||||
appendFloat32List(builder.Field(graphArrowRouteX2Field), func(valueBuilder *array.Float32Builder) {
|
||||
for _, value := range routes.X2 {
|
||||
valueBuilder.Append(value)
|
||||
}
|
||||
})
|
||||
appendFloat32List(builder.Field(graphArrowRouteY2Field), func(valueBuilder *array.Float32Builder) {
|
||||
for _, value := range routes.Y2 {
|
||||
valueBuilder.Append(value)
|
||||
}
|
||||
})
|
||||
|
||||
return writeGraphArrowRecord(writer, builder)
|
||||
}
|
||||
|
||||
func writeGraphArrowRecord(writer *ipc.Writer, builder *array.RecordBuilder) error {
|
||||
record := builder.NewRecord()
|
||||
defer record.Release()
|
||||
return writer.Write(record)
|
||||
}
|
||||
|
||||
func appendStringBuilderValue(builder array.Builder, value string) {
|
||||
builder.(*array.StringBuilder).Append(value)
|
||||
}
|
||||
|
||||
func appendInt32BuilderValue(builder array.Builder, value int32) {
|
||||
builder.(*array.Int32Builder).Append(value)
|
||||
}
|
||||
|
||||
func appendUint32List(builder array.Builder, appendValues func(*array.Uint32Builder)) {
|
||||
listBuilder := builder.(*array.ListBuilder)
|
||||
listBuilder.Append(true)
|
||||
valueBuilder := listBuilder.ValueBuilder().(*array.Uint32Builder)
|
||||
appendValues(valueBuilder)
|
||||
}
|
||||
|
||||
func appendFloat32List(builder array.Builder, appendValues func(*array.Float32Builder)) {
|
||||
listBuilder := builder.(*array.ListBuilder)
|
||||
listBuilder.Append(true)
|
||||
valueBuilder := listBuilder.ValueBuilder().(*array.Float32Builder)
|
||||
appendValues(valueBuilder)
|
||||
}
|
||||
|
||||
func appendStringList(builder array.Builder, appendValues func(*array.StringBuilder)) {
|
||||
listBuilder := builder.(*array.ListBuilder)
|
||||
listBuilder.Append(true)
|
||||
valueBuilder := listBuilder.ValueBuilder().(*array.StringBuilder)
|
||||
appendValues(valueBuilder)
|
||||
}
|
||||
|
||||
func appendNullableStringList(builder array.Builder, appendValues func(*array.StringBuilder)) {
|
||||
listBuilder := builder.(*array.ListBuilder)
|
||||
listBuilder.Append(true)
|
||||
valueBuilder := listBuilder.ValueBuilder().(*array.StringBuilder)
|
||||
appendValues(valueBuilder)
|
||||
}
|
||||
|
||||
func appendNullTopLevel(builder array.Builder) {
|
||||
switch typed := builder.(type) {
|
||||
case *array.StringBuilder:
|
||||
typed.AppendNull()
|
||||
case *array.Int32Builder:
|
||||
typed.AppendNull()
|
||||
case *array.ListBuilder:
|
||||
typed.AppendNull()
|
||||
default:
|
||||
panic(fmt.Sprintf("unsupported top-level builder type %T", builder))
|
||||
}
|
||||
}
|
||||
|
||||
func flattenGraphRouteLines(routeSegments []RouteSegment) graphArrowRouteLines {
|
||||
lineCount := 0
|
||||
for _, route := range routeSegments {
|
||||
if len(route.Points) > 1 {
|
||||
lineCount += len(route.Points) - 1
|
||||
}
|
||||
}
|
||||
|
||||
out := graphArrowRouteLines{
|
||||
X1: make([]float32, 0, lineCount),
|
||||
Y1: make([]float32, 0, lineCount),
|
||||
X2: make([]float32, 0, lineCount),
|
||||
Y2: make([]float32, 0, lineCount),
|
||||
}
|
||||
for _, route := range routeSegments {
|
||||
for i := 1; i < len(route.Points); i++ {
|
||||
prev := route.Points[i-1]
|
||||
curr := route.Points[i]
|
||||
out.X1 = append(out.X1, float32(prev.X))
|
||||
out.Y1 = append(out.Y1, float32(prev.Y))
|
||||
out.X2 = append(out.X2, float32(curr.X))
|
||||
out.Y2 = append(out.Y2, float32(curr.Y))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -155,7 +155,9 @@ func (s *APIServer) handleGraph(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, snap)
|
||||
if err := writeGraphArrow(w, snap); err != nil {
|
||||
log.Printf("handleGraph: arrow encode error graph_query_id=%s node_limit=%d edge_limit=%d err=%v", graphQueryID, nodeLimit, edgeLimit, err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *APIServer) handleGraphQueries(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
263
frontend/package-lock.json
generated
263
frontend/package-lock.json
generated
@@ -10,6 +10,7 @@
|
||||
"dependencies": {
|
||||
"@cosmos.gl/graph": "^2.6.4",
|
||||
"@webgpu/types": "^0.1.69",
|
||||
"apache-arrow": "^21.1.0",
|
||||
"clsx": "2.1.1",
|
||||
"react": "19.2.3",
|
||||
"react-dom": "19.2.3",
|
||||
@@ -1184,6 +1185,15 @@
|
||||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/@swc/helpers": {
|
||||
"version": "0.5.20",
|
||||
"resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.20.tgz",
|
||||
"integrity": "sha512-2egEBHUMasdypIzrprsu8g+OEVd7Vp2MM3a2eVlM/cyFYto0nGz5BX5BTgh/ShZZI9ed+ozEq+Ngt+rgmUs8tw==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"tslib": "^2.8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@tailwindcss/node": {
|
||||
"version": "4.1.17",
|
||||
"resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.1.17.tgz",
|
||||
@@ -1501,6 +1511,18 @@
|
||||
"@babel/types": "^7.28.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/command-line-args": {
|
||||
"version": "5.2.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.3.tgz",
|
||||
"integrity": "sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/command-line-usage": {
|
||||
"version": "5.0.4",
|
||||
"resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.4.tgz",
|
||||
"integrity": "sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/estree": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz",
|
||||
@@ -1572,6 +1594,65 @@
|
||||
"integrity": "sha512-RPmm6kgRbI8e98zSD3RVACvnuktIja5+yLgDAkTmxLr90BEwdTXRQWNLF3ETTTyH/8mKhznZuN5AveXYFEsMGQ==",
|
||||
"license": "BSD-3-Clause"
|
||||
},
|
||||
"node_modules/ansi-styles": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz",
|
||||
"integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"color-convert": "^2.0.1"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/chalk/ansi-styles?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/apache-arrow": {
|
||||
"version": "21.1.0",
|
||||
"resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-21.1.0.tgz",
|
||||
"integrity": "sha512-kQrYLxhC+NTVVZ4CCzGF6L/uPVOzJmD1T3XgbiUnP7oTeVFOFgEUu6IKNwCDkpFoBVqDKQivlX4RUFqqnWFlEA==",
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"@swc/helpers": "^0.5.11",
|
||||
"@types/command-line-args": "^5.2.3",
|
||||
"@types/command-line-usage": "^5.0.4",
|
||||
"@types/node": "^24.0.3",
|
||||
"command-line-args": "^6.0.1",
|
||||
"command-line-usage": "^7.0.1",
|
||||
"flatbuffers": "^25.1.24",
|
||||
"json-bignum": "^0.0.3",
|
||||
"tslib": "^2.6.2"
|
||||
},
|
||||
"bin": {
|
||||
"arrow2csv": "bin/arrow2csv.js"
|
||||
}
|
||||
},
|
||||
"node_modules/apache-arrow/node_modules/@types/node": {
|
||||
"version": "24.12.0",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.0.tgz",
|
||||
"integrity": "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"undici-types": "~7.16.0"
|
||||
}
|
||||
},
|
||||
"node_modules/apache-arrow/node_modules/undici-types": {
|
||||
"version": "7.16.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz",
|
||||
"integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/array-back": {
|
||||
"version": "6.2.3",
|
||||
"resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.3.tgz",
|
||||
"integrity": "sha512-SGDvmg6QTYiTxCBkYVmThcoa67uLl35pyzRHdpCGBOcqFy6BtwnphoFPk7LhJshD+Yk1Kt35WGWeZPTgwR4Fhw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12.17"
|
||||
}
|
||||
},
|
||||
"node_modules/baseline-browser-mapping": {
|
||||
"version": "2.9.19",
|
||||
"resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.19.tgz",
|
||||
@@ -1650,6 +1731,37 @@
|
||||
],
|
||||
"license": "CC-BY-4.0"
|
||||
},
|
||||
"node_modules/chalk": {
|
||||
"version": "4.1.2",
|
||||
"resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz",
|
||||
"integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"ansi-styles": "^4.1.0",
|
||||
"supports-color": "^7.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=10"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/chalk/chalk?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/chalk-template": {
|
||||
"version": "0.4.0",
|
||||
"resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz",
|
||||
"integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"chalk": "^4.1.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://github.com/chalk/chalk-template?sponsor=1"
|
||||
}
|
||||
},
|
||||
"node_modules/clsx": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
|
||||
@@ -1659,6 +1771,62 @@
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/color-convert": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
|
||||
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"color-name": "~1.1.4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=7.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/color-name": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
|
||||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/command-line-args": {
|
||||
"version": "6.0.2",
|
||||
"resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-6.0.2.tgz",
|
||||
"integrity": "sha512-AIjYVxrV9X752LmPDLbVYv8aMCuHPSLZJXEo2qo/xJfv+NYhaZ4sMSF01rM+gHPaMgvPM0l5D/F+Qx+i2WfSmQ==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"array-back": "^6.2.3",
|
||||
"find-replace": "^5.0.2",
|
||||
"lodash.camelcase": "^4.3.0",
|
||||
"typical": "^7.3.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.20"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@75lb/nature": "latest"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@75lb/nature": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/command-line-usage": {
|
||||
"version": "7.0.4",
|
||||
"resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.4.tgz",
|
||||
"integrity": "sha512-85UdvzTNx/+s5CkSgBm/0hzP80RFHAa7PsfeADE5ezZF3uHz3/Tqj9gIKGT9PTtpycc3Ua64T0oVulGfKxzfqg==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"array-back": "^6.2.2",
|
||||
"chalk-template": "^0.4.0",
|
||||
"table-layout": "^4.1.1",
|
||||
"typical": "^7.3.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.20.0"
|
||||
}
|
||||
},
|
||||
"node_modules/convert-source-map": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz",
|
||||
@@ -1980,6 +2148,29 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/find-replace": {
|
||||
"version": "5.0.2",
|
||||
"resolved": "https://registry.npmjs.org/find-replace/-/find-replace-5.0.2.tgz",
|
||||
"integrity": "sha512-Y45BAiE3mz2QsrN2fb5QEtO4qb44NcS7en/0y9PEVsg351HsLeVclP8QPMH79Le9sH3rs5RSwJu99W0WPZO43Q==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=14"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@75lb/nature": "latest"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@75lb/nature": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/flatbuffers": {
|
||||
"version": "25.9.23",
|
||||
"resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz",
|
||||
"integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==",
|
||||
"license": "Apache-2.0"
|
||||
},
|
||||
"node_modules/fsevents": {
|
||||
"version": "2.3.3",
|
||||
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
|
||||
@@ -2037,6 +2228,15 @@
|
||||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/has-flag": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz",
|
||||
"integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/internmap": {
|
||||
"version": "2.0.3",
|
||||
"resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz",
|
||||
@@ -2086,6 +2286,14 @@
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/json-bignum": {
|
||||
"version": "0.0.3",
|
||||
"resolved": "https://registry.npmjs.org/json-bignum/-/json-bignum-0.0.3.tgz",
|
||||
"integrity": "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==",
|
||||
"engines": {
|
||||
"node": ">=0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/json5": {
|
||||
"version": "2.2.3",
|
||||
"resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz",
|
||||
@@ -2360,6 +2568,12 @@
|
||||
"url": "https://opencollective.com/parcel"
|
||||
}
|
||||
},
|
||||
"node_modules/lodash.camelcase": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
|
||||
"integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/lru-cache": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz",
|
||||
@@ -2625,6 +2839,31 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/supports-color": {
|
||||
"version": "7.2.0",
|
||||
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz",
|
||||
"integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"has-flag": "^4.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/table-layout": {
|
||||
"version": "4.1.1",
|
||||
"resolved": "https://registry.npmjs.org/table-layout/-/table-layout-4.1.1.tgz",
|
||||
"integrity": "sha512-iK5/YhZxq5GO5z8wb0bY1317uDF3Zjpha0QFFLA8/trAoiLbQD0HUbMesEaxyzUgDxi2QlcbM8IvqOlEjgoXBA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"array-back": "^6.2.2",
|
||||
"wordwrapjs": "^5.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=12.17"
|
||||
}
|
||||
},
|
||||
"node_modules/tailwind-merge": {
|
||||
"version": "3.4.0",
|
||||
"resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-3.4.0.tgz",
|
||||
@@ -2686,6 +2925,12 @@
|
||||
"node": ">=8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/tslib": {
|
||||
"version": "2.8.1",
|
||||
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz",
|
||||
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==",
|
||||
"license": "0BSD"
|
||||
},
|
||||
"node_modules/tsx": {
|
||||
"version": "4.21.0",
|
||||
"resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz",
|
||||
@@ -3204,6 +3449,15 @@
|
||||
"node": ">=14.17"
|
||||
}
|
||||
},
|
||||
"node_modules/typical": {
|
||||
"version": "7.3.0",
|
||||
"resolved": "https://registry.npmjs.org/typical/-/typical-7.3.0.tgz",
|
||||
"integrity": "sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12.17"
|
||||
}
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "6.21.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
|
||||
@@ -3334,6 +3588,15 @@
|
||||
"vite": "^5.4.11 || ^6.0.0 || ^7.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/wordwrapjs": {
|
||||
"version": "5.1.1",
|
||||
"resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-5.1.1.tgz",
|
||||
"integrity": "sha512-0yweIbkINJodk27gX9LBGMzyQdBDan3s/dEAiwBOj+Mf0PPyWL6/rikalkv8EeD0E8jm4o5RXEOrFTP3NXbhJg==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12.17"
|
||||
}
|
||||
},
|
||||
"node_modules/yallist": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
"dependencies": {
|
||||
"@cosmos.gl/graph": "^2.6.4",
|
||||
"@webgpu/types": "^0.1.69",
|
||||
"apache-arrow": "^21.1.0",
|
||||
"clsx": "2.1.1",
|
||||
"react": "19.2.3",
|
||||
"react-dom": "19.2.3",
|
||||
|
||||
@@ -4,77 +4,21 @@ import { fetchGraphQueries } from "./graph_queries";
|
||||
import type { GraphQueryMeta } from "./graph_queries";
|
||||
import { fetchSelectionQueries, runSelectionQuery, runSelectionTripleQuery } from "./selection_queries";
|
||||
import { cosmosRuntimeConfig } from "./cosmos_config";
|
||||
import type { GraphMeta, GraphRoutePoint, GraphRouteSegment, SelectionQueryMeta, SelectionTriple } from "./selection_queries";
|
||||
import type { GraphMeta, SelectionQueryMeta, SelectionTriple } from "./selection_queries";
|
||||
import { TripleGraphView } from "./TripleGraphView";
|
||||
import { buildTripleGraphModel, type TripleGraphModel } from "./triple_graph";
|
||||
import { readGraphArrow } from "./graph_arrow";
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((r) => setTimeout(r, ms));
|
||||
}
|
||||
|
||||
type GraphNodeMeta = {
|
||||
id?: number;
|
||||
iri?: string;
|
||||
label?: string;
|
||||
x?: number;
|
||||
y?: number;
|
||||
type GraphNodeLookup = {
|
||||
vertexIds: Uint32Array;
|
||||
labels: (string | undefined)[];
|
||||
iris: string[];
|
||||
};
|
||||
|
||||
function graphRoutePoint(value: unknown): GraphRoutePoint | null {
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const record = value as Record<string, unknown>;
|
||||
if (typeof record.x !== "number" || typeof record.y !== "number") return null;
|
||||
return {
|
||||
x: record.x,
|
||||
y: record.y,
|
||||
};
|
||||
}
|
||||
|
||||
function graphRouteSegmentArray(value: unknown): GraphRouteSegment[] {
|
||||
if (!Array.isArray(value)) return [];
|
||||
const out: GraphRouteSegment[] = [];
|
||||
for (const item of value) {
|
||||
if (!item || typeof item !== "object") continue;
|
||||
const record = item as Record<string, unknown>;
|
||||
if (typeof record.edge_index !== "number" || typeof record.kind !== "string") continue;
|
||||
if (!Array.isArray(record.points)) continue;
|
||||
const points: GraphRoutePoint[] = [];
|
||||
for (const point of record.points) {
|
||||
const parsed = graphRoutePoint(point);
|
||||
if (!parsed) continue;
|
||||
points.push(parsed);
|
||||
}
|
||||
if (points.length < 2) continue;
|
||||
out.push({
|
||||
edge_index: record.edge_index,
|
||||
kind: record.kind,
|
||||
points,
|
||||
});
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function buildRouteLineVertices(routeSegments: GraphRouteSegment[]): Float32Array {
|
||||
let lineCount = 0;
|
||||
for (const route of routeSegments) {
|
||||
lineCount += Math.max(0, route.points.length - 1);
|
||||
}
|
||||
|
||||
const out = new Float32Array(lineCount * 4);
|
||||
let offset = 0;
|
||||
for (const route of routeSegments) {
|
||||
for (let i = 1; i < route.points.length; i++) {
|
||||
const previous = route.points[i - 1];
|
||||
const current = route.points[i];
|
||||
out[offset++] = previous.x;
|
||||
out[offset++] = previous.y;
|
||||
out[offset++] = current.x;
|
||||
out[offset++] = current.y;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
type TripleResultState = {
|
||||
status: "idle" | "loading" | "ready" | "error";
|
||||
queryId: string;
|
||||
@@ -92,6 +36,37 @@ function idleTripleResult(queryId: string): TripleResultState {
|
||||
};
|
||||
}
|
||||
|
||||
function formatBytes(bytes: number): string {
|
||||
if (!Number.isFinite(bytes) || bytes <= 0) return "0 B";
|
||||
const units = ["B", "KB", "MB", "GB", "TB"];
|
||||
let value = bytes;
|
||||
let unitIndex = 0;
|
||||
while (value >= 1024 && unitIndex < units.length - 1) {
|
||||
value /= 1024;
|
||||
unitIndex++;
|
||||
}
|
||||
return `${value.toFixed(value >= 100 || unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`;
|
||||
}
|
||||
|
||||
function estimateFrontendTypedArrayBytes(nodeCount: number, edgeCount: number, routeLineFloatCount: number): { app: number; renderer: number; total: number } {
|
||||
const app =
|
||||
nodeCount * Float32Array.BYTES_PER_ELEMENT * 2 +
|
||||
nodeCount * Uint32Array.BYTES_PER_ELEMENT +
|
||||
edgeCount * Uint32Array.BYTES_PER_ELEMENT * 2 +
|
||||
routeLineFloatCount * Float32Array.BYTES_PER_ELEMENT;
|
||||
|
||||
const renderer =
|
||||
nodeCount * Float32Array.BYTES_PER_ELEMENT * 2 +
|
||||
nodeCount * Uint32Array.BYTES_PER_ELEMENT * 4 +
|
||||
edgeCount * Uint32Array.BYTES_PER_ELEMENT * 4;
|
||||
|
||||
return {
|
||||
app,
|
||||
renderer,
|
||||
total: app + renderer,
|
||||
};
|
||||
}
|
||||
|
||||
export default function App() {
|
||||
const canvasRef = useRef<HTMLCanvasElement>(null);
|
||||
const rendererRef = useRef<Renderer | null>(null);
|
||||
@@ -121,14 +96,34 @@ export default function App() {
|
||||
|
||||
// Store mouse position in a ref so it can be accessed in render loop without re-renders
|
||||
const mousePos = useRef({ x: 0, y: 0 });
|
||||
const nodesRef = useRef<GraphNodeMeta[]>([]);
|
||||
const nodeLookupRef = useRef<GraphNodeLookup | null>(null);
|
||||
|
||||
async function loadGraph(graphQueryId: string, signal: AbortSignal): Promise<void> {
|
||||
const renderer = rendererRef.current;
|
||||
if (!renderer) return;
|
||||
|
||||
setStatus("Fetching graph…");
|
||||
const loadStartedAt = performance.now();
|
||||
const logPhase = (phase: string, extra?: Record<string, unknown>) => {
|
||||
console.log(`[graph-load] ${phase}`, {
|
||||
elapsed_ms: Math.round(performance.now() - loadStartedAt),
|
||||
graph_query_id: graphQueryId,
|
||||
...(extra || {}),
|
||||
});
|
||||
};
|
||||
|
||||
const updateStatus = async (nextStatus: string, extra?: Record<string, unknown>): Promise<void> => {
|
||||
setStatus(nextStatus);
|
||||
logPhase(nextStatus, extra);
|
||||
await sleep(0);
|
||||
};
|
||||
|
||||
await updateStatus("Fetching graph…");
|
||||
const graphRes = await fetch(`/api/graph?graph_query_id=${encodeURIComponent(graphQueryId)}`, { signal });
|
||||
logPhase("graph response headers received", {
|
||||
status: graphRes.status,
|
||||
content_type: graphRes.headers.get("content-type"),
|
||||
content_length: graphRes.headers.get("content-length"),
|
||||
});
|
||||
if (!graphRes.ok) {
|
||||
let detail = "";
|
||||
try {
|
||||
@@ -141,44 +136,57 @@ export default function App() {
|
||||
}
|
||||
throw new Error(`Failed to fetch graph: ${graphRes.status}${detail ? ` (${detail})` : ""}`);
|
||||
}
|
||||
const graph = await graphRes.json();
|
||||
await updateStatus("Streaming Arrow graph…");
|
||||
await updateStatus("Decoding Arrow batches…");
|
||||
const decodeStartedAt = performance.now();
|
||||
let graph: Awaited<ReturnType<typeof readGraphArrow>>;
|
||||
try {
|
||||
graph = await readGraphArrow(graphRes, logPhase);
|
||||
} catch (e) {
|
||||
logPhase("arrow graph decode failed", {
|
||||
error: e instanceof Error ? e.message : String(e),
|
||||
});
|
||||
throw e;
|
||||
}
|
||||
logPhase("arrow graph decoded", {
|
||||
decode_ms: Math.round(performance.now() - decodeStartedAt),
|
||||
});
|
||||
if (signal.aborted) return;
|
||||
|
||||
const nodes = Array.isArray(graph.nodes) ? graph.nodes : [];
|
||||
const edges = Array.isArray(graph.edges) ? graph.edges : [];
|
||||
const routeSegments = graphRouteSegmentArray(graph.route_segments);
|
||||
const meta = graph.meta || null;
|
||||
const count = nodes.length;
|
||||
const vertexIds = graph.vertexIds;
|
||||
const xs = graph.xs;
|
||||
const ys = graph.ys;
|
||||
const edgeData = graph.edgeData;
|
||||
const routeLineVertices = graph.routeLineVertices;
|
||||
const labels = graph.labels;
|
||||
const iris = graph.iris;
|
||||
const count = vertexIds.length;
|
||||
const edgeCount = edgeData.length / 2;
|
||||
logPhase("graph payload ready", {
|
||||
nodes: count,
|
||||
edges: edgeCount,
|
||||
route_line_segments: routeLineVertices.length / 4,
|
||||
backend_nodes: meta && typeof meta.nodes === "number" ? meta.nodes : undefined,
|
||||
backend_edges: meta && typeof meta.edges === "number" ? meta.edges : undefined,
|
||||
});
|
||||
|
||||
nodesRef.current = nodes;
|
||||
nodeLookupRef.current = { vertexIds, labels, iris };
|
||||
graphMetaRef.current = meta && typeof meta === "object" ? (meta as GraphMeta) : null;
|
||||
setTripleResult(idleTripleResult(activeSelectionQueryId));
|
||||
|
||||
// Build positions from backend-provided node coordinates.
|
||||
setStatus("Preparing buffers…");
|
||||
const xs = new Float32Array(count);
|
||||
const ys = new Float32Array(count);
|
||||
for (let i = 0; i < count; i++) {
|
||||
const nx = nodes[i]?.x;
|
||||
const ny = nodes[i]?.y;
|
||||
xs[i] = typeof nx === "number" ? nx : 0;
|
||||
ys[i] = typeof ny === "number" ? ny : 0;
|
||||
}
|
||||
const vertexIds = new Uint32Array(count);
|
||||
for (let i = 0; i < count; i++) {
|
||||
const id = nodes[i]?.id;
|
||||
vertexIds[i] = typeof id === "number" ? id >>> 0 : i;
|
||||
}
|
||||
|
||||
// Build edges as vertex-id pairs.
|
||||
const edgeData = new Uint32Array(edges.length * 2);
|
||||
for (let i = 0; i < edges.length; i++) {
|
||||
const s = edges[i]?.source;
|
||||
const t = edges[i]?.target;
|
||||
edgeData[i * 2] = typeof s === "number" ? s >>> 0 : 0;
|
||||
edgeData[i * 2 + 1] = typeof t === "number" ? t >>> 0 : 0;
|
||||
}
|
||||
const routeLineVertices = buildRouteLineVertices(routeSegments);
|
||||
await updateStatus("Preparing buffers…", {
|
||||
nodes: count,
|
||||
edges: edgeCount,
|
||||
});
|
||||
const bufferPrepStartedAt = performance.now();
|
||||
const typedArrayBytes = estimateFrontendTypedArrayBytes(count, edgeCount, routeLineVertices.length);
|
||||
logPhase("buffer prep done", {
|
||||
buffer_prep_ms: Math.round(performance.now() - bufferPrepStartedAt),
|
||||
app_typed_arrays: formatBytes(typedArrayBytes.app),
|
||||
renderer_typed_arrays_estimate: formatBytes(typedArrayBytes.renderer),
|
||||
total_typed_arrays_estimate: formatBytes(typedArrayBytes.total),
|
||||
});
|
||||
|
||||
// Use /api/graph meta; don't do a second expensive backend call.
|
||||
if (meta && typeof meta.nodes === "number" && typeof meta.edges === "number") {
|
||||
@@ -188,33 +196,48 @@ export default function App() {
|
||||
backend: typeof meta.backend === "string" ? meta.backend : undefined,
|
||||
});
|
||||
} else {
|
||||
setBackendStats({ nodes: nodes.length, edges: edges.length });
|
||||
setBackendStats({ nodes: count, edges: edgeCount });
|
||||
}
|
||||
|
||||
setStatus("Building spatial index…");
|
||||
await new Promise((r) => setTimeout(r, 0));
|
||||
await updateStatus("Building spatial index…", {
|
||||
nodes: count,
|
||||
edges: edgeCount,
|
||||
});
|
||||
if (signal.aborted) return;
|
||||
|
||||
const buildMs = renderer.init(
|
||||
xs,
|
||||
ys,
|
||||
vertexIds,
|
||||
edgeData,
|
||||
routeLineVertices.length > 0 ? routeLineVertices : null
|
||||
);
|
||||
let buildMs: number;
|
||||
try {
|
||||
buildMs = renderer.init(
|
||||
xs,
|
||||
ys,
|
||||
vertexIds,
|
||||
edgeData,
|
||||
routeLineVertices.length > 0 ? routeLineVertices : null
|
||||
);
|
||||
} catch (e) {
|
||||
logPhase("renderer.init failed", {
|
||||
error: e instanceof Error ? e.message : String(e),
|
||||
});
|
||||
throw e;
|
||||
}
|
||||
setNodeCount(renderer.getNodeCount());
|
||||
setSelectedNodes(new Set());
|
||||
setStatus("");
|
||||
console.log(`Init complete: ${count.toLocaleString()} nodes, ${edges.length.toLocaleString()} edges in ${buildMs.toFixed(0)}ms`);
|
||||
logPhase("init complete", {
|
||||
renderer_init_ms: Math.round(buildMs),
|
||||
nodes: count,
|
||||
edges: edgeCount,
|
||||
});
|
||||
}
|
||||
|
||||
function getSelectedIds(renderer: Renderer, selected: Set<number>): number[] {
|
||||
const lookup = nodeLookupRef.current;
|
||||
if (!lookup) return [];
|
||||
const selectedIds: number[] = [];
|
||||
for (const sortedIdx of selected) {
|
||||
const origIdx = renderer.sortedIndexToOriginalIndex(sortedIdx);
|
||||
if (origIdx === null) continue;
|
||||
const node = nodesRef.current?.[origIdx];
|
||||
const nodeId = node?.id;
|
||||
const nodeId = lookup.vertexIds[origIdx];
|
||||
if (typeof nodeId !== "number") continue;
|
||||
selectedIds.push(nodeId);
|
||||
}
|
||||
@@ -370,14 +393,16 @@ export default function App() {
|
||||
const hit = renderer.findNodeIndexAt(mousePos.current.x, mousePos.current.y);
|
||||
if (hit) {
|
||||
const origIdx = renderer.sortedIndexToOriginalIndex(hit.index);
|
||||
const meta = origIdx === null ? null : nodesRef.current[origIdx];
|
||||
const lookup = nodeLookupRef.current;
|
||||
const label = origIdx === null || !lookup ? undefined : lookup.labels[origIdx];
|
||||
const iri = origIdx === null || !lookup ? undefined : lookup.iris[origIdx];
|
||||
setHoveredNode({
|
||||
x: hit.x,
|
||||
y: hit.y,
|
||||
screenX: mousePos.current.x,
|
||||
screenY: mousePos.current.y,
|
||||
label: meta && typeof meta.label === "string" ? meta.label : undefined,
|
||||
iri: meta && typeof meta.iri === "string" ? meta.iri : undefined,
|
||||
label: typeof label === "string" ? label : undefined,
|
||||
iri: typeof iri === "string" ? iri : undefined,
|
||||
});
|
||||
} else {
|
||||
setHoveredNode(null);
|
||||
|
||||
301
frontend/src/graph_arrow.ts
Normal file
301
frontend/src/graph_arrow.ts
Normal file
@@ -0,0 +1,301 @@
|
||||
import { RecordBatchReader } from "apache-arrow";
|
||||
import type { GraphMeta } from "./selection_queries";
|
||||
|
||||
export type ArrowGraphLoadResult = {
|
||||
meta: GraphMeta | null;
|
||||
vertexIds: Uint32Array;
|
||||
xs: Float32Array;
|
||||
ys: Float32Array;
|
||||
edgeData: Uint32Array;
|
||||
routeLineVertices: Float32Array;
|
||||
labels: (string | undefined)[];
|
||||
iris: string[];
|
||||
};
|
||||
|
||||
type ArrowBatchLog = (phase: string, extra?: Record<string, unknown>) => void;
|
||||
|
||||
type ArrowLikeVector = {
|
||||
data?: Array<{ valueOffsets?: ArrayLike<number | bigint> }>;
|
||||
getChildAt?: (index: number) => ArrowLikeVector | null;
|
||||
get?: (index: number) => unknown;
|
||||
toArray?: () => ArrayLike<number>;
|
||||
};
|
||||
|
||||
const graphTransportVersion = "1";
|
||||
|
||||
export async function readGraphArrow(response: Response, logPhase?: ArrowBatchLog): Promise<ArrowGraphLoadResult> {
|
||||
validateArrowResponse(response);
|
||||
|
||||
const reader = await openArrowReader(response);
|
||||
|
||||
let meta: GraphMeta | null = null;
|
||||
let vertexIds: Uint32Array | null = null;
|
||||
let xs: Float32Array | null = null;
|
||||
let ys: Float32Array | null = null;
|
||||
let edgeData: Uint32Array | null = null;
|
||||
let routeLineVertices: Float32Array | null = null;
|
||||
let labels: (string | undefined)[] | null = null;
|
||||
let iris: string[] | null = null;
|
||||
|
||||
let batchIndex = 0;
|
||||
for await (const batch of reader) {
|
||||
const batchStartedAt = performance.now();
|
||||
|
||||
switch (batchIndex) {
|
||||
case 0: {
|
||||
meta = decodeMetaBatch(batch);
|
||||
const nodeCount = typeof meta.nodes === "number" ? meta.nodes : 0;
|
||||
const edgeCount = typeof meta.edges === "number" ? meta.edges : 0;
|
||||
const routeLineSegments = readScalarNumber(batch, "meta_route_line_segments") ?? 0;
|
||||
|
||||
vertexIds = new Uint32Array(nodeCount);
|
||||
xs = new Float32Array(nodeCount);
|
||||
ys = new Float32Array(nodeCount);
|
||||
edgeData = new Uint32Array(edgeCount * 2);
|
||||
routeLineVertices = new Float32Array(routeLineSegments * 4);
|
||||
labels = new Array<string | undefined>(nodeCount);
|
||||
iris = new Array<string>(nodeCount);
|
||||
|
||||
logPhase?.("arrow_meta_batch_decoded", {
|
||||
batch_index: batchIndex,
|
||||
rows: batch.numRows,
|
||||
node_count: nodeCount,
|
||||
edge_count: edgeCount,
|
||||
route_line_segments: routeLineSegments,
|
||||
decode_ms: Math.round(performance.now() - batchStartedAt),
|
||||
});
|
||||
break;
|
||||
}
|
||||
case 1: {
|
||||
ensureAllocated(meta, vertexIds, xs, ys, edgeData, routeLineVertices, labels, iris);
|
||||
const nodeIDs = readUint32List(batch, "node_id");
|
||||
const nodeXs = readFloat32List(batch, "node_x");
|
||||
const nodeYs = readFloat32List(batch, "node_y");
|
||||
const nodeIRIs = readStringList(batch, "node_iri");
|
||||
const nodeLabels = readNullableStringList(batch, "node_label");
|
||||
|
||||
if (nodeIDs.length !== vertexIds.length || nodeXs.length !== xs.length || nodeYs.length !== ys.length) {
|
||||
throw new Error("Arrow node batch length mismatch");
|
||||
}
|
||||
if (nodeIRIs.length !== iris.length || nodeLabels.length !== labels.length) {
|
||||
throw new Error("Arrow node metadata batch length mismatch");
|
||||
}
|
||||
|
||||
vertexIds.set(nodeIDs);
|
||||
xs.set(nodeXs);
|
||||
ys.set(nodeYs);
|
||||
for (let i = 0; i < iris.length; i++) {
|
||||
iris[i] = nodeIRIs[i] ?? "";
|
||||
labels[i] = nodeLabels[i];
|
||||
}
|
||||
|
||||
logPhase?.("arrow_node_batch_decoded", {
|
||||
batch_index: batchIndex,
|
||||
rows: batch.numRows,
|
||||
nodes: vertexIds.length,
|
||||
decode_ms: Math.round(performance.now() - batchStartedAt),
|
||||
});
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
ensureAllocated(meta, vertexIds, xs, ys, edgeData, routeLineVertices, labels, iris);
|
||||
const edgeSources = readUint32List(batch, "edge_source");
|
||||
const edgeTargets = readUint32List(batch, "edge_target");
|
||||
if (edgeSources.length !== edgeTargets.length) {
|
||||
throw new Error("Arrow edge batch source/target length mismatch");
|
||||
}
|
||||
if (edgeData.length !== edgeSources.length * 2) {
|
||||
throw new Error("Arrow edge batch size mismatch");
|
||||
}
|
||||
for (let i = 0; i < edgeSources.length; i++) {
|
||||
edgeData[i * 2] = edgeSources[i];
|
||||
edgeData[i * 2 + 1] = edgeTargets[i];
|
||||
}
|
||||
logPhase?.("arrow_edge_batch_decoded", {
|
||||
batch_index: batchIndex,
|
||||
rows: batch.numRows,
|
||||
edges: edgeSources.length,
|
||||
decode_ms: Math.round(performance.now() - batchStartedAt),
|
||||
});
|
||||
break;
|
||||
}
|
||||
case 3: {
|
||||
ensureAllocated(meta, vertexIds, xs, ys, edgeData, routeLineVertices, labels, iris);
|
||||
const routeX1 = readFloat32List(batch, "route_x1");
|
||||
const routeY1 = readFloat32List(batch, "route_y1");
|
||||
const routeX2 = readFloat32List(batch, "route_x2");
|
||||
const routeY2 = readFloat32List(batch, "route_y2");
|
||||
if (
|
||||
routeX1.length !== routeY1.length ||
|
||||
routeX1.length !== routeX2.length ||
|
||||
routeX1.length !== routeY2.length
|
||||
) {
|
||||
throw new Error("Arrow route batch axis length mismatch");
|
||||
}
|
||||
if (routeLineVertices.length !== routeX1.length * 4) {
|
||||
throw new Error("Arrow route batch size mismatch");
|
||||
}
|
||||
for (let i = 0; i < routeX1.length; i++) {
|
||||
routeLineVertices[i * 4] = routeX1[i];
|
||||
routeLineVertices[i * 4 + 1] = routeY1[i];
|
||||
routeLineVertices[i * 4 + 2] = routeX2[i];
|
||||
routeLineVertices[i * 4 + 3] = routeY2[i];
|
||||
}
|
||||
logPhase?.("arrow_route_batch_decoded", {
|
||||
batch_index: batchIndex,
|
||||
rows: batch.numRows,
|
||||
route_line_segments: routeX1.length,
|
||||
decode_ms: Math.round(performance.now() - batchStartedAt),
|
||||
});
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new Error(`Unexpected Arrow batch index ${batchIndex}`);
|
||||
}
|
||||
|
||||
batchIndex++;
|
||||
}
|
||||
|
||||
if (batchIndex !== 4) {
|
||||
throw new Error(`Expected 4 Arrow batches, received ${batchIndex}`);
|
||||
}
|
||||
ensureAllocated(meta, vertexIds, xs, ys, edgeData, routeLineVertices, labels, iris);
|
||||
|
||||
return {
|
||||
meta,
|
||||
vertexIds,
|
||||
xs,
|
||||
ys,
|
||||
edgeData,
|
||||
routeLineVertices,
|
||||
labels,
|
||||
iris,
|
||||
};
|
||||
}
|
||||
|
||||
async function openArrowReader(response: Response) {
|
||||
if (response.body) {
|
||||
return RecordBatchReader.from(response.body);
|
||||
}
|
||||
return RecordBatchReader.from(await response.arrayBuffer());
|
||||
}
|
||||
|
||||
function validateArrowResponse(response: Response): void {
|
||||
const contentType = response.headers.get("content-type") || "";
|
||||
if (!contentType.includes("application/vnd.apache.arrow.stream")) {
|
||||
throw new Error(`Unexpected graph content type: ${contentType || "(missing)"}`);
|
||||
}
|
||||
const version = response.headers.get("X-Graph-Transport-Version");
|
||||
if (version !== graphTransportVersion) {
|
||||
throw new Error(`Unexpected graph transport version: ${version || "(missing)"}`);
|
||||
}
|
||||
}
|
||||
|
||||
function decodeMetaBatch(batch: any): GraphMeta {
|
||||
return {
|
||||
backend: readScalarString(batch, "meta_backend"),
|
||||
graph_query_id: readScalarString(batch, "meta_graph_query_id"),
|
||||
node_limit: readScalarNumber(batch, "meta_node_limit"),
|
||||
edge_limit: readScalarNumber(batch, "meta_edge_limit"),
|
||||
nodes: readScalarNumber(batch, "meta_nodes"),
|
||||
edges: readScalarNumber(batch, "meta_edges"),
|
||||
};
|
||||
}
|
||||
|
||||
function ensureAllocated(
|
||||
meta: GraphMeta | null,
|
||||
vertexIds: Uint32Array | null,
|
||||
xs: Float32Array | null,
|
||||
ys: Float32Array | null,
|
||||
edgeData: Uint32Array | null,
|
||||
routeLineVertices: Float32Array | null,
|
||||
labels: (string | undefined)[] | null,
|
||||
iris: string[] | null
|
||||
): asserts meta is GraphMeta & {} & {
|
||||
nodes?: number;
|
||||
edges?: number;
|
||||
} {
|
||||
if (!meta || !vertexIds || !xs || !ys || !edgeData || !routeLineVertices || !labels || !iris) {
|
||||
throw new Error("Arrow graph stream is missing the meta batch");
|
||||
}
|
||||
}
|
||||
|
||||
function readScalarString(batch: any, name: string): string | undefined {
|
||||
const vector = batch.getChild(name);
|
||||
if (!vector || batch.numRows < 1) return undefined;
|
||||
const value = vector.get(0);
|
||||
return typeof value === "string" ? value : undefined;
|
||||
}
|
||||
|
||||
function readScalarNumber(batch: any, name: string): number | undefined {
|
||||
const vector = batch.getChild(name);
|
||||
if (!vector || batch.numRows < 1) return undefined;
|
||||
const value = vector.get(0);
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : undefined;
|
||||
}
|
||||
|
||||
function readUint32List(batch: any, name: string): Uint32Array {
|
||||
return readPrimitiveList(batch, name, Uint32Array);
|
||||
}
|
||||
|
||||
function readFloat32List(batch: any, name: string): Float32Array {
|
||||
return readPrimitiveList(batch, name, Float32Array);
|
||||
}
|
||||
|
||||
function readPrimitiveList<T extends Uint32Array | Float32Array>(
|
||||
batch: any,
|
||||
name: string,
|
||||
ctor: { new(length: number): T }
|
||||
): T {
|
||||
const vector = batch.getChild(name) as ArrowLikeVector | null;
|
||||
if (!vector) return new ctor(0);
|
||||
|
||||
const [begin, end] = getListBounds(vector);
|
||||
const child = vector.getChildAt?.(0);
|
||||
if (!child || typeof child.toArray !== "function") {
|
||||
return new ctor(0);
|
||||
}
|
||||
const values = child.toArray();
|
||||
if (typeof (values as any).subarray === "function") {
|
||||
return (values as T).subarray(begin, end) as T;
|
||||
}
|
||||
|
||||
const out = new ctor(end - begin);
|
||||
for (let i = begin; i < end; i++) {
|
||||
out[i - begin] = Number((values as ArrayLike<number>)[i]) as T[number];
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function readStringList(batch: any, name: string): string[] {
|
||||
return readNullableStringList(batch, name).map((value) => value ?? "");
|
||||
}
|
||||
|
||||
function readNullableStringList(batch: any, name: string): (string | undefined)[] {
|
||||
const vector = batch.getChild(name) as ArrowLikeVector | null;
|
||||
if (!vector) return [];
|
||||
|
||||
const [begin, end] = getListBounds(vector);
|
||||
const child = vector.getChildAt?.(0);
|
||||
if (!child || typeof child.get !== "function") {
|
||||
return [];
|
||||
}
|
||||
|
||||
const out = new Array<string | undefined>(Math.max(0, end - begin));
|
||||
for (let i = begin; i < end; i++) {
|
||||
const value = child.get(i);
|
||||
out[i - begin] = typeof value === "string" ? value : undefined;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function getListBounds(vector: ArrowLikeVector): [number, number] {
|
||||
const offsets = vector.data?.[0]?.valueOffsets;
|
||||
if (!offsets || offsets.length < 2) return [0, 0];
|
||||
return [offsetToNumber(offsets[0]), offsetToNumber(offsets[1])];
|
||||
}
|
||||
|
||||
function offsetToNumber(value: number | bigint | undefined): number {
|
||||
if (typeof value === "bigint") return Number(value);
|
||||
return typeof value === "number" ? value : 0;
|
||||
}
|
||||
@@ -1,8 +1,6 @@
|
||||
export { fetchSelectionQueries, runSelectionQuery, runSelectionTripleQuery } from "./api";
|
||||
export type {
|
||||
GraphMeta,
|
||||
GraphRoutePoint,
|
||||
GraphRouteSegment,
|
||||
SelectionQueryMeta,
|
||||
SelectionTriple,
|
||||
SelectionTripleResult,
|
||||
|
||||
@@ -1,26 +1,10 @@
|
||||
export type GraphMeta = {
|
||||
backend?: string;
|
||||
ttl_path?: string | null;
|
||||
sparql_endpoint?: string | null;
|
||||
include_bnodes?: boolean;
|
||||
graph_query_id?: string;
|
||||
node_limit?: number;
|
||||
edge_limit?: number;
|
||||
nodes?: number;
|
||||
edges?: number;
|
||||
layout_engine?: string;
|
||||
layout_root_iri?: string | null;
|
||||
};
|
||||
|
||||
export type GraphRoutePoint = {
|
||||
x: number;
|
||||
y: number;
|
||||
};
|
||||
|
||||
export type GraphRouteSegment = {
|
||||
edge_index: number;
|
||||
kind: string;
|
||||
points: GraphRoutePoint[];
|
||||
};
|
||||
|
||||
export type SelectionQueryMeta = {
|
||||
|
||||
Reference in New Issue
Block a user