diff --git a/backend_go/Dockerfile b/backend_go/Dockerfile index 53ce558..158ffa8 100644 --- a/backend_go/Dockerfile +++ b/backend_go/Dockerfile @@ -11,12 +11,10 @@ FROM golang:${GO_VERSION}-alpine AS go-builder WORKDIR /src/backend_go -COPY backend_go/go.mod /src/backend_go/go.mod - -RUN go mod download - COPY backend_go /src/backend_go +RUN go mod tidy + RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/backend ./ FROM debian:bookworm-slim diff --git a/backend_go/go.mod b/backend_go/go.mod index 4e9d16f..7095521 100644 --- a/backend_go/go.mod +++ b/backend_go/go.mod @@ -1,3 +1,5 @@ module visualizador_instanciados/backend_go go 1.22 + +require github.com/apache/arrow/go/v17 v17.0.0 diff --git a/backend_go/graph_arrow.go b/backend_go/graph_arrow.go new file mode 100644 index 0000000..eec11d4 --- /dev/null +++ b/backend_go/graph_arrow.go @@ -0,0 +1,359 @@ +package main + +import ( + "fmt" + "log" + "net/http" + "time" + + "github.com/apache/arrow/go/v17/arrow" + "github.com/apache/arrow/go/v17/arrow/array" + "github.com/apache/arrow/go/v17/arrow/ipc" + "github.com/apache/arrow/go/v17/arrow/memory" +) + +const graphTransportVersion = "1" + +const ( + graphArrowMetaBackendField = iota + graphArrowMetaGraphQueryIDField + graphArrowMetaNodeLimitField + graphArrowMetaEdgeLimitField + graphArrowMetaNodesField + graphArrowMetaEdgesField + graphArrowMetaRouteLineSegmentsField + graphArrowNodeIDField + graphArrowNodeXField + graphArrowNodeYField + graphArrowNodeIRIField + graphArrowNodeLabelField + graphArrowEdgeSourceField + graphArrowEdgeTargetField + graphArrowRouteX1Field + graphArrowRouteY1Field + graphArrowRouteX2Field + graphArrowRouteY2Field +) + +var graphArrowSchema = arrow.NewSchema([]arrow.Field{ + {Name: "meta_backend", Type: arrow.BinaryTypes.String, Nullable: true}, + {Name: "meta_graph_query_id", Type: arrow.BinaryTypes.String, Nullable: true}, + {Name: "meta_node_limit", Type: arrow.PrimitiveTypes.Int32, Nullable: true}, + {Name: "meta_edge_limit", Type: arrow.PrimitiveTypes.Int32, Nullable: true}, + {Name: "meta_nodes", Type: arrow.PrimitiveTypes.Int32, Nullable: true}, + {Name: "meta_edges", Type: arrow.PrimitiveTypes.Int32, Nullable: true}, + {Name: "meta_route_line_segments", Type: arrow.PrimitiveTypes.Int32, Nullable: true}, + {Name: "node_id", Type: arrow.ListOf(arrow.PrimitiveTypes.Uint32), Nullable: true}, + {Name: "node_x", Type: arrow.ListOf(arrow.PrimitiveTypes.Float32), Nullable: true}, + {Name: "node_y", Type: arrow.ListOf(arrow.PrimitiveTypes.Float32), Nullable: true}, + {Name: "node_iri", Type: arrow.ListOf(arrow.BinaryTypes.String), Nullable: true}, + {Name: "node_label", Type: arrow.ListOf(arrow.BinaryTypes.String), Nullable: true}, + {Name: "edge_source", Type: arrow.ListOf(arrow.PrimitiveTypes.Uint32), Nullable: true}, + {Name: "edge_target", Type: arrow.ListOf(arrow.PrimitiveTypes.Uint32), Nullable: true}, + {Name: "route_x1", Type: arrow.ListOf(arrow.PrimitiveTypes.Float32), Nullable: true}, + {Name: "route_y1", Type: arrow.ListOf(arrow.PrimitiveTypes.Float32), Nullable: true}, + {Name: "route_x2", Type: arrow.ListOf(arrow.PrimitiveTypes.Float32), Nullable: true}, + {Name: "route_y2", Type: arrow.ListOf(arrow.PrimitiveTypes.Float32), Nullable: true}, +}, nil) + +type graphArrowRouteLines struct { + X1 []float32 + Y1 []float32 + X2 []float32 + Y2 []float32 +} + +func writeGraphArrow(w http.ResponseWriter, snap GraphResponse) error { + start := time.Now() + routes := flattenGraphRouteLines(snap.RouteSegments) + meta := snap.Meta + if meta == nil { + meta = &GraphMeta{ + GraphQueryID: "default", + NodeLimit: len(snap.Nodes), + EdgeLimit: len(snap.Edges), + Nodes: len(snap.Nodes), + Edges: len(snap.Edges), + } + } + + alloc := memory.NewGoAllocator() + w.Header().Set("Content-Type", "application/vnd.apache.arrow.stream") + w.Header().Set("X-Graph-Transport-Version", graphTransportVersion) + w.WriteHeader(http.StatusOK) + + writer := ipc.NewWriter(w, ipc.WithSchema(graphArrowSchema)) + closed := false + defer func() { + if !closed { + _ = writer.Close() + } + }() + + if err := writeGraphArrowMetaBatch(writer, alloc, meta, len(routes.X1)); err != nil { + return fmt.Errorf("write meta batch failed: %w", err) + } + if err := writeGraphArrowNodeBatch(writer, alloc, snap.Nodes); err != nil { + return fmt.Errorf("write node batch failed: %w", err) + } + if err := writeGraphArrowEdgeBatch(writer, alloc, snap.Edges); err != nil { + return fmt.Errorf("write edge batch failed: %w", err) + } + if err := writeGraphArrowRouteBatch(writer, alloc, routes); err != nil { + return fmt.Errorf("write route batch failed: %w", err) + } + if err := writer.Close(); err != nil { + return fmt.Errorf("close arrow writer failed: %w", err) + } + closed = true + + log.Printf( + "[graph-arrow] encode_done graph_query_id=%s nodes=%d edges=%d route_line_segments=%d encode_time=%s", + meta.GraphQueryID, + len(snap.Nodes), + len(snap.Edges), + len(routes.X1), + time.Since(start).Truncate(time.Millisecond), + ) + return nil +} + +func writeGraphArrowMetaBatch(writer *ipc.Writer, alloc memory.Allocator, meta *GraphMeta, routeLineSegments int) error { + builder := array.NewRecordBuilder(alloc, graphArrowSchema) + defer builder.Release() + + appendStringBuilderValue(builder.Field(graphArrowMetaBackendField), meta.Backend) + appendStringBuilderValue(builder.Field(graphArrowMetaGraphQueryIDField), meta.GraphQueryID) + appendInt32BuilderValue(builder.Field(graphArrowMetaNodeLimitField), int32(meta.NodeLimit)) + appendInt32BuilderValue(builder.Field(graphArrowMetaEdgeLimitField), int32(meta.EdgeLimit)) + appendInt32BuilderValue(builder.Field(graphArrowMetaNodesField), int32(meta.Nodes)) + appendInt32BuilderValue(builder.Field(graphArrowMetaEdgesField), int32(meta.Edges)) + appendInt32BuilderValue(builder.Field(graphArrowMetaRouteLineSegmentsField), int32(routeLineSegments)) + + appendNullTopLevel(builder.Field(graphArrowNodeIDField)) + appendNullTopLevel(builder.Field(graphArrowNodeXField)) + appendNullTopLevel(builder.Field(graphArrowNodeYField)) + appendNullTopLevel(builder.Field(graphArrowNodeIRIField)) + appendNullTopLevel(builder.Field(graphArrowNodeLabelField)) + appendNullTopLevel(builder.Field(graphArrowEdgeSourceField)) + appendNullTopLevel(builder.Field(graphArrowEdgeTargetField)) + appendNullTopLevel(builder.Field(graphArrowRouteX1Field)) + appendNullTopLevel(builder.Field(graphArrowRouteY1Field)) + appendNullTopLevel(builder.Field(graphArrowRouteX2Field)) + appendNullTopLevel(builder.Field(graphArrowRouteY2Field)) + + return writeGraphArrowRecord(writer, builder) +} + +func writeGraphArrowNodeBatch(writer *ipc.Writer, alloc memory.Allocator, nodes []Node) error { + builder := array.NewRecordBuilder(alloc, graphArrowSchema) + defer builder.Release() + + appendNullTopLevel(builder.Field(graphArrowMetaBackendField)) + appendNullTopLevel(builder.Field(graphArrowMetaGraphQueryIDField)) + appendNullTopLevel(builder.Field(graphArrowMetaNodeLimitField)) + appendNullTopLevel(builder.Field(graphArrowMetaEdgeLimitField)) + appendNullTopLevel(builder.Field(graphArrowMetaNodesField)) + appendNullTopLevel(builder.Field(graphArrowMetaEdgesField)) + appendNullTopLevel(builder.Field(graphArrowMetaRouteLineSegmentsField)) + + appendUint32List(builder.Field(graphArrowNodeIDField), func(valueBuilder *array.Uint32Builder) { + for _, node := range nodes { + valueBuilder.Append(node.ID) + } + }) + appendFloat32List(builder.Field(graphArrowNodeXField), func(valueBuilder *array.Float32Builder) { + for _, node := range nodes { + valueBuilder.Append(float32(node.X)) + } + }) + appendFloat32List(builder.Field(graphArrowNodeYField), func(valueBuilder *array.Float32Builder) { + for _, node := range nodes { + valueBuilder.Append(float32(node.Y)) + } + }) + appendStringList(builder.Field(graphArrowNodeIRIField), func(valueBuilder *array.StringBuilder) { + for _, node := range nodes { + valueBuilder.Append(node.IRI) + } + }) + appendNullableStringList(builder.Field(graphArrowNodeLabelField), func(valueBuilder *array.StringBuilder) { + for _, node := range nodes { + if node.Label == nil { + valueBuilder.AppendNull() + continue + } + valueBuilder.Append(*node.Label) + } + }) + + appendNullTopLevel(builder.Field(graphArrowEdgeSourceField)) + appendNullTopLevel(builder.Field(graphArrowEdgeTargetField)) + appendNullTopLevel(builder.Field(graphArrowRouteX1Field)) + appendNullTopLevel(builder.Field(graphArrowRouteY1Field)) + appendNullTopLevel(builder.Field(graphArrowRouteX2Field)) + appendNullTopLevel(builder.Field(graphArrowRouteY2Field)) + + return writeGraphArrowRecord(writer, builder) +} + +func writeGraphArrowEdgeBatch(writer *ipc.Writer, alloc memory.Allocator, edges []Edge) error { + builder := array.NewRecordBuilder(alloc, graphArrowSchema) + defer builder.Release() + + appendNullTopLevel(builder.Field(graphArrowMetaBackendField)) + appendNullTopLevel(builder.Field(graphArrowMetaGraphQueryIDField)) + appendNullTopLevel(builder.Field(graphArrowMetaNodeLimitField)) + appendNullTopLevel(builder.Field(graphArrowMetaEdgeLimitField)) + appendNullTopLevel(builder.Field(graphArrowMetaNodesField)) + appendNullTopLevel(builder.Field(graphArrowMetaEdgesField)) + appendNullTopLevel(builder.Field(graphArrowMetaRouteLineSegmentsField)) + appendNullTopLevel(builder.Field(graphArrowNodeIDField)) + appendNullTopLevel(builder.Field(graphArrowNodeXField)) + appendNullTopLevel(builder.Field(graphArrowNodeYField)) + appendNullTopLevel(builder.Field(graphArrowNodeIRIField)) + appendNullTopLevel(builder.Field(graphArrowNodeLabelField)) + + appendUint32List(builder.Field(graphArrowEdgeSourceField), func(valueBuilder *array.Uint32Builder) { + for _, edge := range edges { + valueBuilder.Append(edge.Source) + } + }) + appendUint32List(builder.Field(graphArrowEdgeTargetField), func(valueBuilder *array.Uint32Builder) { + for _, edge := range edges { + valueBuilder.Append(edge.Target) + } + }) + + appendNullTopLevel(builder.Field(graphArrowRouteX1Field)) + appendNullTopLevel(builder.Field(graphArrowRouteY1Field)) + appendNullTopLevel(builder.Field(graphArrowRouteX2Field)) + appendNullTopLevel(builder.Field(graphArrowRouteY2Field)) + + return writeGraphArrowRecord(writer, builder) +} + +func writeGraphArrowRouteBatch(writer *ipc.Writer, alloc memory.Allocator, routes graphArrowRouteLines) error { + builder := array.NewRecordBuilder(alloc, graphArrowSchema) + defer builder.Release() + + appendNullTopLevel(builder.Field(graphArrowMetaBackendField)) + appendNullTopLevel(builder.Field(graphArrowMetaGraphQueryIDField)) + appendNullTopLevel(builder.Field(graphArrowMetaNodeLimitField)) + appendNullTopLevel(builder.Field(graphArrowMetaEdgeLimitField)) + appendNullTopLevel(builder.Field(graphArrowMetaNodesField)) + appendNullTopLevel(builder.Field(graphArrowMetaEdgesField)) + appendNullTopLevel(builder.Field(graphArrowMetaRouteLineSegmentsField)) + appendNullTopLevel(builder.Field(graphArrowNodeIDField)) + appendNullTopLevel(builder.Field(graphArrowNodeXField)) + appendNullTopLevel(builder.Field(graphArrowNodeYField)) + appendNullTopLevel(builder.Field(graphArrowNodeIRIField)) + appendNullTopLevel(builder.Field(graphArrowNodeLabelField)) + appendNullTopLevel(builder.Field(graphArrowEdgeSourceField)) + appendNullTopLevel(builder.Field(graphArrowEdgeTargetField)) + + appendFloat32List(builder.Field(graphArrowRouteX1Field), func(valueBuilder *array.Float32Builder) { + for _, value := range routes.X1 { + valueBuilder.Append(value) + } + }) + appendFloat32List(builder.Field(graphArrowRouteY1Field), func(valueBuilder *array.Float32Builder) { + for _, value := range routes.Y1 { + valueBuilder.Append(value) + } + }) + appendFloat32List(builder.Field(graphArrowRouteX2Field), func(valueBuilder *array.Float32Builder) { + for _, value := range routes.X2 { + valueBuilder.Append(value) + } + }) + appendFloat32List(builder.Field(graphArrowRouteY2Field), func(valueBuilder *array.Float32Builder) { + for _, value := range routes.Y2 { + valueBuilder.Append(value) + } + }) + + return writeGraphArrowRecord(writer, builder) +} + +func writeGraphArrowRecord(writer *ipc.Writer, builder *array.RecordBuilder) error { + record := builder.NewRecord() + defer record.Release() + return writer.Write(record) +} + +func appendStringBuilderValue(builder array.Builder, value string) { + builder.(*array.StringBuilder).Append(value) +} + +func appendInt32BuilderValue(builder array.Builder, value int32) { + builder.(*array.Int32Builder).Append(value) +} + +func appendUint32List(builder array.Builder, appendValues func(*array.Uint32Builder)) { + listBuilder := builder.(*array.ListBuilder) + listBuilder.Append(true) + valueBuilder := listBuilder.ValueBuilder().(*array.Uint32Builder) + appendValues(valueBuilder) +} + +func appendFloat32List(builder array.Builder, appendValues func(*array.Float32Builder)) { + listBuilder := builder.(*array.ListBuilder) + listBuilder.Append(true) + valueBuilder := listBuilder.ValueBuilder().(*array.Float32Builder) + appendValues(valueBuilder) +} + +func appendStringList(builder array.Builder, appendValues func(*array.StringBuilder)) { + listBuilder := builder.(*array.ListBuilder) + listBuilder.Append(true) + valueBuilder := listBuilder.ValueBuilder().(*array.StringBuilder) + appendValues(valueBuilder) +} + +func appendNullableStringList(builder array.Builder, appendValues func(*array.StringBuilder)) { + listBuilder := builder.(*array.ListBuilder) + listBuilder.Append(true) + valueBuilder := listBuilder.ValueBuilder().(*array.StringBuilder) + appendValues(valueBuilder) +} + +func appendNullTopLevel(builder array.Builder) { + switch typed := builder.(type) { + case *array.StringBuilder: + typed.AppendNull() + case *array.Int32Builder: + typed.AppendNull() + case *array.ListBuilder: + typed.AppendNull() + default: + panic(fmt.Sprintf("unsupported top-level builder type %T", builder)) + } +} + +func flattenGraphRouteLines(routeSegments []RouteSegment) graphArrowRouteLines { + lineCount := 0 + for _, route := range routeSegments { + if len(route.Points) > 1 { + lineCount += len(route.Points) - 1 + } + } + + out := graphArrowRouteLines{ + X1: make([]float32, 0, lineCount), + Y1: make([]float32, 0, lineCount), + X2: make([]float32, 0, lineCount), + Y2: make([]float32, 0, lineCount), + } + for _, route := range routeSegments { + for i := 1; i < len(route.Points); i++ { + prev := route.Points[i-1] + curr := route.Points[i] + out.X1 = append(out.X1, float32(prev.X)) + out.Y1 = append(out.Y1, float32(prev.Y)) + out.X2 = append(out.X2, float32(curr.X)) + out.Y2 = append(out.Y2, float32(curr.Y)) + } + } + return out +} diff --git a/backend_go/server.go b/backend_go/server.go index 69e014e..60f3538 100644 --- a/backend_go/server.go +++ b/backend_go/server.go @@ -155,7 +155,9 @@ func (s *APIServer) handleGraph(w http.ResponseWriter, r *http.Request) { return } - writeJSON(w, http.StatusOK, snap) + if err := writeGraphArrow(w, snap); err != nil { + log.Printf("handleGraph: arrow encode error graph_query_id=%s node_limit=%d edge_limit=%d err=%v", graphQueryID, nodeLimit, edgeLimit, err) + } } func (s *APIServer) handleGraphQueries(w http.ResponseWriter, r *http.Request) { diff --git a/frontend/package-lock.json b/frontend/package-lock.json index af5f4d3..b9a52f7 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -10,6 +10,7 @@ "dependencies": { "@cosmos.gl/graph": "^2.6.4", "@webgpu/types": "^0.1.69", + "apache-arrow": "^21.1.0", "clsx": "2.1.1", "react": "19.2.3", "react-dom": "19.2.3", @@ -1184,6 +1185,15 @@ "win32" ] }, + "node_modules/@swc/helpers": { + "version": "0.5.20", + "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.20.tgz", + "integrity": "sha512-2egEBHUMasdypIzrprsu8g+OEVd7Vp2MM3a2eVlM/cyFYto0nGz5BX5BTgh/ShZZI9ed+ozEq+Ngt+rgmUs8tw==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.8.0" + } + }, "node_modules/@tailwindcss/node": { "version": "4.1.17", "resolved": "https://registry.npmjs.org/@tailwindcss/node/-/node-4.1.17.tgz", @@ -1501,6 +1511,18 @@ "@babel/types": "^7.28.2" } }, + "node_modules/@types/command-line-args": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/command-line-args/-/command-line-args-5.2.3.tgz", + "integrity": "sha512-uv0aG6R0Y8WHZLTamZwtfsDLVRnOa+n+n5rEvFWL5Na5gZ8V2Teab/duDPFzIIIhs9qizDpcavCusCLJZu62Kw==", + "license": "MIT" + }, + "node_modules/@types/command-line-usage": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/@types/command-line-usage/-/command-line-usage-5.0.4.tgz", + "integrity": "sha512-BwR5KP3Es/CSht0xqBcUXS3qCAUVXwpRKsV2+arxeb65atasuXG9LykC9Ab10Cw3s2raH92ZqOeILaQbsB2ACg==", + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz", @@ -1572,6 +1594,65 @@ "integrity": "sha512-RPmm6kgRbI8e98zSD3RVACvnuktIja5+yLgDAkTmxLr90BEwdTXRQWNLF3ETTTyH/8mKhznZuN5AveXYFEsMGQ==", "license": "BSD-3-Clause" }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/apache-arrow": { + "version": "21.1.0", + "resolved": "https://registry.npmjs.org/apache-arrow/-/apache-arrow-21.1.0.tgz", + "integrity": "sha512-kQrYLxhC+NTVVZ4CCzGF6L/uPVOzJmD1T3XgbiUnP7oTeVFOFgEUu6IKNwCDkpFoBVqDKQivlX4RUFqqnWFlEA==", + "license": "Apache-2.0", + "dependencies": { + "@swc/helpers": "^0.5.11", + "@types/command-line-args": "^5.2.3", + "@types/command-line-usage": "^5.0.4", + "@types/node": "^24.0.3", + "command-line-args": "^6.0.1", + "command-line-usage": "^7.0.1", + "flatbuffers": "^25.1.24", + "json-bignum": "^0.0.3", + "tslib": "^2.6.2" + }, + "bin": { + "arrow2csv": "bin/arrow2csv.js" + } + }, + "node_modules/apache-arrow/node_modules/@types/node": { + "version": "24.12.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.0.tgz", + "integrity": "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/apache-arrow/node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "license": "MIT" + }, + "node_modules/array-back": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/array-back/-/array-back-6.2.3.tgz", + "integrity": "sha512-SGDvmg6QTYiTxCBkYVmThcoa67uLl35pyzRHdpCGBOcqFy6BtwnphoFPk7LhJshD+Yk1Kt35WGWeZPTgwR4Fhw==", + "license": "MIT", + "engines": { + "node": ">=12.17" + } + }, "node_modules/baseline-browser-mapping": { "version": "2.9.19", "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.19.tgz", @@ -1650,6 +1731,37 @@ ], "license": "CC-BY-4.0" }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chalk-template": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/chalk-template/-/chalk-template-0.4.0.tgz", + "integrity": "sha512-/ghrgmhfY8RaSdeo43hNXxpoHAtxdbskUHjPpfqUWGttFgycUhYPGx3YZBCnUCvOa7Doivn1IZec3DEGFoMgLg==", + "license": "MIT", + "dependencies": { + "chalk": "^4.1.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/chalk-template?sponsor=1" + } + }, "node_modules/clsx": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", @@ -1659,6 +1771,62 @@ "node": ">=6" } }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/command-line-args": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/command-line-args/-/command-line-args-6.0.2.tgz", + "integrity": "sha512-AIjYVxrV9X752LmPDLbVYv8aMCuHPSLZJXEo2qo/xJfv+NYhaZ4sMSF01rM+gHPaMgvPM0l5D/F+Qx+i2WfSmQ==", + "license": "MIT", + "dependencies": { + "array-back": "^6.2.3", + "find-replace": "^5.0.2", + "lodash.camelcase": "^4.3.0", + "typical": "^7.3.0" + }, + "engines": { + "node": ">=12.20" + }, + "peerDependencies": { + "@75lb/nature": "latest" + }, + "peerDependenciesMeta": { + "@75lb/nature": { + "optional": true + } + } + }, + "node_modules/command-line-usage": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/command-line-usage/-/command-line-usage-7.0.4.tgz", + "integrity": "sha512-85UdvzTNx/+s5CkSgBm/0hzP80RFHAa7PsfeADE5ezZF3uHz3/Tqj9gIKGT9PTtpycc3Ua64T0oVulGfKxzfqg==", + "license": "MIT", + "dependencies": { + "array-back": "^6.2.2", + "chalk-template": "^0.4.0", + "table-layout": "^4.1.1", + "typical": "^7.3.0" + }, + "engines": { + "node": ">=12.20.0" + } + }, "node_modules/convert-source-map": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", @@ -1980,6 +2148,29 @@ "node": ">=8" } }, + "node_modules/find-replace": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/find-replace/-/find-replace-5.0.2.tgz", + "integrity": "sha512-Y45BAiE3mz2QsrN2fb5QEtO4qb44NcS7en/0y9PEVsg351HsLeVclP8QPMH79Le9sH3rs5RSwJu99W0WPZO43Q==", + "license": "MIT", + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@75lb/nature": "latest" + }, + "peerDependenciesMeta": { + "@75lb/nature": { + "optional": true + } + } + }, + "node_modules/flatbuffers": { + "version": "25.9.23", + "resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-25.9.23.tgz", + "integrity": "sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==", + "license": "Apache-2.0" + }, "node_modules/fsevents": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", @@ -2037,6 +2228,15 @@ "dev": true, "license": "ISC" }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/internmap": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/internmap/-/internmap-2.0.3.tgz", @@ -2086,6 +2286,14 @@ "node": ">=6" } }, + "node_modules/json-bignum": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/json-bignum/-/json-bignum-0.0.3.tgz", + "integrity": "sha512-2WHyXj3OfHSgNyuzDbSxI1w2jgw5gkWSWhS7Qg4bWXx1nLk3jnbwfUeS0PSba3IzpTUWdHxBieELUzXRjQB2zg==", + "engines": { + "node": ">=0.8" + } + }, "node_modules/json5": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", @@ -2360,6 +2568,12 @@ "url": "https://opencollective.com/parcel" } }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, "node_modules/lru-cache": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", @@ -2625,6 +2839,31 @@ "node": ">=0.10.0" } }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/table-layout": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/table-layout/-/table-layout-4.1.1.tgz", + "integrity": "sha512-iK5/YhZxq5GO5z8wb0bY1317uDF3Zjpha0QFFLA8/trAoiLbQD0HUbMesEaxyzUgDxi2QlcbM8IvqOlEjgoXBA==", + "license": "MIT", + "dependencies": { + "array-back": "^6.2.2", + "wordwrapjs": "^5.1.0" + }, + "engines": { + "node": ">=12.17" + } + }, "node_modules/tailwind-merge": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/tailwind-merge/-/tailwind-merge-3.4.0.tgz", @@ -2686,6 +2925,12 @@ "node": ">=8.0" } }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, "node_modules/tsx": { "version": "4.21.0", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", @@ -3204,6 +3449,15 @@ "node": ">=14.17" } }, + "node_modules/typical": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/typical/-/typical-7.3.0.tgz", + "integrity": "sha512-ya4mg/30vm+DOWfBg4YK3j2WD6TWtRkCbasOJr40CseYENzCUby/7rIvXA99JGsQHeNxLbnXdyLLxKSv3tauFw==", + "license": "MIT", + "engines": { + "node": ">=12.17" + } + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", @@ -3334,6 +3588,15 @@ "vite": "^5.4.11 || ^6.0.0 || ^7.0.0" } }, + "node_modules/wordwrapjs": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/wordwrapjs/-/wordwrapjs-5.1.1.tgz", + "integrity": "sha512-0yweIbkINJodk27gX9LBGMzyQdBDan3s/dEAiwBOj+Mf0PPyWL6/rikalkv8EeD0E8jm4o5RXEOrFTP3NXbhJg==", + "license": "MIT", + "engines": { + "node": ">=12.17" + } + }, "node_modules/yallist": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index b681701..243ea10 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -12,6 +12,7 @@ "dependencies": { "@cosmos.gl/graph": "^2.6.4", "@webgpu/types": "^0.1.69", + "apache-arrow": "^21.1.0", "clsx": "2.1.1", "react": "19.2.3", "react-dom": "19.2.3", diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index e6021e7..15c5f1b 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -4,77 +4,21 @@ import { fetchGraphQueries } from "./graph_queries"; import type { GraphQueryMeta } from "./graph_queries"; import { fetchSelectionQueries, runSelectionQuery, runSelectionTripleQuery } from "./selection_queries"; import { cosmosRuntimeConfig } from "./cosmos_config"; -import type { GraphMeta, GraphRoutePoint, GraphRouteSegment, SelectionQueryMeta, SelectionTriple } from "./selection_queries"; +import type { GraphMeta, SelectionQueryMeta, SelectionTriple } from "./selection_queries"; import { TripleGraphView } from "./TripleGraphView"; import { buildTripleGraphModel, type TripleGraphModel } from "./triple_graph"; +import { readGraphArrow } from "./graph_arrow"; function sleep(ms: number): Promise { return new Promise((r) => setTimeout(r, ms)); } -type GraphNodeMeta = { - id?: number; - iri?: string; - label?: string; - x?: number; - y?: number; +type GraphNodeLookup = { + vertexIds: Uint32Array; + labels: (string | undefined)[]; + iris: string[]; }; -function graphRoutePoint(value: unknown): GraphRoutePoint | null { - if (!value || typeof value !== "object") return null; - const record = value as Record; - if (typeof record.x !== "number" || typeof record.y !== "number") return null; - return { - x: record.x, - y: record.y, - }; -} - -function graphRouteSegmentArray(value: unknown): GraphRouteSegment[] { - if (!Array.isArray(value)) return []; - const out: GraphRouteSegment[] = []; - for (const item of value) { - if (!item || typeof item !== "object") continue; - const record = item as Record; - if (typeof record.edge_index !== "number" || typeof record.kind !== "string") continue; - if (!Array.isArray(record.points)) continue; - const points: GraphRoutePoint[] = []; - for (const point of record.points) { - const parsed = graphRoutePoint(point); - if (!parsed) continue; - points.push(parsed); - } - if (points.length < 2) continue; - out.push({ - edge_index: record.edge_index, - kind: record.kind, - points, - }); - } - return out; -} - -function buildRouteLineVertices(routeSegments: GraphRouteSegment[]): Float32Array { - let lineCount = 0; - for (const route of routeSegments) { - lineCount += Math.max(0, route.points.length - 1); - } - - const out = new Float32Array(lineCount * 4); - let offset = 0; - for (const route of routeSegments) { - for (let i = 1; i < route.points.length; i++) { - const previous = route.points[i - 1]; - const current = route.points[i]; - out[offset++] = previous.x; - out[offset++] = previous.y; - out[offset++] = current.x; - out[offset++] = current.y; - } - } - return out; -} - type TripleResultState = { status: "idle" | "loading" | "ready" | "error"; queryId: string; @@ -92,6 +36,37 @@ function idleTripleResult(queryId: string): TripleResultState { }; } +function formatBytes(bytes: number): string { + if (!Number.isFinite(bytes) || bytes <= 0) return "0 B"; + const units = ["B", "KB", "MB", "GB", "TB"]; + let value = bytes; + let unitIndex = 0; + while (value >= 1024 && unitIndex < units.length - 1) { + value /= 1024; + unitIndex++; + } + return `${value.toFixed(value >= 100 || unitIndex === 0 ? 0 : 1)} ${units[unitIndex]}`; +} + +function estimateFrontendTypedArrayBytes(nodeCount: number, edgeCount: number, routeLineFloatCount: number): { app: number; renderer: number; total: number } { + const app = + nodeCount * Float32Array.BYTES_PER_ELEMENT * 2 + + nodeCount * Uint32Array.BYTES_PER_ELEMENT + + edgeCount * Uint32Array.BYTES_PER_ELEMENT * 2 + + routeLineFloatCount * Float32Array.BYTES_PER_ELEMENT; + + const renderer = + nodeCount * Float32Array.BYTES_PER_ELEMENT * 2 + + nodeCount * Uint32Array.BYTES_PER_ELEMENT * 4 + + edgeCount * Uint32Array.BYTES_PER_ELEMENT * 4; + + return { + app, + renderer, + total: app + renderer, + }; +} + export default function App() { const canvasRef = useRef(null); const rendererRef = useRef(null); @@ -121,14 +96,34 @@ export default function App() { // Store mouse position in a ref so it can be accessed in render loop without re-renders const mousePos = useRef({ x: 0, y: 0 }); - const nodesRef = useRef([]); + const nodeLookupRef = useRef(null); async function loadGraph(graphQueryId: string, signal: AbortSignal): Promise { const renderer = rendererRef.current; if (!renderer) return; - setStatus("Fetching graph…"); + const loadStartedAt = performance.now(); + const logPhase = (phase: string, extra?: Record) => { + console.log(`[graph-load] ${phase}`, { + elapsed_ms: Math.round(performance.now() - loadStartedAt), + graph_query_id: graphQueryId, + ...(extra || {}), + }); + }; + + const updateStatus = async (nextStatus: string, extra?: Record): Promise => { + setStatus(nextStatus); + logPhase(nextStatus, extra); + await sleep(0); + }; + + await updateStatus("Fetching graph…"); const graphRes = await fetch(`/api/graph?graph_query_id=${encodeURIComponent(graphQueryId)}`, { signal }); + logPhase("graph response headers received", { + status: graphRes.status, + content_type: graphRes.headers.get("content-type"), + content_length: graphRes.headers.get("content-length"), + }); if (!graphRes.ok) { let detail = ""; try { @@ -141,44 +136,57 @@ export default function App() { } throw new Error(`Failed to fetch graph: ${graphRes.status}${detail ? ` (${detail})` : ""}`); } - const graph = await graphRes.json(); + await updateStatus("Streaming Arrow graph…"); + await updateStatus("Decoding Arrow batches…"); + const decodeStartedAt = performance.now(); + let graph: Awaited>; + try { + graph = await readGraphArrow(graphRes, logPhase); + } catch (e) { + logPhase("arrow graph decode failed", { + error: e instanceof Error ? e.message : String(e), + }); + throw e; + } + logPhase("arrow graph decoded", { + decode_ms: Math.round(performance.now() - decodeStartedAt), + }); if (signal.aborted) return; - const nodes = Array.isArray(graph.nodes) ? graph.nodes : []; - const edges = Array.isArray(graph.edges) ? graph.edges : []; - const routeSegments = graphRouteSegmentArray(graph.route_segments); const meta = graph.meta || null; - const count = nodes.length; + const vertexIds = graph.vertexIds; + const xs = graph.xs; + const ys = graph.ys; + const edgeData = graph.edgeData; + const routeLineVertices = graph.routeLineVertices; + const labels = graph.labels; + const iris = graph.iris; + const count = vertexIds.length; + const edgeCount = edgeData.length / 2; + logPhase("graph payload ready", { + nodes: count, + edges: edgeCount, + route_line_segments: routeLineVertices.length / 4, + backend_nodes: meta && typeof meta.nodes === "number" ? meta.nodes : undefined, + backend_edges: meta && typeof meta.edges === "number" ? meta.edges : undefined, + }); - nodesRef.current = nodes; + nodeLookupRef.current = { vertexIds, labels, iris }; graphMetaRef.current = meta && typeof meta === "object" ? (meta as GraphMeta) : null; setTripleResult(idleTripleResult(activeSelectionQueryId)); - // Build positions from backend-provided node coordinates. - setStatus("Preparing buffers…"); - const xs = new Float32Array(count); - const ys = new Float32Array(count); - for (let i = 0; i < count; i++) { - const nx = nodes[i]?.x; - const ny = nodes[i]?.y; - xs[i] = typeof nx === "number" ? nx : 0; - ys[i] = typeof ny === "number" ? ny : 0; - } - const vertexIds = new Uint32Array(count); - for (let i = 0; i < count; i++) { - const id = nodes[i]?.id; - vertexIds[i] = typeof id === "number" ? id >>> 0 : i; - } - - // Build edges as vertex-id pairs. - const edgeData = new Uint32Array(edges.length * 2); - for (let i = 0; i < edges.length; i++) { - const s = edges[i]?.source; - const t = edges[i]?.target; - edgeData[i * 2] = typeof s === "number" ? s >>> 0 : 0; - edgeData[i * 2 + 1] = typeof t === "number" ? t >>> 0 : 0; - } - const routeLineVertices = buildRouteLineVertices(routeSegments); + await updateStatus("Preparing buffers…", { + nodes: count, + edges: edgeCount, + }); + const bufferPrepStartedAt = performance.now(); + const typedArrayBytes = estimateFrontendTypedArrayBytes(count, edgeCount, routeLineVertices.length); + logPhase("buffer prep done", { + buffer_prep_ms: Math.round(performance.now() - bufferPrepStartedAt), + app_typed_arrays: formatBytes(typedArrayBytes.app), + renderer_typed_arrays_estimate: formatBytes(typedArrayBytes.renderer), + total_typed_arrays_estimate: formatBytes(typedArrayBytes.total), + }); // Use /api/graph meta; don't do a second expensive backend call. if (meta && typeof meta.nodes === "number" && typeof meta.edges === "number") { @@ -188,33 +196,48 @@ export default function App() { backend: typeof meta.backend === "string" ? meta.backend : undefined, }); } else { - setBackendStats({ nodes: nodes.length, edges: edges.length }); + setBackendStats({ nodes: count, edges: edgeCount }); } - setStatus("Building spatial index…"); - await new Promise((r) => setTimeout(r, 0)); + await updateStatus("Building spatial index…", { + nodes: count, + edges: edgeCount, + }); if (signal.aborted) return; - const buildMs = renderer.init( - xs, - ys, - vertexIds, - edgeData, - routeLineVertices.length > 0 ? routeLineVertices : null - ); + let buildMs: number; + try { + buildMs = renderer.init( + xs, + ys, + vertexIds, + edgeData, + routeLineVertices.length > 0 ? routeLineVertices : null + ); + } catch (e) { + logPhase("renderer.init failed", { + error: e instanceof Error ? e.message : String(e), + }); + throw e; + } setNodeCount(renderer.getNodeCount()); setSelectedNodes(new Set()); setStatus(""); - console.log(`Init complete: ${count.toLocaleString()} nodes, ${edges.length.toLocaleString()} edges in ${buildMs.toFixed(0)}ms`); + logPhase("init complete", { + renderer_init_ms: Math.round(buildMs), + nodes: count, + edges: edgeCount, + }); } function getSelectedIds(renderer: Renderer, selected: Set): number[] { + const lookup = nodeLookupRef.current; + if (!lookup) return []; const selectedIds: number[] = []; for (const sortedIdx of selected) { const origIdx = renderer.sortedIndexToOriginalIndex(sortedIdx); if (origIdx === null) continue; - const node = nodesRef.current?.[origIdx]; - const nodeId = node?.id; + const nodeId = lookup.vertexIds[origIdx]; if (typeof nodeId !== "number") continue; selectedIds.push(nodeId); } @@ -370,14 +393,16 @@ export default function App() { const hit = renderer.findNodeIndexAt(mousePos.current.x, mousePos.current.y); if (hit) { const origIdx = renderer.sortedIndexToOriginalIndex(hit.index); - const meta = origIdx === null ? null : nodesRef.current[origIdx]; + const lookup = nodeLookupRef.current; + const label = origIdx === null || !lookup ? undefined : lookup.labels[origIdx]; + const iri = origIdx === null || !lookup ? undefined : lookup.iris[origIdx]; setHoveredNode({ x: hit.x, y: hit.y, screenX: mousePos.current.x, screenY: mousePos.current.y, - label: meta && typeof meta.label === "string" ? meta.label : undefined, - iri: meta && typeof meta.iri === "string" ? meta.iri : undefined, + label: typeof label === "string" ? label : undefined, + iri: typeof iri === "string" ? iri : undefined, }); } else { setHoveredNode(null); diff --git a/frontend/src/graph_arrow.ts b/frontend/src/graph_arrow.ts new file mode 100644 index 0000000..fb84531 --- /dev/null +++ b/frontend/src/graph_arrow.ts @@ -0,0 +1,301 @@ +import { RecordBatchReader } from "apache-arrow"; +import type { GraphMeta } from "./selection_queries"; + +export type ArrowGraphLoadResult = { + meta: GraphMeta | null; + vertexIds: Uint32Array; + xs: Float32Array; + ys: Float32Array; + edgeData: Uint32Array; + routeLineVertices: Float32Array; + labels: (string | undefined)[]; + iris: string[]; +}; + +type ArrowBatchLog = (phase: string, extra?: Record) => void; + +type ArrowLikeVector = { + data?: Array<{ valueOffsets?: ArrayLike }>; + getChildAt?: (index: number) => ArrowLikeVector | null; + get?: (index: number) => unknown; + toArray?: () => ArrayLike; +}; + +const graphTransportVersion = "1"; + +export async function readGraphArrow(response: Response, logPhase?: ArrowBatchLog): Promise { + validateArrowResponse(response); + + const reader = await openArrowReader(response); + + let meta: GraphMeta | null = null; + let vertexIds: Uint32Array | null = null; + let xs: Float32Array | null = null; + let ys: Float32Array | null = null; + let edgeData: Uint32Array | null = null; + let routeLineVertices: Float32Array | null = null; + let labels: (string | undefined)[] | null = null; + let iris: string[] | null = null; + + let batchIndex = 0; + for await (const batch of reader) { + const batchStartedAt = performance.now(); + + switch (batchIndex) { + case 0: { + meta = decodeMetaBatch(batch); + const nodeCount = typeof meta.nodes === "number" ? meta.nodes : 0; + const edgeCount = typeof meta.edges === "number" ? meta.edges : 0; + const routeLineSegments = readScalarNumber(batch, "meta_route_line_segments") ?? 0; + + vertexIds = new Uint32Array(nodeCount); + xs = new Float32Array(nodeCount); + ys = new Float32Array(nodeCount); + edgeData = new Uint32Array(edgeCount * 2); + routeLineVertices = new Float32Array(routeLineSegments * 4); + labels = new Array(nodeCount); + iris = new Array(nodeCount); + + logPhase?.("arrow_meta_batch_decoded", { + batch_index: batchIndex, + rows: batch.numRows, + node_count: nodeCount, + edge_count: edgeCount, + route_line_segments: routeLineSegments, + decode_ms: Math.round(performance.now() - batchStartedAt), + }); + break; + } + case 1: { + ensureAllocated(meta, vertexIds, xs, ys, edgeData, routeLineVertices, labels, iris); + const nodeIDs = readUint32List(batch, "node_id"); + const nodeXs = readFloat32List(batch, "node_x"); + const nodeYs = readFloat32List(batch, "node_y"); + const nodeIRIs = readStringList(batch, "node_iri"); + const nodeLabels = readNullableStringList(batch, "node_label"); + + if (nodeIDs.length !== vertexIds.length || nodeXs.length !== xs.length || nodeYs.length !== ys.length) { + throw new Error("Arrow node batch length mismatch"); + } + if (nodeIRIs.length !== iris.length || nodeLabels.length !== labels.length) { + throw new Error("Arrow node metadata batch length mismatch"); + } + + vertexIds.set(nodeIDs); + xs.set(nodeXs); + ys.set(nodeYs); + for (let i = 0; i < iris.length; i++) { + iris[i] = nodeIRIs[i] ?? ""; + labels[i] = nodeLabels[i]; + } + + logPhase?.("arrow_node_batch_decoded", { + batch_index: batchIndex, + rows: batch.numRows, + nodes: vertexIds.length, + decode_ms: Math.round(performance.now() - batchStartedAt), + }); + break; + } + case 2: { + ensureAllocated(meta, vertexIds, xs, ys, edgeData, routeLineVertices, labels, iris); + const edgeSources = readUint32List(batch, "edge_source"); + const edgeTargets = readUint32List(batch, "edge_target"); + if (edgeSources.length !== edgeTargets.length) { + throw new Error("Arrow edge batch source/target length mismatch"); + } + if (edgeData.length !== edgeSources.length * 2) { + throw new Error("Arrow edge batch size mismatch"); + } + for (let i = 0; i < edgeSources.length; i++) { + edgeData[i * 2] = edgeSources[i]; + edgeData[i * 2 + 1] = edgeTargets[i]; + } + logPhase?.("arrow_edge_batch_decoded", { + batch_index: batchIndex, + rows: batch.numRows, + edges: edgeSources.length, + decode_ms: Math.round(performance.now() - batchStartedAt), + }); + break; + } + case 3: { + ensureAllocated(meta, vertexIds, xs, ys, edgeData, routeLineVertices, labels, iris); + const routeX1 = readFloat32List(batch, "route_x1"); + const routeY1 = readFloat32List(batch, "route_y1"); + const routeX2 = readFloat32List(batch, "route_x2"); + const routeY2 = readFloat32List(batch, "route_y2"); + if ( + routeX1.length !== routeY1.length || + routeX1.length !== routeX2.length || + routeX1.length !== routeY2.length + ) { + throw new Error("Arrow route batch axis length mismatch"); + } + if (routeLineVertices.length !== routeX1.length * 4) { + throw new Error("Arrow route batch size mismatch"); + } + for (let i = 0; i < routeX1.length; i++) { + routeLineVertices[i * 4] = routeX1[i]; + routeLineVertices[i * 4 + 1] = routeY1[i]; + routeLineVertices[i * 4 + 2] = routeX2[i]; + routeLineVertices[i * 4 + 3] = routeY2[i]; + } + logPhase?.("arrow_route_batch_decoded", { + batch_index: batchIndex, + rows: batch.numRows, + route_line_segments: routeX1.length, + decode_ms: Math.round(performance.now() - batchStartedAt), + }); + break; + } + default: + throw new Error(`Unexpected Arrow batch index ${batchIndex}`); + } + + batchIndex++; + } + + if (batchIndex !== 4) { + throw new Error(`Expected 4 Arrow batches, received ${batchIndex}`); + } + ensureAllocated(meta, vertexIds, xs, ys, edgeData, routeLineVertices, labels, iris); + + return { + meta, + vertexIds, + xs, + ys, + edgeData, + routeLineVertices, + labels, + iris, + }; +} + +async function openArrowReader(response: Response) { + if (response.body) { + return RecordBatchReader.from(response.body); + } + return RecordBatchReader.from(await response.arrayBuffer()); +} + +function validateArrowResponse(response: Response): void { + const contentType = response.headers.get("content-type") || ""; + if (!contentType.includes("application/vnd.apache.arrow.stream")) { + throw new Error(`Unexpected graph content type: ${contentType || "(missing)"}`); + } + const version = response.headers.get("X-Graph-Transport-Version"); + if (version !== graphTransportVersion) { + throw new Error(`Unexpected graph transport version: ${version || "(missing)"}`); + } +} + +function decodeMetaBatch(batch: any): GraphMeta { + return { + backend: readScalarString(batch, "meta_backend"), + graph_query_id: readScalarString(batch, "meta_graph_query_id"), + node_limit: readScalarNumber(batch, "meta_node_limit"), + edge_limit: readScalarNumber(batch, "meta_edge_limit"), + nodes: readScalarNumber(batch, "meta_nodes"), + edges: readScalarNumber(batch, "meta_edges"), + }; +} + +function ensureAllocated( + meta: GraphMeta | null, + vertexIds: Uint32Array | null, + xs: Float32Array | null, + ys: Float32Array | null, + edgeData: Uint32Array | null, + routeLineVertices: Float32Array | null, + labels: (string | undefined)[] | null, + iris: string[] | null +): asserts meta is GraphMeta & {} & { + nodes?: number; + edges?: number; +} { + if (!meta || !vertexIds || !xs || !ys || !edgeData || !routeLineVertices || !labels || !iris) { + throw new Error("Arrow graph stream is missing the meta batch"); + } +} + +function readScalarString(batch: any, name: string): string | undefined { + const vector = batch.getChild(name); + if (!vector || batch.numRows < 1) return undefined; + const value = vector.get(0); + return typeof value === "string" ? value : undefined; +} + +function readScalarNumber(batch: any, name: string): number | undefined { + const vector = batch.getChild(name); + if (!vector || batch.numRows < 1) return undefined; + const value = vector.get(0); + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +} + +function readUint32List(batch: any, name: string): Uint32Array { + return readPrimitiveList(batch, name, Uint32Array); +} + +function readFloat32List(batch: any, name: string): Float32Array { + return readPrimitiveList(batch, name, Float32Array); +} + +function readPrimitiveList( + batch: any, + name: string, + ctor: { new(length: number): T } +): T { + const vector = batch.getChild(name) as ArrowLikeVector | null; + if (!vector) return new ctor(0); + + const [begin, end] = getListBounds(vector); + const child = vector.getChildAt?.(0); + if (!child || typeof child.toArray !== "function") { + return new ctor(0); + } + const values = child.toArray(); + if (typeof (values as any).subarray === "function") { + return (values as T).subarray(begin, end) as T; + } + + const out = new ctor(end - begin); + for (let i = begin; i < end; i++) { + out[i - begin] = Number((values as ArrayLike)[i]) as T[number]; + } + return out; +} + +function readStringList(batch: any, name: string): string[] { + return readNullableStringList(batch, name).map((value) => value ?? ""); +} + +function readNullableStringList(batch: any, name: string): (string | undefined)[] { + const vector = batch.getChild(name) as ArrowLikeVector | null; + if (!vector) return []; + + const [begin, end] = getListBounds(vector); + const child = vector.getChildAt?.(0); + if (!child || typeof child.get !== "function") { + return []; + } + + const out = new Array(Math.max(0, end - begin)); + for (let i = begin; i < end; i++) { + const value = child.get(i); + out[i - begin] = typeof value === "string" ? value : undefined; + } + return out; +} + +function getListBounds(vector: ArrowLikeVector): [number, number] { + const offsets = vector.data?.[0]?.valueOffsets; + if (!offsets || offsets.length < 2) return [0, 0]; + return [offsetToNumber(offsets[0]), offsetToNumber(offsets[1])]; +} + +function offsetToNumber(value: number | bigint | undefined): number { + if (typeof value === "bigint") return Number(value); + return typeof value === "number" ? value : 0; +} diff --git a/frontend/src/selection_queries/index.ts b/frontend/src/selection_queries/index.ts index a89f097..31fcd37 100644 --- a/frontend/src/selection_queries/index.ts +++ b/frontend/src/selection_queries/index.ts @@ -1,8 +1,6 @@ export { fetchSelectionQueries, runSelectionQuery, runSelectionTripleQuery } from "./api"; export type { GraphMeta, - GraphRoutePoint, - GraphRouteSegment, SelectionQueryMeta, SelectionTriple, SelectionTripleResult, diff --git a/frontend/src/selection_queries/types.ts b/frontend/src/selection_queries/types.ts index a85a998..938b407 100644 --- a/frontend/src/selection_queries/types.ts +++ b/frontend/src/selection_queries/types.ts @@ -1,26 +1,10 @@ export type GraphMeta = { backend?: string; - ttl_path?: string | null; - sparql_endpoint?: string | null; - include_bnodes?: boolean; graph_query_id?: string; node_limit?: number; edge_limit?: number; nodes?: number; edges?: number; - layout_engine?: string; - layout_root_iri?: string | null; -}; - -export type GraphRoutePoint = { - x: number; - y: number; -}; - -export type GraphRouteSegment = { - edge_index: number; - kind: string; - points: GraphRoutePoint[]; }; export type SelectionQueryMeta = {