Skip to content
This repository was archived by the owner on Feb 6, 2026. It is now read-only.

Commit c064155

Browse files
si-bors-ng[bot]zacharyhamm
si-bors-ng[bot]
and
zacharyhamm
authored
merge: #3483
3483: feat(dal,si-events): Move node weights off the graph r=zacharyhamm a=zacharyhamm Moves node weights off the internal petgraph, storing them in the layerdb. Local node weights now only store the address and the merkel tree hash (although we may move that off the local node weight as well). Co-authored-by: Zachary Hamm <zack@systeminit.com>
2 parents bbd1995 + 30be50d commit c064155

60 files changed

Lines changed: 6955 additions & 8493 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Cargo.lock‎

Lines changed: 66 additions & 68 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ directories = "5.0.1"
9494
docker-api = "0.14.0"
9595
dyn-clone = "1.0.17"
9696
flate2 = "1.0.28"
97-
futures = "0.3.30"
97+
futures = {version = "0.3.30", features = ["executor"]}
9898
futures-lite = "2.3.0"
9999
hex = "0.4.3"
100100
http = "0.2.12" # todo: upgrade this alongside hyper/axum/tokio-tungstenite/tower-http

‎lib/dal-test/BUCK‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@ rust_library(
55
deps = [
66
"//lib/buck2-resources:buck2-resources",
77
"//lib/council-server:council-server",
8-
"//lib/dal:dal",
8+
"//lib/dal:dal-integration-test",
99
"//lib/module-index-client:module-index-client",
10-
"//lib/pinga-server:pinga-server",
11-
"//lib/rebaser-server:rebaser-server",
10+
"//lib/pinga-server:pinga-server-integration-test",
11+
"//lib/rebaser-server:rebaser-server-integration-test",
1212
"//lib/si-crypto:si-crypto",
1313
"//lib/si-data-nats:si-data-nats",
1414
"//lib/si-data-pg:si-data-pg",

‎lib/dal-test/src/lib.rs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,8 @@ impl Config {
138138
config.pg.port = env::var(ENV_VAR_PG_PORT)
139139
.unwrap_or_else(|_| DEFAULT_TEST_PG_PORT_STR.to_string())
140140
.parse()?;
141-
config.pg.pool_max_size *= 32;
141+
142+
config.pg.pool_max_size = 4;
142143
config.pg.certificate_path = Some(config.postgres_key_path.clone().try_into()?);
143144

144145
if let Ok(value) = env::var(ENV_VAR_PG_HOSTNAME) {
@@ -151,7 +152,7 @@ impl Config {
151152
config.layer_cache_pg_pool.port = env::var(ENV_VAR_PG_PORT)
152153
.unwrap_or_else(|_| DEFAULT_TEST_PG_PORT_STR.to_string())
153154
.parse()?;
154-
config.layer_cache_pg_pool.pool_max_size *= 32;
155+
config.layer_cache_pg_pool.pool_max_size = 4;
155156
config.layer_cache_pg_pool.certificate_path =
156157
Some(config.postgres_key_path.clone().try_into()?);
157158

‎lib/dal/BUCK‎

Lines changed: 81 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
load(
22
"@prelude-si//:macros.bzl",
33
"rust_library",
4+
"rust_library_integration_test",
45
"rust_test",
56
)
67

@@ -75,6 +76,83 @@ rust_library(
7576
test_unit_deps = [
7677
"//third-party/rust:tempfile",
7778
],
79+
)
80+
81+
rust_library_integration_test(
82+
name = "dal-integration-test",
83+
crate = "dal",
84+
deps = [
85+
"//lib/si-cbor:si-cbor",
86+
"//lib/council-server:council-server",
87+
"//lib/nats-subscriber:nats-subscriber",
88+
"//lib/object-tree:object-tree",
89+
"//lib/si-crypto:si-crypto",
90+
"//lib/si-data-nats:si-data-nats",
91+
"//lib/si-data-pg:si-data-pg",
92+
"//lib/si-events-rs:si-events",
93+
"//lib/si-hash:si-hash",
94+
"//lib/si-layer-cache:si-layer-cache",
95+
"//lib/si-pkg:si-pkg",
96+
"//lib/si-std:si-std",
97+
"//lib/telemetry-rs:telemetry",
98+
"//lib/telemetry-nats-rs:telemetry-nats",
99+
"//lib/veritech-client:veritech-client",
100+
"//third-party/rust:async-recursion",
101+
"//third-party/rust:async-trait",
102+
"//third-party/rust:base64",
103+
"//third-party/rust:blake3",
104+
"//third-party/rust:chrono",
105+
"//third-party/rust:ciborium",
106+
"//third-party/rust:convert_case",
107+
"//third-party/rust:derive_more",
108+
"//third-party/rust:diff",
109+
"//third-party/rust:dyn-clone",
110+
"//third-party/rust:futures",
111+
"//third-party/rust:hex",
112+
"//third-party/rust:iftree",
113+
"//third-party/rust:itertools",
114+
"//third-party/rust:jwt-simple",
115+
"//third-party/rust:lazy_static",
116+
"//third-party/rust:once_cell",
117+
"//third-party/rust:paste",
118+
"//third-party/rust:petgraph",
119+
"//third-party/rust:postcard",
120+
"//third-party/rust:postgres-types",
121+
"//third-party/rust:pretty_assertions_sorted",
122+
"//third-party/rust:rand",
123+
"//third-party/rust:refinery",
124+
"//third-party/rust:regex",
125+
"//third-party/rust:remain",
126+
"//third-party/rust:serde",
127+
"//third-party/rust:serde-aux",
128+
"//third-party/rust:serde_json",
129+
"//third-party/rust:serde_with",
130+
"//third-party/rust:sled",
131+
"//third-party/rust:sodiumoxide",
132+
"//third-party/rust:strum",
133+
"//third-party/rust:thiserror",
134+
"//third-party/rust:tokio",
135+
"//third-party/rust:tokio-stream",
136+
"//third-party/rust:ulid",
137+
"//third-party/rust:url",
138+
],
139+
rustc_flags = [
140+
"--cfg=integration_test",
141+
],
142+
srcs = glob([
143+
"src/**/*.rs",
144+
"src/builtins/func/**",
145+
"src/builtins/schema/data/**/*.json",
146+
"src/builtins/schema/definitions/**/*.json",
147+
"src/migrations/**/*.sql",
148+
"src/queries/**/*.sql",
149+
]),
150+
env = {
151+
"CARGO_MANIFEST_DIR": ".",
152+
},
153+
test_unit_deps = [
154+
"//third-party/rust:tempfile",
155+
],
78156
extra_test_targets = [":test-integration"],
79157
)
80158

@@ -83,7 +161,8 @@ rust_test(
83161
deps = [
84162
"//lib/dal-test:dal-test",
85163
"//lib/rebaser-core:rebaser-core",
86-
"//lib/rebaser-server:rebaser-server",
164+
"//lib/rebaser-server:rebaser-server-integration-test",
165+
"//lib/si-events-rs:si-events",
87166
"//lib/si-pkg:si-pkg",
88167
"//lib/veritech-client:veritech-client",
89168
"//third-party/rust:base64",
@@ -98,7 +177,7 @@ rust_test(
98177
"//third-party/rust:tokio",
99178
"//third-party/rust:tokio-util",
100179
"//third-party/rust:ulid",
101-
":dal",
180+
":dal-integration-test",
102181
],
103182
crate_root = "tests/integration.rs",
104183
srcs = glob([

‎lib/dal/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,5 +70,6 @@ veritech-client = { path = "../../lib/veritech-client" }
7070

7171
itertools = { workspace = true }
7272
pretty_assertions_sorted = { workspace = true }
73+
si-events = { path = "../../lib/si-events-rs" }
7374
tempfile = { workspace = true }
7475
tokio-util = { workspace = true }

‎lib/dal/src/action/prototype.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -282,7 +282,7 @@ impl ActionPrototype {
282282
{
283283
let node_weight = workspace_snapshot.get_node_weight(node_index).await?;
284284
let id = node_weight.id();
285-
if NodeWeightDiscriminants::Func == node_weight.into() {
285+
if NodeWeightDiscriminants::Func == node_weight.as_ref().into() {
286286
return Ok(id.into());
287287
}
288288
}

‎lib/dal/src/attribute/prototype.rs‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ impl AttributePrototype {
157157
{
158158
let node_weight = workspace_snapshot.get_node_weight(node_index).await?;
159159
let node_weight_id = node_weight.id();
160-
if NodeWeightDiscriminants::Func == node_weight.into() {
160+
if NodeWeightDiscriminants::Func == node_weight.as_ref().into() {
161161
return Ok(node_weight_id.into());
162162
}
163163
}
@@ -356,6 +356,7 @@ impl AttributePrototype {
356356
let (target_id, edge_weight_discrim) = match workspace_snapshot
357357
.get_node_weight(prototype_edge_source)
358358
.await?
359+
.as_ref()
359360
{
360361
NodeWeight::Prop(prop_inner) => {
361362
(prop_inner.id(), EdgeWeightKindDiscriminants::Prop)
@@ -390,6 +391,7 @@ impl AttributePrototype {
390391
if let NodeWeight::AttributeValue(av_node_weight) = workspace_snapshot
391392
.get_node_weight(attribute_value_target)
392393
.await?
394+
.as_ref()
393395
{
394396
attribute_value_ids.push(av_node_weight.id().into())
395397
}
@@ -426,8 +428,10 @@ impl AttributePrototype {
426428

427429
Ok(match maybe_value_idxs.first().copied() {
428430
Some(value_idx) => {
429-
if let NodeWeight::AttributeValue(av_node_weight) =
430-
workspace_snapshot.get_node_weight(value_idx).await?
431+
if let NodeWeight::AttributeValue(av_node_weight) = workspace_snapshot
432+
.get_node_weight(value_idx)
433+
.await?
434+
.as_ref()
431435
{
432436
Some(av_node_weight.id().into())
433437
} else {

‎lib/dal/src/attribute/prototype/argument.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ impl AttributePrototypeArgument {
134134
)
135135
.await?
136136
{
137-
match workspace_snapshot.get_node_weight(node_idx).await? {
137+
match workspace_snapshot.get_node_weight(node_idx).await?.as_ref() {
138138
NodeWeight::Content(inner) => {
139139
let inner_addr_discrim: ContentAddressDiscriminants =
140140
inner.content_address().into();
@@ -318,7 +318,7 @@ impl AttributePrototypeArgument {
318318
.into_iter()
319319
.next()
320320
{
321-
match workspace_snapshot.get_node_weight(target).await? {
321+
match workspace_snapshot.get_node_weight(target).await?.as_ref() {
322322
NodeWeight::Prop(inner) => {
323323
return Ok(Some(ValueSource::Prop(inner.id().into())));
324324
}
@@ -513,7 +513,7 @@ impl AttributePrototypeArgument {
513513

514514
for idx in apa_node_idxs {
515515
let node_weight = workspace_snapshot.get_node_weight(idx).await?;
516-
if let NodeWeight::AttributePrototypeArgument(apa_weight) = &node_weight {
516+
if let NodeWeight::AttributePrototypeArgument(apa_weight) = node_weight.as_ref() {
517517
if let Some(ArgumentTargets {
518518
destination_component_id,
519519
..

‎lib/dal/src/attribute/value.rs‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -946,8 +946,10 @@ impl AttributeValue {
946946
let workspace_snapshot = ctx.workspace_snapshot()?;
947947

948948
let prop_node_index = workspace_snapshot.get_node_index_by_id(prop_id).await?;
949-
if let NodeWeight::Prop(prop_inner) =
950-
workspace_snapshot.get_node_weight(prop_node_index).await?
949+
if let NodeWeight::Prop(prop_inner) = workspace_snapshot
950+
.get_node_weight(prop_node_index)
951+
.await?
952+
.as_ref()
951953
{
952954
prop_inner.kind()
953955
} else {
@@ -1275,6 +1277,7 @@ impl AttributeValue {
12751277
.workspace_snapshot()?
12761278
.get_node_weight(node_index)
12771279
.await?
1280+
.as_ref()
12781281
{
12791282
prop_map.insert(
12801283
prop_inner.name().to_string(),
@@ -1372,6 +1375,7 @@ impl AttributeValue {
13721375
match workspace_snapshot
13731376
.get_node_weight(element_prop_index)
13741377
.await?
1378+
.as_ref()
13751379
{
13761380
NodeWeight::Prop(prop_inner) => (prop_inner.id(), prop_inner.kind()),
13771381
_ => {
@@ -1616,6 +1620,7 @@ impl AttributeValue {
16161620
match workspace_snapshot
16171621
.get_node_weight(element_prop_index)
16181622
.await?
1623+
.as_ref()
16191624
{
16201625
NodeWeight::Prop(prop_inner) => (prop_inner.id(), prop_inner.kind()),
16211626
_ => {
@@ -1817,7 +1822,7 @@ impl AttributeValue {
18171822
view: Option<serde_json::Value>,
18181823
) -> AttributeValueResult<()> {
18191824
let workspace_snapshot = ctx.workspace_snapshot()?;
1820-
let (av_idx, av_node_weight) = {
1825+
let (_, av_node_weight) = {
18211826
let av_idx = workspace_snapshot
18221827
.get_node_index_by_id(attribute_value_id)
18231828
.await?;
@@ -1862,7 +1867,6 @@ impl AttributeValue {
18621867
workspace_snapshot
18631868
.add_node(NodeWeight::AttributeValue(new_av_node_weight))
18641869
.await?;
1865-
workspace_snapshot.replace_references(av_idx).await?;
18661870

18671871
Ok(())
18681872
}
@@ -1877,7 +1881,7 @@ impl AttributeValue {
18771881
func_execution_pk: FuncExecutionPk,
18781882
) -> AttributeValueResult<()> {
18791883
let workspace_snapshot = ctx.workspace_snapshot()?;
1880-
let (av_idx, av_node_weight) = {
1884+
let (_av_idx, av_node_weight) = {
18811885
let av_idx = workspace_snapshot
18821886
.get_node_index_by_id(attribute_value_id)
18831887
.await?;
@@ -1938,7 +1942,6 @@ impl AttributeValue {
19381942
workspace_snapshot
19391943
.add_node(NodeWeight::AttributeValue(new_av_node_weight))
19401944
.await?;
1941-
workspace_snapshot.replace_references(av_idx).await?;
19421945

19431946
Ok(())
19441947
}
@@ -1975,7 +1978,7 @@ impl AttributeValue {
19751978
.await?
19761979
{
19771980
let target_node_weight = workspace_snapshot.get_node_weight(target).await?;
1978-
if let NodeWeight::Prop(prop_node_weight) = &target_node_weight {
1981+
if let NodeWeight::Prop(prop_node_weight) = target_node_weight.as_ref() {
19791982
maybe_prop_id = match maybe_prop_id {
19801983
Some(already_found_prop_id) => {
19811984
return Err(AttributeValueError::MultiplePropsFound(
@@ -2094,7 +2097,7 @@ impl AttributeValue {
20942097
.pop()
20952098
{
20962099
let node_weight = workspace_snapshot.get_node_weight(ordering).await?;
2097-
if let NodeWeight::Ordering(ordering_weight) = node_weight {
2100+
if let NodeWeight::Ordering(ordering_weight) = node_weight.as_ref() {
20982101
Ok(ordering_weight
20992102
.order()
21002103
.clone()

0 commit comments

Comments
 (0)