Skip to content

Commit

Permalink
merge: #2719
Browse files Browse the repository at this point in the history
2719: feat(dal,sdf): change_set aware export of workspaces r=zacharyhamm a=zacharyhamm

Adds the ability for the nodes in our package graph to be empty of data
but still have child nodes. Adds the ability to track deleted objects
when exporting workspaces. Import is currently unchanged and only
supports "modules", which are a closure of the state of the active
change_set+head for specific schema variants.

Co-authored-by: Zachary Hamm <[email protected]>
  • Loading branch information
si-bors-ng[bot] and zacharyhamm authored Sep 12, 2023
2 parents 109e4b7 + 8cb9e7a commit 5499742
Show file tree
Hide file tree
Showing 62 changed files with 3,874 additions and 2,688 deletions.
25 changes: 8 additions & 17 deletions lib/dal/examples/dal-pkg-export/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ use std::{env, path::Path, sync::Arc};
use tokio::fs;

use dal::{
pkg::export_pkg_as_bytes, DalContext, JobQueueProcessor, NatsProcessor, Schema,
ServicesContext, Tenancy, Workspace,
pkg::PkgExporter, DalContext, JobQueueProcessor, NatsProcessor, Schema, ServicesContext,
StandardModel, Tenancy, Workspace,
};
use si_data_nats::{NatsClient, NatsConfig};
use si_data_pg::{PgPool, PgPoolConfig};
Expand All @@ -31,25 +31,16 @@ async fn main() -> Result<()> {
let workspace = Workspace::builtin(&ctx).await?;
ctx.update_tenancy(Tenancy::new(*workspace.pk()));

let mut variant_ids = Vec::new();
let mut schema_ids = Vec::new();
for schema_name in schema_names {
variant_ids.push(Schema::default_schema_variant_id_for_name(&ctx, schema_name).await?);
schema_ids.push(*Schema::find_by_name(&ctx, schema_name.trim()).await?.id());
}

println!("--- Exporting pkg: {tar_file}");
fs::write(
&tar_file,
export_pkg_as_bytes(
&ctx,
name,
version,
Some(description),
created_by,
variant_ids,
)
.await?,
)
.await?;
let mut exporter =
PkgExporter::new_module_exporter(name, version, Some(description), created_by, schema_ids);

fs::write(&tar_file, exporter.export_as_bytes(&ctx).await?).await?;

println!("--- Committing database transaction");
ctx.commit().await?;
Expand Down
15 changes: 13 additions & 2 deletions lib/dal/src/attribute/prototype.rs
Original file line number Diff line number Diff line change
Expand Up @@ -504,7 +504,7 @@ impl AttributePrototype {
ctx: &DalContext,
context: AttributeContext,
backend_response_type: FuncBackendResponseType,
) -> AttributePrototypeResult<Vec<Func>> {
) -> AttributePrototypeResult<Vec<(Self, Func)>> {
let rows = ctx
.txns()
.await?
Expand All @@ -521,7 +521,18 @@ impl AttributePrototype {
)
.await?;

Ok(standard_model::objects_from_rows(rows)?)
let mut result = Vec::new();
for row in rows.into_iter() {
let func_json: serde_json::Value = row.try_get("func_object")?;
let func: Func = serde_json::from_value(func_json)?;

let ap_json: serde_json::Value = row.try_get("prototype_object")?;
let ap: Self = serde_json::from_value(ap_json)?;

result.push((ap, func));
}

Ok(result)
}

pub async fn list_for_schema_variant(
Expand Down
5 changes: 5 additions & 0 deletions lib/dal/src/builtins/schema.rs
Original file line number Diff line number Diff line change
Expand Up @@ -102,11 +102,15 @@ pub async fn migrate_for_tests(
migrate_pkg(ctx, super::SI_AWS_EC2_PKG, None).await?;
migrate_pkg(ctx, super::SI_COREOS_PKG, None).await?;
migrate_pkg(ctx, super::SI_DOCKER_IMAGE_PKG, None).await?;
migrate_pkg(ctx, super::SI_GENERIC_FRAME_PKG, None).await?;
for test_schema in [BuiltinSchema::Starfield, BuiltinSchema::Fallout] {
migrate_schema(ctx, test_schema, &driver).await?;
ctx.blocking_commit().await?;
}
} else if migrate_test_exclusive {
// We migrate generic frame to get "si:resourceToPayloadValue" cheaply. This function
// should be converted to an intrinsic (or removed?)
migrate_pkg(ctx, super::SI_GENERIC_FRAME_PKG, None).await?;
for test_schema in [BuiltinSchema::Starfield, BuiltinSchema::Fallout] {
migrate_schema(ctx, test_schema, &driver).await?;
ctx.blocking_commit().await?;
Expand All @@ -120,6 +124,7 @@ pub async fn migrate_for_tests(
migrate_pkg(ctx, super::SI_AWS_EC2_PKG, Some(schemas.to_owned())).await?;
migrate_pkg(ctx, super::SI_COREOS_PKG, Some(schemas.to_owned())).await?;
migrate_pkg(ctx, super::SI_DOCKER_IMAGE_PKG, Some(schemas.to_owned())).await?;
migrate_pkg(ctx, super::SI_GENERIC_FRAME_PKG, Some(schemas.to_owned())).await?;
for test_schema in [BuiltinSchema::Starfield, BuiltinSchema::Fallout] {
if specific_builtin_schemas.contains(test_schema.real_schema_name()) {
migrate_schema(ctx, test_schema, &driver).await?;
Expand Down
98 changes: 72 additions & 26 deletions lib/dal/src/builtins/schema/test_exclusive_fallout.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
use si_pkg::{
ActionFuncSpec, AttrFuncInputSpec, AttrFuncInputSpecKind, FuncArgumentSpec, FuncSpec,
FuncSpecBackendKind, FuncSpecBackendResponseType, PkgSpec, PropSpec, SchemaSpec,
SchemaVariantSpec, SiPkg, SocketSpec, SocketSpecKind, ValidationSpec, ValidationSpecKind,
SchemaVariantSpec, SchemaVariantSpecData, SiPkg, SocketSpec, SocketSpecData, SocketSpecKind,
ValidationSpec, ValidationSpecKind,
};
use si_pkg::{FuncSpecData, SchemaSpecData};

use crate::func::argument::FuncArgumentKind;
use crate::func::intrinsics::IntrinsicFunc;
Expand All @@ -24,35 +26,56 @@ impl MigrationDriver {
let fallout_create_action_code = "async function create() {
return { payload: { \"poop\": true }, status: \"ok\" };
}";
let fn_name = "test:createActionFallout";
let fallout_create_action_func = FuncSpec::builder()
.name("test:createActionFallout")
.code_plaintext(fallout_create_action_code)
.handler("create")
.backend_kind(FuncSpecBackendKind::JsAction)
.response_type(FuncSpecBackendResponseType::Action)
.name(fn_name)
.unique_id(fn_name)
.data(
FuncSpecData::builder()
.name(fn_name)
.code_plaintext(fallout_create_action_code)
.handler("create")
.backend_kind(FuncSpecBackendKind::JsAction)
.response_type(FuncSpecBackendResponseType::Action)
.build()?,
)
.build()?;

let fallout_scaffold_func = "function createAsset() {\
return new AssetBuilder().build();
}";
let fn_name = "test:scaffoldFalloutAsset";
let fallout_authoring_schema_func = FuncSpec::builder()
.name("test:scaffoldFalloutAsset")
.code_plaintext(fallout_scaffold_func)
.handler("createAsset")
.backend_kind(FuncSpecBackendKind::JsSchemaVariantDefinition)
.response_type(FuncSpecBackendResponseType::SchemaVariantDefinition)
.name(fn_name)
.unique_id(fn_name)
.data(
FuncSpecData::builder()
.name(fn_name)
.code_plaintext(fallout_scaffold_func)
.handler("createAsset")
.backend_kind(FuncSpecBackendKind::JsSchemaVariantDefinition)
.response_type(FuncSpecBackendResponseType::SchemaVariantDefinition)
.build()?,
)
.build()?;

let fallout_resource_payload_to_value_func_code =
"async function translate(arg: Input): Promise<Output> {\
return arg.payload ?? {};
}";
let fn_name = "test:resourcePayloadToValue";
let fallout_resource_payload_to_value_func = FuncSpec::builder()
.name("test:resourcePayloadToValue")
.code_plaintext(fallout_resource_payload_to_value_func_code)
.handler("translate")
.backend_kind(FuncSpecBackendKind::JsAttribute)
.response_type(FuncSpecBackendResponseType::Json)
.name(fn_name)
.unique_id(fn_name)
.data(
FuncSpecData::builder()
.name(fn_name)
.code_plaintext(fallout_resource_payload_to_value_func_code)
.handler("translate")
.backend_kind(FuncSpecBackendKind::JsAttribute)
.response_type(FuncSpecBackendResponseType::Json)
.build()?,
)
.argument(
FuncArgumentSpec::builder()
.name("payload")
Expand All @@ -63,18 +86,31 @@ impl MigrationDriver {

let fallout_schema = SchemaSpec::builder()
.name("fallout")
.category("test exclusive")
.category_name("fallout")
.data(
SchemaSpecData::builder()
.name("fallout")
.category("test exclusive")
.category_name("fallout")
.build()
.expect("build schema spec data"),
)
.variant(
SchemaVariantSpec::builder()
.color("#ffffff")
.name("v0")
.func_unique_id(fallout_authoring_schema_func.unique_id)
.unique_id("fallout_sv")
.data(
SchemaVariantSpecData::builder()
.name("v0")
.color("#ffffff")
.func_unique_id(&fallout_authoring_schema_func.unique_id)
.build()
.expect("fallout variant data"),
)
.domain_prop(
PropSpec::builder()
.name("name")
.kind(PropKind::String)
.func_unique_id(identity_func_spec.unique_id)
.func_unique_id(&identity_func_spec.unique_id)
.input(
AttrFuncInputSpec::builder()
.kind(AttrFuncInputSpecKind::Prop)
Expand Down Expand Up @@ -118,8 +154,13 @@ impl MigrationDriver {
.socket(
SocketSpec::builder()
.name("bethesda")
.kind(SocketSpecKind::Output)
.func_unique_id(identity_func_spec.unique_id)
.data(
SocketSpecData::builder()
.name("bethesda")
.kind(SocketSpecKind::Output)
.func_unique_id(&identity_func_spec.unique_id)
.build()?,
)
.input(
AttrFuncInputSpec::builder()
.name("identity")
Expand All @@ -132,8 +173,13 @@ impl MigrationDriver {
.socket(
SocketSpec::builder()
.name("fallout")
.kind(SocketSpecKind::Output)
.func_unique_id(identity_func_spec.unique_id)
.data(
SocketSpecData::builder()
.name("fallout")
.kind(SocketSpecKind::Output)
.func_unique_id(&identity_func_spec.unique_id)
.build()?,
)
.input(
AttrFuncInputSpec::builder()
.name("identity")
Expand All @@ -146,7 +192,7 @@ impl MigrationDriver {
.action_func(
ActionFuncSpec::builder()
.kind(&ActionKind::Create)
.func_unique_id(fallout_create_action_func.unique_id)
.func_unique_id(&fallout_create_action_func.unique_id)
.build()?,
)
.build()?,
Expand Down
Loading

0 comments on commit 5499742

Please sign in to comment.