Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Parameter transformation #587

Merged
merged 38 commits into from
Feb 20, 2024
Merged
Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
2374530
update wit
Natoandro Feb 6, 2024
bf75c3a
parameter transform tree
Natoandro Feb 7, 2024
494be63
feat: conversion
Natoandro Feb 9, 2024
7839418
update conversion - add failing test
Natoandro Feb 9, 2024
235f2f3
first working test
Natoandro Feb 9, 2024
f228610
type aware transformer compiler
Natoandro Feb 10, 2024
c3a9377
more tests/fixes
Natoandro Feb 10, 2024
5403bbc
test in-list injection
Natoandro Feb 10, 2024
61d519e
address comments (security)
Natoandro Feb 12, 2024
afc60b5
Merge branch 'main' into feat/apply
Natoandro Feb 12, 2024
d8b869e
inline validators
Natoandro Feb 12, 2024
188f13d
visitor context
Natoandro Feb 12, 2024
293e7ca
inheritance validator
Natoandro Feb 13, 2024
5d4875f
fix validators
Natoandro Feb 14, 2024
47da8bc
test from_parent validation
Natoandro Feb 14, 2024
2a214a0
some doc comments
Natoandro Feb 14, 2024
a798ff1
Merge branch 'main' into feat/apply
Natoandro Feb 14, 2024
b3eb807
TypeScript client SDK for apply
Natoandro Feb 14, 2024
d3734fc
docs: Add doc for parameter transformation
Natoandro Feb 14, 2024
46fb126
upgrade pnpm
Natoandro Feb 14, 2024
78c619c
tests: fix typegraph_core tests
Natoandro Feb 15, 2024
50b1d2a
more tests
Natoandro Feb 15, 2024
85f68d8
fix tests
Natoandro Feb 15, 2024
8211dc2
docs: update docs
Natoandro Feb 15, 2024
6f91dbe
disable docker test on arm64
Natoandro Feb 15, 2024
2070366
fix tests
Natoandro Feb 15, 2024
9a54f26
Merge branch 'main' into feat/apply
Natoandro Feb 15, 2024
ff1c92f
fix visitor
Natoandro Feb 16, 2024
8ed22ef
fix tests
Natoandro Feb 16, 2024
e282c41
fix input validator test
Natoandro Feb 16, 2024
1214cc9
Merge branch 'main' into feat/apply
Natoandro Feb 16, 2024
ddca6e8
update snapshots
Natoandro Feb 16, 2024
85df23a
fix tests
Natoandro Feb 17, 2024
c6b6687
Merge branch 'main' into feat/apply
Natoandro Feb 19, 2024
32ccc87
fix merge confusions
Natoandro Feb 19, 2024
113b81a
fix test
Natoandro Feb 19, 2024
a519f1f
update snapshot
Natoandro Feb 19, 2024
3e4af7d
Merge branch 'main' into feat/apply
Natoandro Feb 19, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
172 changes: 81 additions & 91 deletions Cargo.lock

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions libs/common/src/typegraph/mod.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Copyright Metatype OÜ, licensed under the Elastic License 2.0.
// SPDX-License-Identifier: Elastic-2.0

pub mod parameter_transform;
pub mod runtimes;
pub mod types;
pub mod utils;
Expand Down
60 changes: 60 additions & 0 deletions libs/common/src/typegraph/parameter_transform.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
// Copyright Metatype OÜ, licensed under the Elastic License 2.0.
// SPDX-License-Identifier: Elastic-2.0

use std::collections::HashMap;

#[cfg(feature = "codegen")]
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};

#[cfg_attr(feature = "codegen", derive(JsonSchema))]
#[derive(Serialize, Deserialize, Clone, Debug)]
#[serde(tag = "source", rename_all = "lowercase")]
pub enum ParameterTransformLeafNode {
#[serde(rename_all = "camelCase")]
Arg { name: String },
#[serde(rename_all = "camelCase")]
Static { value_json: String },
#[serde(rename_all = "camelCase")]
Secret { key: String },
#[serde(rename_all = "camelCase")]
Context { key: String },
#[serde(rename_all = "camelCase")]
Parent { parent_idx: u32 },
}

#[cfg_attr(feature = "codegen", derive(JsonSchema))]
#[derive(Serialize, Deserialize, Clone, Debug)]
#[serde(tag = "type", rename_all = "lowercase")]
pub enum ParameterTransformParentNode {
#[serde(rename_all = "camelCase")]
Object {
fields: HashMap<String, ParameterTransformNode>,
},
#[serde(rename_all = "camelCase")]
Array { items: Vec<ParameterTransformNode> },
}

#[cfg_attr(feature = "codegen", derive(JsonSchema))]
#[derive(Serialize, Deserialize, Clone, Debug)]
#[serde(untagged)]
pub enum ParameterTransformNodeData {
Leaf(ParameterTransformLeafNode),
Parent(ParameterTransformParentNode),
}

#[cfg_attr(feature = "codegen", derive(JsonSchema))]
#[derive(Serialize, Deserialize, Clone, Debug)]
#[serde(rename_all = "camelCase")]
pub struct ParameterTransformNode {
pub type_idx: u32,
// #[serde(flatten)]
pub data: ParameterTransformNodeData,
}

#[cfg_attr(feature = "codegen", derive(JsonSchema))]
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct FunctionParameterTransform {
pub resolver_input: u32,
pub transform_root: ParameterTransformNode,
}
6 changes: 4 additions & 2 deletions libs/common/src/typegraph/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use serde_with::skip_serializing_none;

use super::{EffectType, PolicyIndices};
use super::{parameter_transform::FunctionParameterTransform, EffectType, PolicyIndices};

#[cfg_attr(feature = "codegen", derive(JsonSchema))]
#[derive(Serialize, Deserialize, Clone, Debug)]
Expand Down Expand Up @@ -105,7 +105,7 @@ pub struct IntegerTypeData {
}

#[cfg_attr(feature = "codegen", derive(JsonSchema))]
#[derive(Serialize, Deserialize, Clone, Debug)]
#[derive(Serialize, Deserialize, Clone, Debug, PartialEq)]
#[serde(rename_all = "kebab-case")]
pub enum StringFormat {
Uuid,
Expand Down Expand Up @@ -166,6 +166,8 @@ pub struct ListTypeData {
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct FunctionTypeData {
pub input: u32,
#[serde(rename = "parameterTransform")]
pub parameter_transform: Option<FunctionParameterTransform>,
pub output: u32,
pub materializer: u32,
#[serialize_always]
Expand Down
93 changes: 93 additions & 0 deletions libs/common/src/typegraph/validator/common.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
// Copyright Metatype OÜ, licensed under the Elastic License 2.0.
// SPDX-License-Identifier: Elastic-2.0

use crate::typegraph::{EitherTypeData, TypeNode, Typegraph, UnionTypeData};

impl Typegraph {
pub fn check_enum_values(
&self,
type_idx: u32,
enum_values: &[String],
) -> Result<(), Vec<String>> {
let type_node = self.types.get(type_idx as usize).unwrap();
let mut errors = Vec::new();
if matches!(type_node, TypeNode::Optional { .. }) {
errors.push("optional not cannot have enumerated values".to_owned());
} else {
for value in enum_values {
match serde_json::from_str::<serde_json::Value>(value) {
Ok(val) => match self.validate_value(type_idx, &val) {
Ok(_) => {}
Err(err) => errors.push(err.to_string()),
},
Err(e) => errors.push(format!(
"Error while deserializing enum value {value:?}: {e:?}"
)),
}
}
}

if errors.is_empty() {
Ok(())
} else {
Err(errors)
}
}

pub fn collect_nested_variants_into(&self, out: &mut Vec<u32>, variants: &[u32]) {
for idx in variants {
let node = self.types.get(*idx as usize).unwrap();
match node {
TypeNode::Union {
data: UnionTypeData { any_of: variants },
..
}
| TypeNode::Either {
data: EitherTypeData { one_of: variants },
..
} => self.collect_nested_variants_into(out, variants),
_ => out.push(*idx),
}
}
}

pub fn check_union_variants(&self, variants: &[u32]) -> Result<(), String> {
let mut object_count = 0;

for variant_type in variants
.iter()
.map(|idx| self.types.get(*idx as usize).unwrap())
{
match variant_type {
TypeNode::Object { .. } => object_count += 1,
TypeNode::Boolean { .. }
| TypeNode::Float { .. }
| TypeNode::Integer { .. }
| TypeNode::String { .. } => {
// scalar
}
TypeNode::List { data, .. } => {
let item_type = self.types.get(data.items as usize).unwrap();
if !item_type.is_scalar() {
return Err(format!(
"array of '{}' not allowed as union/either variant",
item_type.type_name()
));
}
}
_ => {
return Err(format!(
"type '{}' not allowed as union/either variant",
variant_type.type_name()
));
}
}
}

if object_count != 0 && object_count != variants.len() {
return Err("union variants must either be all scalars or all objects".to_owned());
}

Ok(())
}
}
124 changes: 124 additions & 0 deletions libs/common/src/typegraph/validator/input.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
// Copyright Metatype OÜ, licensed under the Elastic License 2.0.
// SPDX-License-Identifier: Elastic-2.0

use serde_json::Value;

use crate::typegraph::{
visitor::{CurrentNode, TypeVisitor, VisitResult},
Injection, TypeNode,
};

use super::{
types::{EnsureSubtypeOf, ErrorCollector, ExtendedTypeNode},
TypeVisitorContext, Validator, ValidatorContext,
};

impl Validator {
pub fn visit_input_type_impl(
&mut self,
current_node: CurrentNode<'_>,
context: &<Validator as TypeVisitor>::Context,
) -> VisitResult<<Validator as TypeVisitor>::Return> {
let typegraph = context.get_typegraph();
let type_node = current_node.type_node;

match type_node {
TypeNode::Function { .. } => {
// TODO suggest to use composition-- when available
self.push_error(current_node.path, "Function is not allowed in input types.");
return VisitResult::Continue(false);
}
TypeNode::Union { .. } | TypeNode::Either { .. } => {
let mut variants = vec![];
typegraph.collect_nested_variants_into(&mut variants, &[current_node.type_idx]);
match typegraph.check_union_variants(&variants) {
Ok(_) => {}
Err(err) => {
self.push_error(current_node.path, err);
return VisitResult::Continue(false);
}
}
}
_ => (),
}

if let Some(injection) = &type_node.base().injection {
self.validate_injection(injection, current_node, context);
}

if let Some(enumeration) = &type_node.base().enumeration {
match context
.get_typegraph()
.check_enum_values(current_node.type_idx, enumeration)
{
Ok(_) => {}
Err(err) => {
for e in err {
self.push_error(current_node.path, e);
}
}
}
}

VisitResult::Continue(true)
}
}

impl Validator {
fn validate_injection(
&mut self,
injection: &Injection,
current_node: CurrentNode<'_>,
context: &ValidatorContext<'_>,
) {
let typegraph = context.get_typegraph();

match injection {
Injection::Static(data) => {
for value in data.values().iter() {
match serde_json::from_str::<Value>(value) {
Ok(val) => match typegraph.validate_value(current_node.type_idx, &val) {
Ok(_) => {}
Err(err) => self.push_error(current_node.path, err.to_string()),
},
Err(e) => {
self.push_error(
current_node.path,
format!(
"Error while parsing static injection value {value:?}: {e:?}",
value = value
),
);
}
}
}
}
Injection::Parent(data) => {
let sources = data.values();
for source_idx in sources.iter().copied() {
self.validate_parent_injection(*source_idx, &current_node, context);
}
}
_ => (),
}
}

fn validate_parent_injection(
&mut self,
source_idx: u32,
current_node: &CurrentNode<'_>,
context: &ValidatorContext<'_>,
) {
let typegraph = context.get_typegraph();
let source = ExtendedTypeNode::new(typegraph, source_idx);
let target = ExtendedTypeNode::new(typegraph, current_node.type_idx);
let mut errors = ErrorCollector::default();
source.ensure_subtype_of(&target, typegraph, &mut errors);
for error in errors.errors.into_iter() {
self.push_error(
current_node.path,
format!("from_parent injection: {error}", error = error),
);
}
}
}
Loading
Loading