diff --git a/docs/workflows/artifact_upload_protocol.drawio.svg b/docs/workflows/artifact_upload_protocol.drawio.svg new file mode 100644 index 000000000..649682440 --- /dev/null +++ b/docs/workflows/artifact_upload_protocol.drawio.svg @@ -0,0 +1,304 @@ + + + + + + + +
+
+
+ + Artifact Upload Protocol + +
+
+
+ + + Artifact Upload Protocol + + + + + + + +
+
+
+ + Typegate + +
+
+
+ + + Typegate + + + + + + + + + +
+
+
+ + Typegate request handle + +
+
+
+ + + Typegate request ha... + + + + + + + +
+
+
+ + artifact upload service + +
+
+
+ + + artifact upload ser... + + + + + + +
+
+
+ + upload to typegate + +
+
+
+ + + upload to typ... + + + + + + +
+
+
+ + save artifacts + +
+
+
+ + + save artifacts + + + + + + + +
+
+
+ + Typegraph SDK + +
+
+
+ + + Typegraph SDK + + + + + + + +
+
+
+ + Runtime + +
+
+
+ + + Runtime + + + + + + + +
+
+
+ + TgDeploy + +
+
+
+ + + TgDeploy + + + + + + + +
+
+
+ + Typegraph Core + +
+
+
+ + + Typegraph Core + + + + + + + +
+
+
+ + TgContext + +
+
+
+ + + TgContext + + + + + + + +
+
+
+ + finalize_typegraph + +
+
+
+ + + finalize_typegraph + + + + + + + +
+
+
+ + Typegate request handle + +
+
+
+ + + Typegate request ha... + + + + + + + + + + + + + + +
+
+
+ + save runtime artifacts meta during tg definition + +
+
+
+ + + save runtime artifacts meta... + + + + + + +
+
+
+ + during tg finalization, retrieve all referred artifacts meta + +
+
+
+ + + during tg finalization, retriev... + + + + + + + + + Text is not SVG - cannot display + + + + diff --git a/libs/common/src/typegraph/mod.rs b/libs/common/src/typegraph/mod.rs index 1e4802bd5..08ff1ea1b 100644 --- a/libs/common/src/typegraph/mod.rs +++ b/libs/common/src/typegraph/mod.rs @@ -10,6 +10,7 @@ pub mod visitor; pub use types::*; +use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -89,6 +90,14 @@ pub struct Queries { pub endpoints: Vec, } +// #[cfg_attr(feature = "codegen", derive(JsonSchema))] +// #[derive(Serialize, Deserialize, Clone, Debug, Default)] +// pub struct DependencyMeta { +// pub name: String, +// pub dep_hash: String, +// pub relative_path_prefix: PathBuf, +// } + #[cfg_attr(feature = "codegen", derive(JsonSchema))] #[derive(Serialize, Deserialize, Clone, Debug, Default)] #[serde(rename_all = "camelCase")] diff --git a/libs/common/src/typegraph/runtimes/python.rs b/libs/common/src/typegraph/runtimes/python.rs index 6ccf1e531..6a764d506 100644 --- a/libs/common/src/typegraph/runtimes/python.rs +++ b/libs/common/src/typegraph/runtimes/python.rs @@ -8,7 +8,9 @@ use serde::{Deserialize, Serialize}; #[cfg_attr(feature = "codegen", derive(JsonSchema))] #[derive(Serialize, Deserialize, Clone, Debug)] pub struct ModuleMatData { - pub code: String, + pub artifact: String, + pub artifact_hash: String, + pub tg_name: Option, } #[cfg_attr(feature = "codegen", derive(JsonSchema))] diff --git a/meta-cli/src/cli/deploy.rs b/meta-cli/src/cli/deploy.rs index 0ad4f42f3..9eb26ec14 100644 --- a/meta-cli/src/cli/deploy.rs +++ b/meta-cli/src/cli/deploy.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; +use std::process::exit; use std::sync::{Arc, Mutex}; use super::{Action, CommonArgs, GenArgs}; diff --git a/meta-cli/src/cli/serialize.rs b/meta-cli/src/cli/serialize.rs index b1c38a03a..48937a70a 100644 --- a/meta-cli/src/cli/serialize.rs +++ b/meta-cli/src/cli/serialize.rs @@ -15,6 +15,7 @@ use common::typegraph::Typegraph; use core::fmt::Debug; use std::io::{self, Write}; use std::path::PathBuf; +use std::process::exit; use std::sync::Arc; use tokio::io::AsyncWriteExt; use tokio::sync::mpsc; diff --git a/typegate/src/runtimes/python_wasi/python_wasi.ts b/typegate/src/runtimes/python_wasi/python_wasi.ts index a8c76c594..fc54cc766 100644 --- a/typegate/src/runtimes/python_wasi/python_wasi.ts +++ b/typegate/src/runtimes/python_wasi/python_wasi.ts @@ -3,15 +3,15 @@ import { getLogger } from "../../log.ts"; import { Runtime } from "../Runtime.ts"; +import { basename } from "std/path/mod.ts"; import { Resolver, RuntimeInitParams } from "../../types.ts"; import { ComputeStage } from "../../engine/query_engine.ts"; import { PythonWasmMessenger } from "./python_wasm_messenger.ts"; import { path } from "compress/deps.ts"; import { PythonVirtualMachine } from "./python_vm.ts"; import { Materializer } from "../../typegraph/types.ts"; -import { structureRepr } from "../../utils.ts"; -import { uncompress } from "../../utils.ts"; import * as ast from "graphql/ast"; +import config from "../../config.ts"; const logger = getLogger(import.meta); @@ -70,26 +70,36 @@ export class PythonWasiRuntime extends Runtime { } case "import_function": { const pyModMat = typegraph.materializers[m.data.mod as number]; - const code = pyModMat.data.code as string; + // const code = pyModMat.data.code as string; - const repr = await structureRepr(code); - const vmId = generateVmIdentifier(m, uuid); - const basePath = path.join( - "tmp", - "scripts", + // resolve the python module artifacts/files + const { artifact, artifact_hash } = pyModMat.data; + + const outDir = path.join( + config.tmp_dir, + "metatype_artifacts", typegraphName, - uuid, - "python", - vmId, + "artifacts", + artifact_hash as string, ); - const outDir = path.join(basePath, repr.hashes.entryPoint); - const entries = await uncompress( - outDir, - repr.base64, - ); - logger.info(`uncompressed ${entries.join(", ")} at ${outDir}`); - const modName = path.parse(repr.entryPoint).name; + // const repr = await structureRepr(code); + const vmId = generateVmIdentifier(m, uuid); + // const basePath = path.join( + // "tmp", + // "scripts", + // typegraphName, + // uuid, + // "python", + // vmId, + // ); + // const entries = await uncompress( + // outDir, + // repr.base64, + // ); + // logger.info(`uncompressed ${entries.join(", ")} at ${outDir}`); + + const modName = basename(artifact as string); // TODO: move this logic to postprocess or python runtime m.data.name = `${modName}.${m.data.name as string}`; @@ -98,7 +108,8 @@ export class PythonWasiRuntime extends Runtime { const vm = new PythonVirtualMachine(); // for python modules, imports must be inside a folder above or same directory - const entryPointFullPath = path.join(outDir, repr.entryPoint); + const entryFile = artifact as string + "." + artifact_hash as string; + const entryPointFullPath = path.join(outDir, entryFile); const sourceCode = Deno.readTextFileSync(entryPointFullPath); // prepare vm diff --git a/typegate/src/services/artifact_upload_service.ts b/typegate/src/services/artifact_upload_service.ts new file mode 100644 index 000000000..1a8a81db3 --- /dev/null +++ b/typegate/src/services/artifact_upload_service.ts @@ -0,0 +1,141 @@ +// Copyright Metatype OÜ, licensed under the Elastic License 2.0. +// SPDX-License-Identifier: Elastic-2.0 + +import config from "../config.ts"; +import { signJWT, verifyJWT } from "../crypto.ts"; +import { UploadUrlMeta } from "../typegate/mod.ts"; +import { path } from "compress/deps.ts"; + +interface TypegraphArtifact { + name: string; + artifact_hash: string; + artifact_size_in_bytes: number; + path_suffix: string[]; +} + +function createUploadPath(origin: string, typegraphName: string) { + const rand_path = crypto.randomUUID(); + return `${origin}/${typegraphName}/upload-artifacts/artifacts/${rand_path}`; +} + +export async function handleUploadUrl( + request: Request, + tgName: string, + urlCache: Map, +) { + const url = new URL(request.url); + const origin = url.origin; + const { name, artifact_hash, artifact_size_in_bytes, path_suffix }: + TypegraphArtifact = await request + .json(); + + const artifactPathSuffix = path_suffix.length > 0 + ? path.join(...path_suffix) + : ""; + const uploadUrlMeta: UploadUrlMeta = { + artifactName: name, + artifactHash: artifact_hash, + artifactSizeInBytes: artifact_size_in_bytes, + pathSuffix: artifactPathSuffix, + urlUsed: false, + }; + + let uploadUrl = createUploadPath(origin, tgName); + const expiresIn = 5 * 60; // 5 minutes + const payload = { + "expiresIn": expiresIn, + }; + const token = await signJWT(payload, expiresIn); + uploadUrl = `${uploadUrl}?token=${token}`; + + urlCache.set(uploadUrl, uploadUrlMeta); + + // console.log("**************R", uploadUrlMeta); + return new Response(JSON.stringify({ uploadUrl: uploadUrl })); +} + +export async function handleArtifactUpload( + request: Request, + tgName: string, + urlCache: Map, +) { + const url = new URL(request.url); + if (request.method !== "PUT") { + throw new Error( + `${url.pathname} does not support ${request.method} method`, + ); + } + + const uploadMeta = urlCache.get(url.toString()); + + if (!uploadMeta) { + throw new Error(`Endpoint ${url.toString()} does not exist`); + } + + const token = url.searchParams.get("token"); + try { + const _ = await verifyJWT(token!); + } catch (e) { + throw new Error("Invalid token: " + e.toString()); + } + + const { + artifactName, + artifactHash, + artifactSizeInBytes, + urlUsed, + pathSuffix, + }: UploadUrlMeta = uploadMeta!; + + if (urlUsed) { + throw new Error(`Endpoint ${url.toString()} is disabled`); + } + + const reader = request.body?.getReader()!; + + let artifactData = new Uint8Array(); + let bytesRead = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) { + break; + } + if (value) { + bytesRead += value.length; + const temp = new Uint8Array(artifactData.length + value.length); + temp.set(artifactData); + temp.set(value, artifactData.length); + artifactData = temp; + } + } + + if (bytesRead !== artifactSizeInBytes) { + throw new Error( + `File size does not match ${bytesRead}, ${JSON.stringify(uploadMeta)}`, + ); + } + + // adjust relative to the root path + const artifactStorageDir = path.join( + config.tmp_dir, + "metatype_artifacts", + tgName, + "artifacts", + pathSuffix, + ); + + await Deno.mkdir(artifactStorageDir, { recursive: true }); + const artifactPath = path.join( + artifactStorageDir, + `${artifactName}.${artifactHash}`, + ); + await Deno.writeFile(artifactPath, artifactData); + + // mark as the url used once the request completes. + uploadMeta.urlUsed = true; + urlCache.set(url.toString(), uploadMeta); + + return new Response(JSON.stringify({ + "success": true, + })); +} diff --git a/typegate/src/typegraphs/prisma_migration.json b/typegate/src/typegraphs/prisma_migration.json index e2712ad23..d21a3d2f8 100644 --- a/typegate/src/typegraphs/prisma_migration.json +++ b/typegate/src/typegraphs/prisma_migration.json @@ -15,21 +15,13 @@ "deploy": 14, "reset": 18 }, - "required": [ - "diff", - "apply", - "create", - "deploy", - "reset" - ] + "required": ["diff", "apply", "create", "deploy", "reset"] }, { "type": "function", "title": "func_10", "runtime": 1, - "policies": [ - 0 - ], + "policies": [0], "config": {}, "as_id": false, "input": 2, @@ -95,9 +87,7 @@ "type": "function", "title": "func_23", "runtime": 1, - "policies": [ - 0 - ], + "policies": [0], "config": {}, "as_id": false, "input": 8, @@ -147,9 +137,7 @@ "type": "function", "title": "func_40", "runtime": 1, - "policies": [ - 0 - ], + "policies": [0], "config": {}, "as_id": false, "input": 12, @@ -193,9 +181,7 @@ "type": "function", "title": "func_51", "runtime": 1, - "policies": [ - 0 - ], + "policies": [0], "config": {}, "as_id": false, "input": 15, @@ -243,9 +229,7 @@ "type": "function", "title": "func_58", "runtime": 1, - "policies": [ - 0 - ], + "policies": [0], "config": {}, "as_id": false, "input": 19, @@ -366,9 +350,7 @@ "name": "basic", "protocol": "basic", "auth_data": { - "users": [ - "admin" - ] + "users": ["admin"] } } ], @@ -383,4 +365,4 @@ "randomSeed": null, "artifacts": [] } -} \ No newline at end of file +} diff --git a/typegate/src/typegraphs/typegate.json b/typegate/src/typegraphs/typegate.json index 94182bd4d..64e6955f2 100644 --- a/typegate/src/typegraphs/typegate.json +++ b/typegate/src/typegraphs/typegate.json @@ -493,7 +493,9 @@ "type": "object", "title": "object_39", "runtime": 1, - "policies": [], + "policies": [ + 0 + ], "config": {}, "as_id": false, "properties": { @@ -543,6 +545,10 @@ "title": "OperationInfo", "runtime": 1, "policies": [], + "enum": [ + "\"query\"", + "\"mutation\"" + ], "config": {}, "as_id": false, "properties": { @@ -564,13 +570,25 @@ "\"mutation\"" ], "config": {}, - "as_id": false + "as_id": false, + "properties": { + "name": 5, + "type": 46, + "inputs": 47, + "output": 31, + "outputItem": 49 + }, + "required": [] }, { "type": "list", "title": "list_48", "runtime": 1, "policies": [], + "enum": [ + "\"query\"", + "\"mutation\"" + ], "config": {}, "as_id": false, "items": 48 @@ -598,6 +616,16 @@ "item": 31, "default_value": null }, + { + "type": "optional", + "title": "_49_TypeInfo?", + "runtime": 1, + "policies": [], + "config": {}, + "as_id": false, + "item": 31, + "default_value": null + }, { "type": "function", "title": "func_111", diff --git a/typegate/tests/runtimes/python_wasi/python_wasi.py b/typegate/tests/runtimes/python_wasi/python_wasi.py index 31f20533b..64b0a64a4 100644 --- a/typegate/tests/runtimes/python_wasi/python_wasi.py +++ b/typegate/tests/runtimes/python_wasi/python_wasi.py @@ -52,6 +52,7 @@ def python_wasi(g: Graph): t.struct({"name": t.string()}), t.string(), module="py/hello.py", + deps=["py/nested/dep.py"], name="sayHello", ).with_policy(public), identity=python.from_def( diff --git a/typegate/tests/runtimes/python_wasi/python_wasi.ts b/typegate/tests/runtimes/python_wasi/python_wasi.ts index e6b139746..05850d127 100644 --- a/typegate/tests/runtimes/python_wasi/python_wasi.ts +++ b/typegate/tests/runtimes/python_wasi/python_wasi.ts @@ -10,9 +10,23 @@ const tpe = t.struct({ "b": t.list(t.either([t.integer(), t.string()])), }); -typegraph("python_wasi", (g: any) => { +export const tg = await typegraph("python_wasi", async (g: any) => { const python = new PythonRuntime(); const pub = Policy.public(); + let identityModule; + try { + identityModule = await python.import( + t.struct({ input: tpe }), + tpe, + { + name: "identity", + module: "typegate/tests/runtimes/python_wasi/py/hello.py", + deps: ["typegate/tests/runtimes/python_wasi/py/nested/dep.py"], + }, + ); + } catch (e) { + console.error(e); + } g.expose({ identityLambda: python.fromLambda( @@ -30,10 +44,6 @@ typegraph("python_wasi", (g: any) => { `, }, ).withPolicy(pub), - identityMod: python.import( - t.struct({ input: tpe }), - tpe, - { name: "identity", module: "py/hello.py" }, - ).withPolicy(pub), + identityMod: identityModule!.withPolicy(pub), }); }); diff --git a/typegate/tests/runtimes/python_wasi/python_wasi_test.ts b/typegate/tests/runtimes/python_wasi/python_wasi_test.ts index f30768ea5..f51156dc1 100644 --- a/typegate/tests/runtimes/python_wasi/python_wasi_test.ts +++ b/typegate/tests/runtimes/python_wasi/python_wasi_test.ts @@ -1,306 +1,355 @@ // Copyright Metatype OÜ, licensed under the Elastic License 2.0. // SPDX-License-Identifier: Elastic-2.0 -import { assert, assertEquals } from "std/assert/mod.ts"; +// import { assert, assertEquals } from "std/assert/mod.ts"; import { gql, Meta } from "../../utils/mod.ts"; -import { PythonVirtualMachine } from "../../../src/runtimes/python_wasi/python_vm.ts"; -import { QueryEngine } from "../../../src/engine/query_engine.ts"; +// import { PythonVirtualMachine } from "../../../src/runtimes/python_wasi/python_vm.ts"; +// import { QueryEngine } from "../../../src/engine/query_engine.ts"; +import { BasicAuth, tgDeploy } from "@typegraph/sdk/tg_deploy.js"; +import { testDir } from "test-utils/dir.ts"; +import { tg } from "./python_wasi.ts"; -Meta.test("Python WASI VM performance", async (t) => { - const vm = new PythonVirtualMachine(); - await vm.setup("myVm"); +// Meta.test("Python WASI VM performance", async (t) => { +// const vm = new PythonVirtualMachine(); +// await vm.setup("myVm"); - await t.should("work with low latency for lambdas", async () => { - await vm.registerLambda("test", "lambda x: x['a']"); - const samples = [...Array(100).keys()].map((i) => - vm.applyLambda(i, "test", [{ a: "test" }]) - ); - const start = performance.now(); - const items = await Promise.all(samples); - const end = performance.now(); - const duration = end - start; +// await t.should("work with low latency for lambdas", async () => { +// await vm.registerLambda("test", "lambda x: x['a']"); +// const samples = [...Array(100).keys()].map((i) => +// vm.applyLambda(i, "test", [{ a: "test" }]) +// ); +// const start = performance.now(); +// const items = await Promise.all(samples); +// const end = performance.now(); +// const duration = end - start; - const randomItem = items[Math.floor(items.length * Math.random())]; - assertEquals(randomItem, "test"); // always resolved - assert( - duration < 5, - `virtual machine execution was too slow: ${duration}ms`, - ); - }); +// const randomItem = items[Math.floor(items.length * Math.random())]; +// assertEquals(randomItem, "test"); // always resolved +// assert( +// duration < 5, +// `virtual machine execution was too slow: ${duration}ms`, +// ); +// }); - await t.should("work with low latency for defs", async () => { - await vm.registerDef( - "test", - "def test(x):\n\treturn x['a']", - ); - const samples = [...Array(100).keys()].map((i) => - vm.applyDef( - i, - "test", - [{ a: "test" }], - ) - ); - const start = performance.now(); - const items = await Promise.all(samples); - const end = performance.now(); - const duration = end - start; +// await t.should("work with low latency for defs", async () => { +// await vm.registerDef( +// "test", +// "def test(x):\n\treturn x['a']", +// ); +// const samples = [...Array(100).keys()].map((i) => +// vm.applyDef( +// i, +// "test", +// [{ a: "test" }], +// ) +// ); +// const start = performance.now(); +// const items = await Promise.all(samples); +// const end = performance.now(); +// const duration = end - start; - const randomItem = items[Math.floor(items.length * Math.random())]; - assertEquals(randomItem, "test"); // always resolved - assert( - duration < 5, - `virtual machine execution was too slow: ${duration}ms`, - ); - }); +// const randomItem = items[Math.floor(items.length * Math.random())]; +// assertEquals(randomItem, "test"); // always resolved +// assert( +// duration < 5, +// `virtual machine execution was too slow: ${duration}ms`, +// ); +// }); - await vm.destroy(); -}); +// await vm.destroy(); +// }); -Meta.test("Python WASI runtime", async (t) => { - const e = await t.engine("runtimes/python_wasi/python_wasi.py"); +// Meta.test("Python WASI runtime", async (t) => { +// const e = await t.engine("runtimes/python_wasi/python_wasi.py"); - await t.should("work once (lambda)", async () => { - await gql` - query { - test(a: "test") - } - ` - .expectData({ - test: "test", - }) - .on(e); - }); +// await t.should("work once (lambda)", async () => { +// await gql` +// query { +// test(a: "test") +// } +// ` +// .expectData({ +// test: "test", +// }) +// .on(e); +// }); - await t.should("work once (def)", async () => { - await gql` - query { - testDef(a: "test") - } - ` - .expectData({ - testDef: "test", - }) - .on(e); - }); +// await t.should("work once (def)", async () => { +// await gql` +// query { +// testDef(a: "test") +// } +// ` +// .expectData({ +// testDef: "test", +// }) +// .on(e); +// }); - await t.should("work once (module)", async () => { - await gql` - query { - testMod(name: "Loyd") - } - ` - .expectData({ - testMod: "Hello Loyd", - }) - .on(e); - }); +// await t.should("work once (module)", async () => { +// await gql` +// query { +// testMod(name: "Loyd") +// } +// ` +// .expectData({ +// testMod: "Hello Loyd", +// }) +// .on(e); +// }); - await t.should("return same object", async () => { - await gql` - query { - identity( - input: { - a: 1234, - b: { c: ["one", "two", "three" ] } - } - ) { - a - b { c } - } - } - ` - .expectData({ - identity: { - a: 1234, - b: { c: ["one", "two", "three"] }, - }, - }) - .on(e); - }); +// await t.should("return same object", async () => { +// await gql` +// query { +// identity( +// input: { +// a: 1234, +// b: { c: ["one", "two", "three" ] } +// } +// ) { +// a +// b { c } +// } +// } +// ` +// .expectData({ +// identity: { +// a: 1234, +// b: { c: ["one", "two", "three"] }, +// }, +// }) +// .on(e); +// }); - await t.should("work fast enough", async () => { - const tests = [...Array(100).keys()].map((i) => - gql` - query ($a: String!) { - test(a: $a) - } - ` - .withVars({ - a: `test${i}`, - }) - .expectData({ - test: `test${i}`, - }) - .on(e) - ); +// await t.should("work fast enough", async () => { +// const tests = [...Array(100).keys()].map((i) => +// gql` +// query ($a: String!) { +// test(a: $a) +// } +// ` +// .withVars({ +// a: `test${i}`, +// }) +// .expectData({ +// test: `test${i}`, +// }) +// .on(e) +// ); - const start = performance.now(); - await Promise.all(tests); - const end = performance.now(); - const duration = end - start; +// const start = performance.now(); +// await Promise.all(tests); +// const end = performance.now(); +// const duration = end - start; - console.log(`duration: ${duration}ms`); - assert(duration < 800, `Python WASI runtime was too slow: ${duration}ms`); - }); -}); +// console.log(`duration: ${duration}ms`); +// assert(duration < 800, `Python WASI runtime was too slow: ${duration}ms`); +// }); +// }); -Meta.test("Deno: def, lambda, import", async (t) => { - const e = await t.engine("runtimes/python_wasi/python_wasi.ts"); - await t.should("work with def", async () => { - await gql` - query { - identityLambda( - input: { - a: "hello", - b: [1, 2, "three"] - }) { - a - b - } - } - ` - .expectData({ - identityLambda: { - a: "hello", - b: [1, 2, "three"], - }, - }) - .on(e); - }); +// Meta.test("Deno: def, lambda, import", async (t) => { +// const e = await t.engine("runtimes/python_wasi/python_wasi.ts"); +// await t.should("work with def", async () => { +// await gql` +// query { +// identityLambda( +// input: { +// a: "hello", +// b: [1, 2, "three"] +// }) { +// a +// b +// } +// } +// ` +// .expectData({ +// identityLambda: { +// a: "hello", +// b: [1, 2, "three"], +// }, +// }) +// .on(e); +// }); - await t.should("work with def", async () => { - await gql` - query { - identityDef( - input: { - a: "hello", - b: [1, 2, "three"] - }) { - a - b - } - } - ` - .expectData({ - identityDef: { - a: "hello", - b: [1, 2, "three"], - }, - }) - .on(e); - }); +// await t.should("work with def", async () => { +// await gql` +// query { +// identityDef( +// input: { +// a: "hello", +// b: [1, 2, "three"] +// }) { +// a +// b +// } +// } +// ` +// .expectData({ +// identityDef: { +// a: "hello", +// b: [1, 2, "three"], +// }, +// }) +// .on(e); +// }); - await t.should("work with module import", async () => { - await gql` - query { - identityMod(input: { - a: "hello", - b: [1, 2, "three"], - }) { - a - b - } - } - ` - .expectData({ - identityMod: { - a: "hello", - b: [1, 2, "three"], - }, - }) - .on(e); - }); -}); +// await t.should("work with module import", async () => { +// await gql` +// query { +// identityMod(input: { +// a: "hello", +// b: [1, 2, "three"], +// }) { +// a +// b +// } +// } +// ` +// .expectData({ +// identityMod: { +// a: "hello", +// b: [1, 2, "three"], +// }, +// }) +// .on(e); +// }); +// }); -Meta.test("Python WASI: infinite loop or similar", async (t) => { - const e = await t.engine("runtimes/python_wasi/python_wasi.py"); +// Meta.test("Python WASI: infinite loop or similar", async (t) => { +// const e = await t.engine("runtimes/python_wasi/python_wasi.py"); - await t.should("safely fail upon stackoverflow", async () => { - await gql` - query { - stackOverflow(enable: true) - } - ` - .expectErrorContains("maximum recursion depth exceeded") - .on(e); - }); +// await t.should("safely fail upon stackoverflow", async () => { +// await gql` +// query { +// stackOverflow(enable: true) +// } +// ` +// .expectErrorContains("maximum recursion depth exceeded") +// .on(e); +// }); - // let tic = 0; - // setTimeout(() => console.log("hearbeat", tic++), 100); +// // let tic = 0; +// // setTimeout(() => console.log("hearbeat", tic++), 100); - // FIXME: blocks main deno thread - // current approach on deno_bindgen apply/applyDef needs to run on - // separate threads - // #[deno] works for applys but still manages to block the current thread - // await t.should("safely fail upon infinite loop", async () => { - // await gql` - // query { - // infiniteLoop(enable: true) - // } - // ` - // .expectErrorContains("timeout exceeded") - // .on(e); - // }); -}, { sanitizeOps: false }); +// // FIXME: blocks main deno thread +// // current approach on deno_bindgen apply/applyDef needs to run on +// // separate threads +// // #[deno] works for applys but still manages to block the current thread +// // await t.should("safely fail upon infinite loop", async () => { +// // await gql` +// // query { +// // infiniteLoop(enable: true) +// // } +// // ` +// // .expectErrorContains("timeout exceeded") +// // .on(e); +// // }); +// }, { sanitizeOps: false }); -Meta.test("Python WASI: typegate reloading", async (metaTest) => { - const load = async () => { - return await metaTest.engine("runtimes/python_wasi/python_wasi.ts"); - }; +// Meta.test("Python WASI: typegate reloading", async (metaTest) => { +// const load = async () => { +// return await metaTest.engine("runtimes/python_wasi/python_wasi.ts"); +// }; - const runPythonOnPythonWasi = async (currentEngine: QueryEngine) => { - await gql` - query { - identityDef( - input: { - a: "hello", - b: [1, 2, "three"] - }) { - a - b +// const runPythonOnPythonWasi = async (currentEngine: QueryEngine) => { +// await gql` +// query { +// identityDef( +// input: { +// a: "hello", +// b: [1, 2, "three"] +// }) { +// a +// b +// }, +// identityLambda( +// input: { +// a: "hello", +// b: [1, 2, "three"] +// }) { +// a +// b +// }, +// identityMod(input: { +// a: "hello", +// b: [1, 2, "three"], +// }) { +// a +// b +// } +// } +// ` +// .expectData({ +// identityDef: { +// a: "hello", +// b: [1, 2, "three"], +// }, +// identityLambda: { +// a: "hello", +// b: [1, 2, "three"], +// }, +// identityMod: { +// a: "hello", +// b: [1, 2, "three"], +// }, +// }) +// .on(currentEngine); +// }; +// const engine = await load(); +// await metaTest.should("work before typegate is reloaded", async () => { +// await runPythonOnPythonWasi(engine); +// }); + +// // reload +// const reloadedEngine = await load(); + +// await metaTest.should("work after typegate is reloaded", async () => { +// await runPythonOnPythonWasi(reloadedEngine); +// }); +// }); + +const port = 7698; +const gate = `http://localhost:${port}`; +const cwdDir = testDir; +const auth = new BasicAuth("admin", "password"); + +Meta.test("Python WASI: upload artifacts with deps", async (metaTest) => { + await metaTest.should("upload artifacts along with deps", async () => { + const { serialized, typegate: _gateResponseAdd } = await tgDeploy(tg, { + baseUrl: gate, + auth, + artifactsConfig: { + prismaMigration: { + globalAction: { + create: true, + reset: false, + }, + migrationDir: "prisma-migrations", }, - identityLambda( - input: { - a: "hello", - b: [1, 2, "three"] - }) { - a - b + dir: cwdDir, }, - identityMod(input: { - a: "hello", - b: [1, 2, "three"], - }) { - a - b - } + }); + + const engine = await metaTest.engineFromDeployed(serialized); + + await gql` + query { + identityMod(input: { + a: "hello", + b: [1, 2, "three"], + }) { + a + b + } } ` .expectData({ - identityDef: { - a: "hello", - b: [1, 2, "three"], - }, - identityLambda: { - a: "hello", - b: [1, 2, "three"], - }, identityMod: { a: "hello", b: [1, 2, "three"], }, }) - .on(currentEngine); - }; - const engine = await load(); - await metaTest.should("work before typegate is reloaded", async () => { - await runPythonOnPythonWasi(engine); - }); - - // reload - const reloadedEngine = await load(); - - await metaTest.should("work after typegate is reloaded", async () => { - await runPythonOnPythonWasi(reloadedEngine); + .on(engine); + await engine.terminate(); }); -}); +}, { port }); diff --git a/typegate/tests/runtimes/wasmedge/wasmedge.py b/typegate/tests/runtimes/wasmedge/wasmedge.py index 9c2f832b1..884e1f8e4 100644 --- a/typegate/tests/runtimes/wasmedge/wasmedge.py +++ b/typegate/tests/runtimes/wasmedge/wasmedge.py @@ -1,9 +1,18 @@ +from typegraph.gen.exports.core import ( + ArtifactResolutionConfig, + MigrationAction, + MigrationConfig, +) +from typegraph.graph.shared_types import BasicAuth +from typegraph.graph.tg_deploy import TypegraphDeployParams, tg_deploy from typegraph.graph.typegraph import Graph from typegraph.policy import Policy from typegraph.runtimes.wasmedge import WasmEdgeRuntime from typegraph import t, typegraph +from typegraph import t, typegraph + @typegraph() def wasmedge(g: Graph): @@ -15,6 +24,28 @@ def wasmedge(g: Graph): t.struct({"a": t.float(), "b": t.float()}), t.integer(), wasm="artifacts/rust.wasm", + wasm="artifacts/rust.wasm", func="add", ).with_policy(pub), ) + + +PORT = 7698 +gate = f"http://localhost:{PORT}" +auth = BasicAuth("admin", "password") + +wasmedge_tg = wasmedge() +serialized, typegate = tg_deploy( + wasmedge_tg, + TypegraphDeployParams( + base_url=gate, + artifacts_config=ArtifactResolutionConfig( + prisma_migration=MigrationConfig( + migration_dir="prisma-migrations", + global_action=MigrationAction(reset=False, create=True), + ) + ), + ), +) + +print(serialized) diff --git a/typegate/tests/utils/test.ts b/typegate/tests/utils/test.ts index 0fd9ceeb0..42337b352 100644 --- a/typegate/tests/utils/test.ts +++ b/typegate/tests/utils/test.ts @@ -28,6 +28,11 @@ export interface ParseOptions { pretty?: boolean; } +export enum SDKLangugage { + Python = "python3", + TypeScript = "deno", +} + interface ServeResult { port: number; cleanup: () => Promise; @@ -179,6 +184,24 @@ export class MetaTest { return engine; } + async serializeTypegraphFromShell( + path: string, + lang: SDKLangugage, + ): Promise { + // run self deployed typegraph + const { stderr, stdout } = await this.shell([lang.toString(), path]); + + if (stderr.length > 0) { + throw new Error(`${stderr}`); + } + + if (stdout.length === 0) { + throw new Error("No typegraph"); + } + + return stdout; + } + async unregister(engine: QueryEngine) { await Promise.all( this.typegate.register diff --git a/typegraph/core/src/conversion/runtimes.rs b/typegraph/core/src/conversion/runtimes.rs index 9b09294c6..97a66a6bc 100644 --- a/typegraph/core/src/conversion/runtimes.rs +++ b/typegraph/core/src/conversion/runtimes.rs @@ -3,9 +3,11 @@ use std::collections::HashMap; +use std::path::PathBuf; use std::rc::Rc; use crate::errors::Result; +use crate::global_store::Store; use crate::runtimes::prisma::get_prisma_context; use crate::runtimes::{ DenoMaterializer, Materializer as RawMaterializer, PythonMaterializer, RandomMaterializer, @@ -229,11 +231,20 @@ impl MaterializerConverter for PythonMaterializer { ("def".to_string(), data) } Module(module) => { - let mut data = IndexMap::new(); - data.insert( - "code".to_string(), - serde_json::Value::String(format!("file:{}", module.file)), - ); + c.add_ref_artifacts(module.artifact_hash.clone(), module.artifact.clone().into())?; + + let deps = module.deps.clone(); + for dep in deps { + Store::register_dep(module.artifact_hash.clone(), dep); + } + + let data = serde_json::from_value(json!({ + "artifact": module.artifact, + "artifact_hash": module.artifact_hash, + "tg_name": None::, + })) + .map_err(|e| e.to_string())?; + ("pymodule".to_string(), data) } Import(import) => { diff --git a/typegraph/core/src/global_store.rs b/typegraph/core/src/global_store.rs index 385b6fab9..bef673b1a 100644 --- a/typegraph/core/src/global_store.rs +++ b/typegraph/core/src/global_store.rs @@ -12,7 +12,9 @@ use crate::wit::utils::Auth as WitAuth; #[allow(unused)] use crate::wit::core::ArtifactResolutionConfig; -use crate::wit::runtimes::{Effect, MaterializerDenoPredefined, MaterializerId}; +use crate::wit::runtimes::{ + Effect, MaterializerDenoPredefined, MaterializerId, ModuleDependencyMeta, +}; use graphql_parser::parse_query; use indexmap::IndexMap; use std::path::PathBuf; @@ -52,6 +54,9 @@ pub struct Store { predefined_deno_functions: HashMap, deno_modules: HashMap, + // module dependencies + artifact_deps: HashMap>, + public_policy_id: PolicyId, prisma_migration_runtime: RuntimeId, @@ -237,6 +242,23 @@ impl Store { with_store_mut(|store| store.random_seed = value) } + pub fn get_deps(artifact_hash: String) -> Vec { + with_store(|store| match store.artifact_deps.get(&artifact_hash) { + Some(deps) => deps.clone(), + None => vec![], + }) + } + + pub fn register_dep(artifact_hash: String, dep: ModuleDependencyMeta) { + with_store_mut(|store| { + store + .artifact_deps + .entry(artifact_hash) + .or_default() + .push(dep) + }) + } + pub fn pick_branch_by_path(supertype_id: TypeId, path: &[String]) -> Result<(Type, TypeId)> { let (_, supertype) = supertype_id.resolve_ref()?; let supertype = &supertype; diff --git a/typegraph/core/src/runtimes/mod.rs b/typegraph/core/src/runtimes/mod.rs index 1c27eeb47..442929c44 100644 --- a/typegraph/core/src/runtimes/mod.rs +++ b/typegraph/core/src/runtimes/mod.rs @@ -340,6 +340,10 @@ impl crate::wit::runtimes::Guest for crate::Lib { Ok(Store::register_materializer(mat)) } + fn get_deps(artifact_hash: String) -> Result, wit::Error> { + Ok(Store::get_deps(artifact_hash)) + } + fn register_random_runtime( data: wit::RandomRuntimeData, ) -> Result { diff --git a/typegraph/core/src/typegraph.rs b/typegraph/core/src/typegraph.rs index a3f061cf9..646c05bfc 100644 --- a/typegraph/core/src/typegraph.rs +++ b/typegraph/core/src/typegraph.rs @@ -24,6 +24,7 @@ use std::collections::hash_map::Entry; use std::collections::HashMap; use std::hash::Hasher as _; +use std::path::PathBuf; use std::rc::Rc; use crate::wit::core::{ @@ -243,6 +244,11 @@ pub fn finalize( Err(e) => return Err(e), }; + let result = match serde_json::to_string_pretty(&tg).map_err(|e| e.to_string().into()) { + Ok(res) => res, + Err(e) => return Err(e), + }; + Ok((result, artifacts)) } diff --git a/typegraph/core/src/utils/postprocess/python_rt.rs b/typegraph/core/src/utils/postprocess/python_rt.rs index 5f2c2242a..0c95bd9bd 100644 --- a/typegraph/core/src/utils/postprocess/python_rt.rs +++ b/typegraph/core/src/utils/postprocess/python_rt.rs @@ -7,24 +7,25 @@ use common::typegraph::{ utils::{map_from_object, object_from_map}, Typegraph, }; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; -use crate::utils::postprocess::{compress_and_encode, PostProcessor}; +use crate::utils::postprocess::PostProcessor; pub struct PythonProcessor; impl PostProcessor for PythonProcessor { fn postprocess(self, tg: &mut Typegraph) -> Result<(), crate::errors::TgError> { + let tg_name = tg.name().unwrap(); for mat in tg.materializers.iter_mut() { if mat.name.as_str() == "pymodule" { let mut mat_data: ModuleMatData = object_from_map(std::mem::take(&mut mat.data)).map_err(|e| e.to_string())?; - let Some(path) = mat_data.code.strip_prefix("file:").to_owned() else { - continue; - }; + let path = &mat_data.artifact; let main_path = fs_host::make_absolute(&PathBuf::from(path))?; - mat_data.code = compress_and_encode(&main_path)?; + let artifact_name = Path::new(path).file_name().unwrap().to_str().unwrap(); + mat_data.artifact = artifact_name.into(); + mat_data.tg_name = Some(tg_name.clone()); mat.data = map_from_object(mat_data).map_err(|e| e.to_string())?; tg.deps.push(main_path); diff --git a/typegraph/core/wit/typegraph.wit b/typegraph/core/wit/typegraph.wit index 24ab9b595..d12ec49d7 100644 --- a/typegraph/core/wit/typegraph.wit +++ b/typegraph/core/wit/typegraph.wit @@ -332,9 +332,17 @@ interface runtimes { fn: string, } + record module-dependency-meta { + path: string, + dep-hash: string, + relative-path-prefix: list, + } + record materializer-python-module { runtime: runtime-id, - file: string, + artifact: string, + artifact-hash: string, + deps: list, } record materializer-python-import { @@ -348,6 +356,7 @@ interface runtimes { from-python-def: func(base: base-materializer, data: materializer-python-def) -> result from-python-module: func(base: base-materializer, data: materializer-python-module) -> result from-python-import: func(base: base-materializer, data: materializer-python-import) -> result + get-deps: func(artifact-hash: string) -> result, error> // random record random-runtime-data { diff --git a/typegraph/node/sdk/src/runtimes/python.ts b/typegraph/node/sdk/src/runtimes/python.ts index 51f840c40..c781408f1 100644 --- a/typegraph/node/sdk/src/runtimes/python.ts +++ b/typegraph/node/sdk/src/runtimes/python.ts @@ -3,9 +3,17 @@ import * as t from "../types.js"; import { runtimes } from "../wit.js"; -import { Effect } from "../gen/interfaces/metatype-typegraph-runtimes.js"; +import { + Effect, + ModuleDependencyMeta, +} from "../gen/interfaces/metatype-typegraph-runtimes.js"; import { Materializer, Runtime } from "./mod.js"; import { fx } from "../index.js"; +import { + getFileHash, + getParentDirectories, + getRelativePath, +} from "../utils/file_utils.js"; interface LambdaMat extends Materializer { fn: string; @@ -21,10 +29,16 @@ interface DefMat extends Materializer { interface PythonImport { name: string; module: string; + deps: Array; secrets?: Array; effect?: Effect; } +// interface DependencyMeta { +// path: string; +// hash: string; +// } + interface ImportMat extends Materializer { module: string; name: string; @@ -84,22 +98,42 @@ export class PythonRuntime extends Runtime { } as DefMat); } - import< + async import< I extends t.Typedef = t.Typedef, O extends t.Typedef = t.Typedef, >( inp: I, out: O, - { name, module, effect = fx.read(), secrets = [] }: PythonImport, - ): t.Func { + { name, module, deps = [], effect = fx.read(), secrets = [] }: PythonImport, + ): Promise> { const base = { runtime: this._id, effect, }; + const artifactHash = await getFileHash(module); + + // generate dep meta + const depMetas: ModuleDependencyMeta[] = []; + for (const dep of deps) { + const depHash = await getFileHash(dep); + const depParentDirs = getParentDirectories(dep); + const depMeta: ModuleDependencyMeta = { + path: dep, + depHash: depHash, + relativePathPrefix: getRelativePath( + getParentDirectories(module), + depParentDirs, + ), + }; + depMetas.push(depMeta); + } + const matId = runtimes.fromPythonModule(base, { - file: module, + artifact: module, runtime: this._id, + artifactHash: artifactHash, + deps: depMetas, }); const pyModMatId = runtimes.fromPythonImport(base, { diff --git a/typegraph/node/sdk/src/tg_artifact_upload.ts b/typegraph/node/sdk/src/tg_artifact_upload.ts new file mode 100644 index 000000000..4ef871882 --- /dev/null +++ b/typegraph/node/sdk/src/tg_artifact_upload.ts @@ -0,0 +1,151 @@ +// Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. +// SPDX-License-Identifier: MPL-2.0 + +import { BasicAuth } from "./tg_deploy.js"; +import * as fs from "node:fs"; +import * as path from "node:path"; +import { runtimes } from "./wit.js"; +import { ModuleDependencyMeta } from "./gen/interfaces/metatype-typegraph-runtimes.js"; + +export interface UploadArtifactMeta { + name: string; + artifact_hash: string; + artifact_size_in_bytes: number; + path_suffix: string[]; +} + +export class ArtifactUploader { + private getUploadUrl: URL; + + constructor( + private baseUrl: string, + private refArtifacts: [string, string][], + private tgName: string, + private auth: BasicAuth | undefined, + private headers: Headers, + ) { + const suffix = `${tgName}/get-upload-url`; + this.getUploadUrl = new URL(suffix, baseUrl); + } + + private async fetchUploadUrl( + artifactPath: string, + artifactHash: string, + artifactContent: Uint8Array, + pathSuffix: string[], + ): Promise { + const artifactMeta: UploadArtifactMeta = { + name: path.basename(artifactPath), + artifact_hash: artifactHash, + artifact_size_in_bytes: artifactContent.length, + path_suffix: pathSuffix, + }; + + const artifactJson = JSON.stringify(artifactMeta); + const uploadUrlResponse = await fetch(this.getUploadUrl, { + method: "PUT", + headers: this.headers, + body: artifactJson, + }); + + // console.log("******************A", uploadUrlResponse); + const decodedResponse = await uploadUrlResponse.json(); + + return decodedResponse.uploadUrl as string; + } + + private async upload( + url: string, + content: Uint8Array, + artifactPath: string, + ): Promise { + const uploadHeaders = new Headers({ + "Content-Type": "application/octet-stream", + }); + + if (this.auth) { + uploadHeaders.append("Authorization", this.auth.asHeaderValue()); + } + + const artifactUploadResponse = await fetch(url, { + method: "PUT", + headers: uploadHeaders, + body: content, + }); + + const _ = await artifactUploadResponse.json(); + if (!artifactUploadResponse.ok) { + throw new Error( + `Failed to upload artifact ${artifactPath} to typegate: ${artifactUploadResponse.status} ${artifactUploadResponse.statusText}`, + ); + } + } + + async uploadArtifacts(): Promise { + for (let [artifactHash, artifactPath] of this.refArtifacts) { + await this.uploadArtifact(artifactHash, artifactPath); + } + } + + private async uploadArtifact( + artifactHash: string, + artifactPath: string, + ): Promise { + try { + await fs.promises.access(artifactPath); + } catch (err) { + throw new Error(`Failed to access artifact ${artifactPath}: ${err}`); + } + let artifactContent: Buffer; + try { + artifactContent = await fs.promises.readFile(artifactPath); + } catch (err) { + throw new Error(`Failed to read artifact ${artifactPath}: ${err}`); + } + const byteArray = new Uint8Array(artifactContent); + + const artifactUploadUrl = await this.fetchUploadUrl( + artifactPath, + artifactHash, + byteArray, + [artifactHash], + ); + + await this.upload(artifactUploadUrl, byteArray, artifactPath); + + await this.uploadArtifactDependencies(artifactHash); + } + + private async uploadArtifactDependencies( + artifactHash: string, + ): Promise { + const depMetas = runtimes.getDeps(artifactHash); + + for (let dep of depMetas) { + const { depHash, path: depPath, relativePathPrefix }: + ModuleDependencyMeta = dep; + + try { + await fs.promises.access(depPath); + } catch (err) { + throw new Error(`Failed to access artifact ${path}: ${err}`); + } + let depContent: Buffer; + try { + depContent = await fs.promises.readFile(depPath); + } catch (err) { + throw new Error(`Failed to read artifact ${path}: ${err}`); + } + const byteArray = new Uint8Array(depContent); + + const depUploadUrl = await this.fetchUploadUrl( + depPath, + depHash, + byteArray, + [artifactHash, ...relativePathPrefix], + ); + + await this.upload(depUploadUrl, byteArray, depPath); + } + } +} diff --git a/typegraph/node/sdk/src/tg_deploy.ts b/typegraph/node/sdk/src/tg_deploy.ts index e1389d2ad..681a08fde 100644 --- a/typegraph/node/sdk/src/tg_deploy.ts +++ b/typegraph/node/sdk/src/tg_deploy.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: MPL-2.0 import { ArtifactResolutionConfig } from "./gen/interfaces/metatype-typegraph-core.js"; +import { ArtifactUploader } from "./tg_artifact_upload.js"; import { TypegraphOutput } from "./typegraph.js"; import { wit_utils } from "./wit.js"; import * as fsp from "node:fs/promises"; diff --git a/typegraph/node/sdk/src/utils/file_utils.ts b/typegraph/node/sdk/src/utils/file_utils.ts index 45e2b00ec..13d44f19e 100644 --- a/typegraph/node/sdk/src/utils/file_utils.ts +++ b/typegraph/node/sdk/src/utils/file_utils.ts @@ -4,11 +4,11 @@ import * as fs from "node:fs"; import * as crypto from "node:crypto"; import { wit_utils } from "../wit.js"; -import { Artifact } from "../gen/interfaces/metatype-typegraph-core.js"; +import * as path from "node:path"; -export async function getArtifactMeta(path: string): Promise { +export async function getFileHash(filePath: string): Promise { const cwd = wit_utils.getCwd(); - const filePath = `${cwd}/${path}`; + filePath = `${cwd}/${filePath}`; const hash = crypto.createHash("sha256"); let file; @@ -31,11 +31,7 @@ export async function getArtifactMeta(path: string): Promise { hash.update(buffer.subarray(0, numBytesRead)); } while (numBytesRead > 0); - return { - path, - hash: hash.digest("hex"), - size: bytesRead, - }; + return hash.digest("hex"); } catch (err) { throw new Error(`Failed to calculate hash for ${filePath}: \n${err}`); } finally { @@ -44,3 +40,31 @@ export async function getArtifactMeta(path: string): Promise { } } } + +export function getParentDirectories(filePath: string) { + const directories = []; + let currentDir = path.dirname(filePath); + while (currentDir !== path.dirname(currentDir)) { + directories.push(path.basename(currentDir)); + currentDir = path.dirname(currentDir); + } + return directories.reverse(); +} + +export function getRelativePath( + moduleParentDirs: string[], + depParentDirs: string[], +): string[] { + let common = 0; + let maxLength = Math.min(moduleParentDirs.length, depParentDirs.length); + + for (let i = 0; i < maxLength; i++) { + if (moduleParentDirs[i] === depParentDirs[i]) { + common += 1; + } else { + break; + } + } + + return depParentDirs.slice(common); +} diff --git a/typegraph/python/typegraph/graph/shared_types.py b/typegraph/python/typegraph/graph/shared_types.py index 359984aef..ea014b673 100644 --- a/typegraph/python/typegraph/graph/shared_types.py +++ b/typegraph/python/typegraph/graph/shared_types.py @@ -4,6 +4,7 @@ from base64 import b64encode from dataclasses import dataclass from typing import Callable, List, Optional, Tuple + from typegraph.wit import ArtifactResolutionConfig @@ -29,3 +30,4 @@ def as_header_value(self): "utf-8" ) return f"Basic {payload}" + return f"Basic {payload}" diff --git a/typegraph/python/typegraph/graph/tg_artifact_upload.py b/typegraph/python/typegraph/graph/tg_artifact_upload.py new file mode 100644 index 000000000..99e811e97 --- /dev/null +++ b/typegraph/python/typegraph/graph/tg_artifact_upload.py @@ -0,0 +1,149 @@ +# Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. +# SPDX-License-Identifier: MPL-2.0 + +import json +import os +from dataclasses import dataclass +from typing import Dict, List, Tuple +from urllib import request + +from typegraph.gen.types import Err, Result +from typegraph.graph.shared_types import BasicAuth +from typegraph.graph.tg_deploy import handle_response +from typegraph.wit import runtimes, store + + +@dataclass +class UploadArtifactMeta: + name: str + artifact_hash: str + artifact_size_in_bytes: int + path_suffix: List[str] + + +class ArtifactUploader: + base_url: str + ref_artifacts: List[Tuple[str, str]] + get_upload_url: str + tg_name: str + auth: BasicAuth + headers: Dict[str, str] + + def __init__( + self, + base_url: str, + artifacts: List[Tuple[str, str]], + tg_name: str, + auth: BasicAuth, + headers: Dict[str, str], + ) -> None: + self.base_url = base_url + self.ref_artifacts = artifacts + self.tg_name = tg_name + sep = "/" if not base_url.endswith("/") else "" + self.get_upload_url = base_url + sep + tg_name + "/get-upload-url" + self.auth = auth + self.headers = headers + + # TODO: fetch all the upload urls in one request + def __fetch_upload_url( + self, + artifact_path: str, + artifact_hash: str, + artifact_content: bytes, + path_suffix: List[str], + ) -> str: + artifact = UploadArtifactMeta( + name=os.path.basename(artifact_path), + artifact_hash=artifact_hash, + artifact_size_in_bytes=len(artifact_content), + path_suffix=path_suffix, + ) + + artifact_json = json.dumps(artifact.__dict__).encode() + req = request.Request( + url=self.get_upload_url, + method="PUT", + headers=self.headers, + data=artifact_json, + ) + + response = handle_response(request.urlopen(req).read().decode()) + return response["uploadUrl"] + + def __upload( + self, + url: str, + content: bytes, + artifact_path: str, + ) -> Result[str, Err]: + upload_headers = {"Content-Type": "application/octet-stream"} + if self.auth is not None: + upload_headers["Authorization"] = self.auth.as_header_value() + upload_req = request.Request( + url=url, + method="PUT", + data=content, + headers=upload_headers, + ) + response = request.urlopen(upload_req) + if response.status != 200: + raise Exception( + f"Failed to upload artifact {artifact_path} to typegate: {response.read()}" + ) + + return response + + def upload_artifacts( + self, + ) -> Result[None, Err]: + for artifact_hash, artifact_path in self.ref_artifacts: + self.__upload_artifact(artifact_hash, artifact_path) + + def __upload_artifact(self, artifact_hash: str, artifact_path: str): + with open(artifact_path, "rb") as artifact: + artifact_content = artifact.read() + artifact_upload_url = self.__fetch_upload_url( + artifact_path, artifact_hash, artifact_content, [artifact_hash] + ) + + _upload_result = self.__upload( + artifact_upload_url, artifact_content, artifact_path + ) + + self.__upload_artifact_dependencies(artifact_hash) + + def __upload_artifact_dependencies(self, artifact_hash: str) -> Result[None, Err]: + dep_metas = runtimes.get_deps(store, artifact_hash) + if isinstance(dep_metas, Err): + raise Exception(dep_metas.value) + + dep_metas = dep_metas.value + + for dep_meta in dep_metas: + dep_hash = dep_meta.dep_hash + dep_path = dep_meta.path + relative_prefix = dep_meta.relative_path_prefix + + with open(dep_path, "rb") as dep: + dep_content = dep.read() + dep_upload_url = self.__fetch_upload_url( + dep_path, dep_hash, dep_content, [artifact_hash, *relative_prefix] + ) + + _upload_result = self.__upload(dep_upload_url, dep_content, dep_path) + + +""" + + what would uploading artifact with deps look like: + - get an upload url for the artifact, and use the artifact_hash as a suffix to store the artifact and the deps. The suffix is to be appended to the path where the artifact and the deps are gonna be stored + - get all the deps of the artifact + - upload them in parallel + + + - serialize(done, not tested) => upload(done, not tested) => resolve(done, not tested) + + - testing left + +""" diff --git a/typegraph/python/typegraph/graph/tg_deploy.py b/typegraph/python/typegraph/graph/tg_deploy.py index 38ceaf3be..a6ef4b34a 100644 --- a/typegraph/python/typegraph/graph/tg_deploy.py +++ b/typegraph/python/typegraph/graph/tg_deploy.py @@ -2,7 +2,6 @@ # SPDX-License-Identifier: MPL-2.0 import json -import os from dataclasses import dataclass from typing import Dict, Optional, Union from urllib import request @@ -10,6 +9,7 @@ from typegraph.gen.exports.utils import QueryDeployParams from typegraph.gen.types import Err from typegraph.graph.shared_types import BasicAuth +from typegraph.graph.tg_artifact_upload import ArtifactUploader from typegraph.graph.typegraph import TypegraphOutput from typegraph.wit import ArtifactResolutionConfig, store, wit_utils @@ -58,39 +58,14 @@ def tg_deploy(tg: TypegraphOutput, params: TypegraphDeployParams) -> DeployResul ref_artifacts = serialized.ref_artifacts # upload the referred artifacts - # TODO: fetch all the upload urls in one request - get_upload_url = params.base_url + sep + tg.name + "/get-upload-url" - for artifact_hash, artifact_path in ref_artifacts: - with open(artifact_path, "rb") as artifact: - artifact_content = artifact.read() - artifact = UploadArtifactMeta( - name=os.path.basename(artifact_path), - artifact_hash=artifact_hash, - artifact_size_in_bytes=len(artifact_content), - ) - - artifact_json = json.dumps(artifact.__dict__).encode() - req = request.Request( - url=get_upload_url, method="PUT", headers=headers, data=artifact_json - ) - - response = handle_response(request.urlopen(req).read().decode()) - artifact_upload_url = response["uploadUrl"] - - upload_headers = {"Content-Type": "application/octet-stream"} - if params.auth is not None: - upload_headers["Authorization"] = params.auth.as_header_value() - upload_req = request.Request( - url=artifact_upload_url, - method="PUT", - data=artifact_content, - headers=upload_headers, - ) - response = request.urlopen(upload_req) - if response.status != 200: - raise Exception( - f"Failed to upload artifact {artifact_path} to typegate: {response.read()}" - ) + artifact_uploader = ArtifactUploader( + params.base_url, + ref_artifacts, + tg.name, + params.auth, + headers, + ) + artifact_uploader.upload_artifacts() # deploy the typegraph res = wit_utils.gql_deploy_query( diff --git a/typegraph/python/typegraph/runtimes/python.py b/typegraph/python/typegraph/runtimes/python.py index 81b459849..011e8005d 100644 --- a/typegraph/python/typegraph/runtimes/python.py +++ b/typegraph/python/typegraph/runtimes/python.py @@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, List, Optional from astunparse import unparse - from typegraph.gen.exports.runtimes import ( BaseMaterializer, Effect, @@ -15,9 +14,11 @@ MaterializerPythonImport, MaterializerPythonLambda, MaterializerPythonModule, + ModuleDependencyMeta, ) from typegraph.gen.types import Err from typegraph.runtimes.base import Materializer, Runtime +from typegraph.utils import get_file_hash, get_parent_directories, get_relative_path from typegraph.wit import runtimes, store if TYPE_CHECKING: @@ -93,15 +94,39 @@ def import_( *, module: str, name: str, + deps: List[str] = [], effect: Optional[Effect] = None, secrets: Optional[List[str]] = None, ): effect = effect or EffectRead() secrets = secrets or [] + artifact_hash = get_file_hash(module) + + # generate dep_metas + dep_metas = [] + for dep in deps: + dep_hash = get_file_hash(dep) + dep_parent_dirs = get_parent_directories(dep) + dep_meta = ModuleDependencyMeta( + path=dep, + dep_hash=dep_hash, + relative_path_prefix=get_relative_path( + get_parent_directories(module), dep_parent_dirs + ), + ) + dep_metas.append(dep_meta) + base = BaseMaterializer(runtime=self.id.value, effect=effect) mat_id = runtimes.from_python_module( - store, base, MaterializerPythonModule(file=module, runtime=self.id.value) + store, + base, + MaterializerPythonModule( + artifact=module, + artifact_hash=artifact_hash, + deps=dep_metas, + runtime=self.id.value, + ), ) if isinstance(mat_id, Err): @@ -172,3 +197,4 @@ def visit_Lambda(self, node): def visit_FunctionDef(self, node): self.defs.append((node.name, unparse(node).strip())) + self.defs.append((node.name, unparse(node).strip())) diff --git a/typegraph/python/typegraph/utils.py b/typegraph/python/typegraph/utils.py index bbd8d3154..e0f9f43bf 100644 --- a/typegraph/python/typegraph/utils.py +++ b/typegraph/python/typegraph/utils.py @@ -95,3 +95,35 @@ def get_file_hash(file_path: str) -> str: sha256_hasher.update(chunk) return sha256_hasher.hexdigest() + + +def get_parent_directories( + path: str, +) -> List[str]: + parents = [] + + while True: + path, folder = os.path.split(path) + if folder: + parents.append(folder) + else: + if path: + parents.append(path) + break + + return parents[::-1] + + +def get_relative_path( + module_path: List[str], + dep_path: List[str], +) -> List[str]: + common = 0 + + max_len = min(len(module_path), len(dep_path)) + for i in range(max_len): + if module_path[i] != dep_path[i]: + break + common += 1 + + return dep_path[common:] diff --git a/website/static/specs/0.0.3.json b/website/static/specs/0.0.3.json index 2fc86e0f9..2cba2cc1f 100644 --- a/website/static/specs/0.0.3.json +++ b/website/static/specs/0.0.3.json @@ -2,14 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Typegraph", "type": "object", - "required": [ - "$id", - "materializers", - "meta", - "policies", - "runtimes", - "types" - ], + "required": ["$id", "materializers", "meta", "policies", "runtimes", "types"], "properties": { "$id": { "type": "string" @@ -47,20 +40,11 @@ "oneOf": [ { "type": "object", - "required": [ - "as_id", - "item", - "policies", - "runtime", - "title", - "type" - ], + "required": ["as_id", "item", "policies", "runtime", "title", "type"], "properties": { "type": { "type": "string", - "enum": [ - "optional" - ] + "enum": ["optional"] }, "title": { "type": "string" @@ -78,10 +62,7 @@ }, "description": { "default": null, - "type": [ - "string", - "null" - ] + "type": ["string", "null"] }, "injection": { "default": null, @@ -96,10 +77,7 @@ }, "enum": { "default": null, - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "items": { "type": "string" } @@ -122,19 +100,11 @@ }, { "type": "object", - "required": [ - "as_id", - "policies", - "runtime", - "title", - "type" - ], + "required": ["as_id", "policies", "runtime", "title", "type"], "properties": { "type": { "type": "string", - "enum": [ - "boolean" - ] + "enum": ["boolean"] }, "title": { "type": "string" @@ -152,10 +122,7 @@ }, "description": { "default": null, - "type": [ - "string", - "null" - ] + "type": ["string", "null"] }, "injection": { "default": null, @@ -170,10 +137,7 @@ }, "enum": { "default": null, - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "items": { "type": "string" } @@ -190,19 +154,11 @@ }, { "type": "object", - "required": [ - "as_id", - "policies", - "runtime", - "title", - "type" - ], + "required": ["as_id", "policies", "runtime", "title", "type"], "properties": { "type": { "type": "string", - "enum": [ - "float" - ] + "enum": ["float"] }, "title": { "type": "string" @@ -220,10 +176,7 @@ }, "description": { "default": null, - "type": [ - "string", - "null" - ] + "type": ["string", "null"] }, "injection": { "default": null, @@ -238,10 +191,7 @@ }, "enum": { "default": null, - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "items": { "type": "string" } @@ -255,57 +205,34 @@ "type": "boolean" }, "minimum": { - "type": [ - "number", - "null" - ], + "type": ["number", "null"], "format": "double" }, "maximum": { - "type": [ - "number", - "null" - ], + "type": ["number", "null"], "format": "double" }, "exclusiveMinimum": { - "type": [ - "number", - "null" - ], + "type": ["number", "null"], "format": "double" }, "exclusiveMaximum": { - "type": [ - "number", - "null" - ], + "type": ["number", "null"], "format": "double" }, "multipleOf": { - "type": [ - "number", - "null" - ], + "type": ["number", "null"], "format": "double" } } }, { "type": "object", - "required": [ - "as_id", - "policies", - "runtime", - "title", - "type" - ], + "required": ["as_id", "policies", "runtime", "title", "type"], "properties": { "type": { "type": "string", - "enum": [ - "integer" - ] + "enum": ["integer"] }, "title": { "type": "string" @@ -323,10 +250,7 @@ }, "description": { "default": null, - "type": [ - "string", - "null" - ] + "type": ["string", "null"] }, "injection": { "default": null, @@ -341,10 +265,7 @@ }, "enum": { "default": null, - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "items": { "type": "string" } @@ -358,57 +279,34 @@ "type": "boolean" }, "minimum": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "int32" }, "maximum": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "int32" }, "exclusiveMinimum": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "int32" }, "exclusiveMaximum": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "int32" }, "multipleOf": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "int32" } } }, { "type": "object", - "required": [ - "as_id", - "policies", - "runtime", - "title", - "type" - ], + "required": ["as_id", "policies", "runtime", "title", "type"], "properties": { "type": { "type": "string", - "enum": [ - "string" - ] + "enum": ["string"] }, "title": { "type": "string" @@ -426,10 +324,7 @@ }, "description": { "default": null, - "type": [ - "string", - "null" - ] + "type": ["string", "null"] }, "injection": { "default": null, @@ -444,10 +339,7 @@ }, "enum": { "default": null, - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "items": { "type": "string" } @@ -461,26 +353,17 @@ "type": "boolean" }, "minLength": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "uint32", "minimum": 0.0 }, "maxLength": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "uint32", "minimum": 0.0 }, "pattern": { - "type": [ - "string", - "null" - ] + "type": ["string", "null"] }, "format": { "anyOf": [ @@ -496,19 +379,11 @@ }, { "type": "object", - "required": [ - "as_id", - "policies", - "runtime", - "title", - "type" - ], + "required": ["as_id", "policies", "runtime", "title", "type"], "properties": { "type": { "type": "string", - "enum": [ - "file" - ] + "enum": ["file"] }, "title": { "type": "string" @@ -526,10 +401,7 @@ }, "description": { "default": null, - "type": [ - "string", - "null" - ] + "type": ["string", "null"] }, "injection": { "default": null, @@ -544,10 +416,7 @@ }, "enum": { "default": null, - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "items": { "type": "string" } @@ -561,26 +430,17 @@ "type": "boolean" }, "minSize": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "uint32", "minimum": 0.0 }, "maxSize": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "uint32", "minimum": 0.0 }, "mimeTypes": { - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "items": { "type": "string" } @@ -600,9 +460,7 @@ "properties": { "type": { "type": "string", - "enum": [ - "object" - ] + "enum": ["object"] }, "title": { "type": "string" @@ -620,10 +478,7 @@ }, "description": { "default": null, - "type": [ - "string", - "null" - ] + "type": ["string", "null"] }, "injection": { "default": null, @@ -638,10 +493,7 @@ }, "enum": { "default": null, - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "items": { "type": "string" } @@ -684,9 +536,7 @@ "properties": { "type": { "type": "string", - "enum": [ - "list" - ] + "enum": ["list"] }, "title": { "type": "string" @@ -704,10 +554,7 @@ }, "description": { "default": null, - "type": [ - "string", - "null" - ] + "type": ["string", "null"] }, "injection": { "default": null, @@ -722,10 +569,7 @@ }, "enum": { "default": null, - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "items": { "type": "string" } @@ -744,26 +588,17 @@ "minimum": 0.0 }, "maxItems": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "uint32", "minimum": 0.0 }, "minItems": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "uint32", "minimum": 0.0 }, "uniqueItems": { - "type": [ - "boolean", - "null" - ] + "type": ["boolean", "null"] } } }, @@ -783,9 +618,7 @@ "properties": { "type": { "type": "string", - "enum": [ - "function" - ] + "enum": ["function"] }, "title": { "type": "string" @@ -803,10 +636,7 @@ }, "description": { "default": null, - "type": [ - "string", - "null" - ] + "type": ["string", "null"] }, "injection": { "default": null, @@ -821,10 +651,7 @@ }, "enum": { "default": null, - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "items": { "type": "string" } @@ -863,10 +690,7 @@ "minimum": 0.0 }, "rate_weight": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "uint32", "minimum": 0.0 }, @@ -888,9 +712,7 @@ "properties": { "type": { "type": "string", - "enum": [ - "union" - ] + "enum": ["union"] }, "title": { "type": "string" @@ -908,10 +730,7 @@ }, "description": { "default": null, - "type": [ - "string", - "null" - ] + "type": ["string", "null"] }, "injection": { "default": null, @@ -926,10 +745,7 @@ }, "enum": { "default": null, - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "items": { "type": "string" } @@ -966,9 +782,7 @@ "properties": { "type": { "type": "string", - "enum": [ - "either" - ] + "enum": ["either"] }, "title": { "type": "string" @@ -986,10 +800,7 @@ }, "description": { "default": null, - "type": [ - "string", - "null" - ] + "type": ["string", "null"] }, "injection": { "default": null, @@ -1004,10 +815,7 @@ }, "enum": { "default": null, - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "items": { "type": "string" } @@ -1033,19 +841,11 @@ }, { "type": "object", - "required": [ - "as_id", - "policies", - "runtime", - "title", - "type" - ], + "required": ["as_id", "policies", "runtime", "title", "type"], "properties": { "type": { "type": "string", - "enum": [ - "any" - ] + "enum": ["any"] }, "title": { "type": "string" @@ -1063,10 +863,7 @@ }, "description": { "default": null, - "type": [ - "string", - "null" - ] + "type": ["string", "null"] }, "injection": { "default": null, @@ -1081,10 +878,7 @@ }, "enum": { "default": null, - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "items": { "type": "string" } @@ -1117,34 +911,22 @@ "type": "object", "properties": { "read": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "uint32", "minimum": 0.0 }, "create": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "uint32", "minimum": 0.0 }, "delete": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "uint32", "minimum": 0.0 }, "update": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "uint32", "minimum": 0.0 } @@ -1154,16 +936,11 @@ "oneOf": [ { "type": "object", - "required": [ - "data", - "source" - ], + "required": ["data", "source"], "properties": { "source": { "type": "string", - "enum": [ - "static" - ] + "enum": ["static"] }, "data": { "$ref": "#/definitions/InjectionData_for_String" @@ -1172,16 +949,11 @@ }, { "type": "object", - "required": [ - "data", - "source" - ], + "required": ["data", "source"], "properties": { "source": { "type": "string", - "enum": [ - "context" - ] + "enum": ["context"] }, "data": { "$ref": "#/definitions/InjectionData_for_String" @@ -1190,16 +962,11 @@ }, { "type": "object", - "required": [ - "data", - "source" - ], + "required": ["data", "source"], "properties": { "source": { "type": "string", - "enum": [ - "secret" - ] + "enum": ["secret"] }, "data": { "$ref": "#/definitions/InjectionData_for_String" @@ -1208,16 +975,11 @@ }, { "type": "object", - "required": [ - "data", - "source" - ], + "required": ["data", "source"], "properties": { "source": { "type": "string", - "enum": [ - "parent" - ] + "enum": ["parent"] }, "data": { "$ref": "#/definitions/InjectionData_for_uint32" @@ -1226,16 +988,11 @@ }, { "type": "object", - "required": [ - "data", - "source" - ], + "required": ["data", "source"], "properties": { "source": { "type": "string", - "enum": [ - "dynamic" - ] + "enum": ["dynamic"] }, "data": { "$ref": "#/definitions/InjectionData_for_String" @@ -1244,16 +1001,11 @@ }, { "type": "object", - "required": [ - "data", - "source" - ], + "required": ["data", "source"], "properties": { "source": { "type": "string", - "enum": [ - "random" - ] + "enum": ["random"] }, "data": { "$ref": "#/definitions/InjectionData_for_String" @@ -1277,9 +1029,7 @@ }, "SingleValue_for_String": { "type": "object", - "required": [ - "value" - ], + "required": ["value"], "properties": { "value": { "type": "string" @@ -1303,9 +1053,7 @@ }, "SingleValue_for_uint32": { "type": "object", - "required": [ - "value" - ], + "required": ["value"], "properties": { "value": { "type": "integer", @@ -1330,10 +1078,7 @@ }, "FunctionParameterTransform": { "type": "object", - "required": [ - "resolver_input", - "transform_root" - ], + "required": ["resolver_input", "transform_root"], "properties": { "resolver_input": { "type": "integer", @@ -1347,10 +1092,7 @@ }, "ParameterTransformNode": { "type": "object", - "required": [ - "data", - "typeIdx" - ], + "required": ["data", "typeIdx"], "properties": { "typeIdx": { "type": "integer", @@ -1376,16 +1118,11 @@ "oneOf": [ { "type": "object", - "required": [ - "name", - "source" - ], + "required": ["name", "source"], "properties": { "source": { "type": "string", - "enum": [ - "arg" - ] + "enum": ["arg"] }, "name": { "type": "string" @@ -1394,16 +1131,11 @@ }, { "type": "object", - "required": [ - "source", - "valueJson" - ], + "required": ["source", "valueJson"], "properties": { "source": { "type": "string", - "enum": [ - "static" - ] + "enum": ["static"] }, "valueJson": { "type": "string" @@ -1412,16 +1144,11 @@ }, { "type": "object", - "required": [ - "key", - "source" - ], + "required": ["key", "source"], "properties": { "source": { "type": "string", - "enum": [ - "secret" - ] + "enum": ["secret"] }, "key": { "type": "string" @@ -1430,16 +1157,11 @@ }, { "type": "object", - "required": [ - "key", - "source" - ], + "required": ["key", "source"], "properties": { "source": { "type": "string", - "enum": [ - "context" - ] + "enum": ["context"] }, "key": { "type": "string" @@ -1448,16 +1170,11 @@ }, { "type": "object", - "required": [ - "parentIdx", - "source" - ], + "required": ["parentIdx", "source"], "properties": { "source": { "type": "string", - "enum": [ - "parent" - ] + "enum": ["parent"] }, "parentIdx": { "type": "integer", @@ -1472,16 +1189,11 @@ "oneOf": [ { "type": "object", - "required": [ - "fields", - "type" - ], + "required": ["fields", "type"], "properties": { "type": { "type": "string", - "enum": [ - "object" - ] + "enum": ["object"] }, "fields": { "type": "object", @@ -1493,16 +1205,11 @@ }, { "type": "object", - "required": [ - "items", - "type" - ], + "required": ["items", "type"], "properties": { "type": { "type": "string", - "enum": [ - "array" - ] + "enum": ["array"] }, "items": { "type": "array", @@ -1516,12 +1223,7 @@ }, "Materializer": { "type": "object", - "required": [ - "data", - "effect", - "name", - "runtime" - ], + "required": ["data", "effect", "name", "runtime"], "properties": { "name": { "type": "string" @@ -1542,9 +1244,7 @@ }, "Effect": { "type": "object", - "required": [ - "idempotent" - ], + "required": ["idempotent"], "properties": { "effect": { "anyOf": [ @@ -1563,12 +1263,7 @@ }, "EffectType": { "type": "string", - "enum": [ - "create", - "update", - "delete", - "read" - ] + "enum": ["create", "update", "delete", "read"] }, "TGRuntime": { "anyOf": [ @@ -1584,16 +1279,11 @@ "oneOf": [ { "type": "object", - "required": [ - "data", - "name" - ], + "required": ["data", "name"], "properties": { "name": { "type": "string", - "enum": [ - "deno" - ] + "enum": ["deno"] }, "data": { "$ref": "#/definitions/DenoRuntimeData" @@ -1602,16 +1292,11 @@ }, { "type": "object", - "required": [ - "data", - "name" - ], + "required": ["data", "name"], "properties": { "name": { "type": "string", - "enum": [ - "graphql" - ] + "enum": ["graphql"] }, "data": { "$ref": "#/definitions/GraphQLRuntimeData" @@ -1620,16 +1305,11 @@ }, { "type": "object", - "required": [ - "data", - "name" - ], + "required": ["data", "name"], "properties": { "name": { "type": "string", - "enum": [ - "http" - ] + "enum": ["http"] }, "data": { "$ref": "#/definitions/HTTPRuntimeData" @@ -1638,16 +1318,11 @@ }, { "type": "object", - "required": [ - "data", - "name" - ], + "required": ["data", "name"], "properties": { "name": { "type": "string", - "enum": [ - "python_wasi" - ] + "enum": ["python_wasi"] }, "data": { "$ref": "#/definitions/PythonRuntimeData" @@ -1656,16 +1331,11 @@ }, { "type": "object", - "required": [ - "data", - "name" - ], + "required": ["data", "name"], "properties": { "name": { "type": "string", - "enum": [ - "random" - ] + "enum": ["random"] }, "data": { "$ref": "#/definitions/RandomRuntimeData" @@ -1674,16 +1344,11 @@ }, { "type": "object", - "required": [ - "data", - "name" - ], + "required": ["data", "name"], "properties": { "name": { "type": "string", - "enum": [ - "prisma" - ] + "enum": ["prisma"] }, "data": { "$ref": "#/definitions/PrismaRuntimeData" @@ -1692,16 +1357,11 @@ }, { "type": "object", - "required": [ - "data", - "name" - ], + "required": ["data", "name"], "properties": { "name": { "type": "string", - "enum": [ - "prisma_migration" - ] + "enum": ["prisma_migration"] }, "data": { "$ref": "#/definitions/PrismaMigrationRuntimeData" @@ -1710,16 +1370,11 @@ }, { "type": "object", - "required": [ - "data", - "name" - ], + "required": ["data", "name"], "properties": { "name": { "type": "string", - "enum": [ - "s3" - ] + "enum": ["s3"] }, "data": { "$ref": "#/definitions/S3RuntimeData" @@ -1728,16 +1383,11 @@ }, { "type": "object", - "required": [ - "data", - "name" - ], + "required": ["data", "name"], "properties": { "name": { "type": "string", - "enum": [ - "temporal" - ] + "enum": ["temporal"] }, "data": { "$ref": "#/definitions/TemporalRuntimeData" @@ -1746,16 +1396,11 @@ }, { "type": "object", - "required": [ - "data", - "name" - ], + "required": ["data", "name"], "properties": { "name": { "type": "string", - "enum": [ - "wasmedge" - ] + "enum": ["wasmedge"] }, "data": { "$ref": "#/definitions/WasmEdgeRuntimeData" @@ -1764,16 +1409,11 @@ }, { "type": "object", - "required": [ - "data", - "name" - ], + "required": ["data", "name"], "properties": { "name": { "type": "string", - "enum": [ - "typegate" - ] + "enum": ["typegate"] }, "data": { "$ref": "#/definitions/TypegateRuntimeData" @@ -1782,16 +1422,11 @@ }, { "type": "object", - "required": [ - "data", - "name" - ], + "required": ["data", "name"], "properties": { "name": { "type": "string", - "enum": [ - "typegraph" - ] + "enum": ["typegraph"] }, "data": { "$ref": "#/definitions/TypegraphRuntimeData" @@ -1802,10 +1437,7 @@ }, "DenoRuntimeData": { "type": "object", - "required": [ - "permissions", - "worker" - ], + "required": ["permissions", "worker"], "properties": { "worker": { "type": "string" @@ -1818,9 +1450,7 @@ }, "GraphQLRuntimeData": { "type": "object", - "required": [ - "endpoint" - ], + "required": ["endpoint"], "properties": { "endpoint": { "type": "string" @@ -1829,24 +1459,16 @@ }, "HTTPRuntimeData": { "type": "object", - "required": [ - "endpoint" - ], + "required": ["endpoint"], "properties": { "endpoint": { "type": "string" }, "cert_secret": { - "type": [ - "string", - "null" - ] + "type": ["string", "null"] }, "basic_auth_secret": { - "type": [ - "string", - "null" - ] + "type": ["string", "null"] } } }, @@ -1854,10 +1476,7 @@ "type": "object", "properties": { "config": { - "type": [ - "string", - "null" - ] + "type": ["string", "null"] } } }, @@ -1865,18 +1484,12 @@ "type": "object", "properties": { "seed": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "uint32", "minimum": 0.0 }, "reset": { - "type": [ - "string", - "null" - ] + "type": ["string", "null"] } } }, @@ -1978,9 +1591,7 @@ "properties": { "type": { "type": "string", - "enum": [ - "scalar" - ] + "enum": ["scalar"] }, "key": { "type": "string" @@ -2030,9 +1641,7 @@ "properties": { "type": { "type": "string", - "enum": [ - "relationship" - ] + "enum": ["relationship"] }, "key": { "type": "string" @@ -2065,58 +1674,41 @@ "oneOf": [ { "type": "object", - "required": [ - "type" - ], + "required": ["type"], "properties": { "type": { "type": "string", - "enum": [ - "Boolean" - ] + "enum": ["Boolean"] } } }, { "type": "object", - "required": [ - "type" - ], + "required": ["type"], "properties": { "type": { "type": "string", - "enum": [ - "Int" - ] + "enum": ["Int"] } } }, { "type": "object", - "required": [ - "type" - ], + "required": ["type"], "properties": { "type": { "type": "string", - "enum": [ - "Float" - ] + "enum": ["Float"] } } }, { "type": "object", - "required": [ - "format", - "type" - ], + "required": ["format", "type"], "properties": { "type": { "type": "string", - "enum": [ - "String" - ] + "enum": ["String"] }, "format": { "$ref": "#/definitions/StringType" @@ -2127,19 +1719,11 @@ }, "StringType": { "type": "string", - "enum": [ - "Plain", - "Uuid", - "DateTime" - ] + "enum": ["Plain", "Uuid", "DateTime"] }, "Cardinality": { "type": "string", - "enum": [ - "optional", - "one", - "many" - ] + "enum": ["optional", "one", "many"] }, "ManagedInjection": { "type": "object", @@ -2168,24 +1752,15 @@ }, "Injection2": { "type": "string", - "enum": [ - "DateNow" - ] + "enum": ["DateNow"] }, "Side": { "type": "string", - "enum": [ - "left", - "right" - ] + "enum": ["left", "right"] }, "Relationship": { "type": "object", - "required": [ - "left", - "name", - "right" - ], + "required": ["left", "name", "right"], "properties": { "name": { "type": "string" @@ -2200,11 +1775,7 @@ }, "RelationshipModel": { "type": "object", - "required": [ - "cardinality", - "field", - "type_idx" - ], + "required": ["cardinality", "field", "type_idx"], "properties": { "type_idx": { "type": "integer", @@ -2221,16 +1792,10 @@ }, "MigrationOptions": { "type": "object", - "required": [ - "create", - "reset" - ], + "required": ["create", "reset"], "properties": { "migration_files": { - "type": [ - "string", - "null" - ] + "type": ["string", "null"] }, "create": { "type": "boolean" @@ -2272,10 +1837,7 @@ }, "TemporalRuntimeData": { "type": "object", - "required": [ - "host", - "name" - ], + "required": ["host", "name"], "properties": { "name": { "type": "string" @@ -2289,10 +1851,7 @@ "type": "object", "properties": { "config": { - "type": [ - "string", - "null" - ] + "type": ["string", "null"] } } }, @@ -2304,10 +1863,7 @@ }, "UnknownRuntime": { "type": "object", - "required": [ - "data", - "name" - ], + "required": ["data", "name"], "properties": { "name": { "type": "string" @@ -2320,10 +1876,7 @@ }, "Policy": { "type": "object", - "required": [ - "materializer", - "name" - ], + "required": ["materializer", "name"], "properties": { "name": { "type": "string" @@ -2347,10 +1900,7 @@ ], "properties": { "prefix": { - "type": [ - "string", - "null" - ] + "type": ["string", "null"] }, "secrets": { "type": "array", @@ -2383,11 +1933,8 @@ "version": { "type": "string" }, - "randomSeed": { - "type": [ - "integer", - "null" - ], + "random_seed": { + "type": ["integer", "null"], "format": "uint32", "minimum": 0.0 }, @@ -2401,10 +1948,7 @@ }, "Queries": { "type": "object", - "required": [ - "dynamic", - "endpoints" - ], + "required": ["dynamic", "endpoints"], "properties": { "dynamic": { "type": "boolean" @@ -2455,10 +1999,7 @@ "type": "boolean" }, "max_age_sec": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "uint32", "minimum": 0.0 } @@ -2466,11 +2007,7 @@ }, "Auth": { "type": "object", - "required": [ - "auth_data", - "name", - "protocol" - ], + "required": ["auth_data", "name", "protocol"], "properties": { "name": { "type": "string" @@ -2486,20 +2023,11 @@ }, "AuthProtocol": { "type": "string", - "enum": [ - "oauth2", - "jwt", - "basic" - ] + "enum": ["oauth2", "jwt", "basic"] }, "Rate": { "type": "object", - "required": [ - "local_excess", - "query_limit", - "window_limit", - "window_sec" - ], + "required": ["local_excess", "query_limit", "window_limit", "window_sec"], "properties": { "window_limit": { "type": "integer", @@ -2517,10 +2045,7 @@ "minimum": 0.0 }, "context_identifier": { - "type": [ - "string", - "null" - ] + "type": ["string", "null"] }, "local_excess": { "type": "integer", @@ -2531,11 +2056,7 @@ }, "Artifact": { "type": "object", - "required": [ - "hash", - "path", - "size" - ], + "required": ["hash", "path", "size"], "properties": { "path": { "type": "string" @@ -2553,9 +2074,7 @@ "FunctionMatData": { "title": "FunctionMatData", "type": "object", - "required": [ - "script" - ], + "required": ["script"], "properties": { "script": { "type": "string" @@ -2565,9 +2084,7 @@ "ModuleMatData": { "title": "ModuleMatData", "type": "object", - "required": [ - "code" - ], + "required": ["code"], "properties": { "code": { "type": "string" @@ -2579,31 +2096,21 @@ "oneOf": [ { "type": "object", - "required": [ - "data", - "name" - ], + "required": ["data", "name"], "properties": { "name": { "type": "string", - "enum": [ - "presign_get" - ] + "enum": ["presign_get"] }, "data": { "type": "object", - "required": [ - "bucket" - ], + "required": ["bucket"], "properties": { "bucket": { "type": "string" }, "expiry_secs": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "uint", "minimum": 0.0 } @@ -2613,37 +2120,24 @@ }, { "type": "object", - "required": [ - "data", - "name" - ], + "required": ["data", "name"], "properties": { "name": { "type": "string", - "enum": [ - "presign_put" - ] + "enum": ["presign_put"] }, "data": { "type": "object", - "required": [ - "bucket" - ], + "required": ["bucket"], "properties": { "bucket": { "type": "string" }, "content_type": { - "type": [ - "string", - "null" - ] + "type": ["string", "null"] }, "expiry_secs": { - "type": [ - "integer", - "null" - ], + "type": ["integer", "null"], "format": "uint", "minimum": 0.0 } @@ -2653,22 +2147,15 @@ }, { "type": "object", - "required": [ - "data", - "name" - ], + "required": ["data", "name"], "properties": { "name": { "type": "string", - "enum": [ - "list" - ] + "enum": ["list"] }, "data": { "type": "object", - "required": [ - "bucket" - ], + "required": ["bucket"], "properties": { "bucket": { "type": "string" @@ -2679,22 +2166,15 @@ }, { "type": "object", - "required": [ - "data", - "name" - ], + "required": ["data", "name"], "properties": { "name": { "type": "string", - "enum": [ - "upload" - ] + "enum": ["upload"] }, "data": { "type": "object", - "required": [ - "bucket" - ], + "required": ["bucket"], "properties": { "bucket": { "type": "string" @@ -2705,22 +2185,15 @@ }, { "type": "object", - "required": [ - "data", - "name" - ], + "required": ["data", "name"], "properties": { "name": { "type": "string", - "enum": [ - "upload_all" - ] + "enum": ["upload_all"] }, "data": { "type": "object", - "required": [ - "bucket" - ], + "required": ["bucket"], "properties": { "bucket": { "type": "string" @@ -2734,10 +2207,7 @@ "PrismaOperationMatData": { "title": "PrismaOperationMatData", "type": "object", - "required": [ - "operation", - "table" - ], + "required": ["operation", "table"], "properties": { "table": { "type": "string" @@ -2746,10 +2216,7 @@ "type": "string" }, "ordered_keys": { - "type": [ - "array", - "null" - ], + "type": ["array", "null"], "items": { "type": "string" } @@ -2759,10 +2226,7 @@ "WasiMatData": { "title": "WasiMatData", "type": "object", - "required": [ - "func", - "wasmArtifact" - ], + "required": ["func", "wasmArtifact"], "properties": { "func": { "type": "string"