Feat: LSP will execute 'kpm metadata' to get the localpath of the kcl…

… dependecies. (#554)

* Feat: LSP will execute 'kpm metadata' to get the localpath of the kcl dependecies.

* fix: fix path error on windows in test case.

* fix: fix CR comments

* fix: add comments for 'pkg_name' in 'ImportStmt'.

* fix: make cargo fmt happy.

* fix: add more comments.

* fix: fix typo.

.github/workflows/macos_test.yaml

@@ -13,6 +13,14 @@ jobs:
         with:
           submodules: "true"
 
+      - name: Set up Go 1.18
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18
+
+      - name: Install kpm
+        run: go install kusionstack.io/kpm@latest
+
       - run: clang --version
       - run: cargo --version
       - run: rustc --print sysroot

.github/workflows/ubuntu_test.yaml

@@ -12,6 +12,10 @@ jobs:
         uses: actions/checkout@v3
         with:
           submodules: "true"
+      - name: Set up Go 1.18
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18
       - name: Delete rust cargo
         run: rm -rf /root/.cargo/bin
         shell: bash
@@ -37,6 +41,9 @@ jobs:
         working-directory: ./kclvm
         run: export PATH=$PATH:$PWD/../_build/dist/ubuntu/kclvm/bin && make install-rustc-wasm && make && make test-runtime
         shell: bash
+      - name: Install kpm
+        run: go install kusionstack.io/kpm@latest
+
       - name: Unit test
         working-directory: ./kclvm
         run: export PATH=$PATH:$PWD/../_build/dist/ubuntu/kclvm/bin && make install-rustc-wasm && make && make codecov-lcov

.github/workflows/windows_test.yaml

@@ -12,6 +12,14 @@ jobs:
         uses: actions/checkout@v2
         with:
           submodules: "true"
+
+      - name: Set up Go 1.18
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.18
+
+      - name: Install kpm
+        run: go install kusionstack.io/kpm@latest
 
       - uses: ilammy/msvc-dev-cmd@v1
 

.gitignore

@@ -92,3 +92,6 @@ _a.out_*.*
 # LLVM
 llvm*
 llvm-*
+
+# kpm
+!.mod.lock

kclvm/ast/src/ast.rs

@@ -424,10 +424,19 @@ pub struct IfStmt {
 /// ```
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct ImportStmt {
+    /// `path` is the import path, if 'import a.b.c' in kcl, `path` is a.b.c
     pub path: String,
     pub rawpath: String,
     pub name: String,
     pub asname: Option<String>,
+    /// `pkg_name` means the name of the package that the current import statement indexs to.
+    ///
+    /// 1. If the current import statement indexs to the kcl plugins, kcl builtin methods or the internal kcl packages,
+    /// `pkg_name` is `__main__`
+    ///
+    /// 2. If the current import statement indexs to the external kcl packages, `pkg_name` is the name of the package.
+    /// if `import k8s.example.apps`, `k8s` is another kcl package, `pkg_name` is `k8s`.
+    pub pkg_name: String,
 }
 
 /// SchemaStmt, e.g.

kclvm/driver/Cargo.toml

@@ -14,4 +14,6 @@ kclvm-utils ={ path = "../utils"}
 kclvm-parser ={ path = "../parser"}
 kclvm-ast ={ path = "../ast"}
 walkdir = "2"
+
+serde = { version = "1.0", features = ["derive"] }
 anyhow = { version = "1.0.70", features = ["backtrace"] }

kclvm/driver/src/kpm_metadata.rs

@@ -0,0 +1,140 @@
+use anyhow::{bail, Ok, Result};
+use kclvm_parser::LoadProgramOptions;
+use serde::{Deserialize, Serialize};
+use std::{collections::HashMap, env, iter, path::PathBuf, process::Command};
+
+const MANIFEST_FILE: &str = "kcl.mod";
+
+/// [`fill_pkg_maps_for_k_file`] will call `kpm metadata` to obtain the metadata
+/// of all dependent packages of the kcl package where the current file is located,
+/// and fill the relevant information of the external packages into compilation option [`LoadProgramOptions`].
+pub(crate) fn fill_pkg_maps_for_k_file(
+    k_file_path: PathBuf,
+    opts: &mut LoadProgramOptions,
+) -> Result<()> {
+    // 1. find the kcl.mod dir for the kcl package contains 'k_file_path'.
+    match lookup_the_nearest_file_dir(k_file_path, MANIFEST_FILE) {
+        Some(mod_dir) => {
+            // 2. call `kpm metadata`.
+            let metadata = fetch_metadata(mod_dir.canonicalize()?)?;
+            // 3. fill the external packages local paths into compilation option [`LoadProgramOptions`].
+            let maps: HashMap<String, String> = metadata
+                .packages
+                .into_iter()
+                .map(|(pname, pkg)| (pname, pkg.manifest_path.display().to_string()))
+                .collect();
+            opts.package_maps.extend(maps);
+        }
+        None => return Ok(()),
+    };
+
+    return Ok(());
+}
+
+#[derive(Deserialize, Serialize, Default, Debug, Clone)]
+/// [`Metadata`] is the metadata of the current KCL package,
+/// currently only the mapping between the name and path of the external dependent package is included.
+pub struct Metadata {
+    pub packages: HashMap<String, Package>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+/// [`Package`] is a kcl package.
+pub struct Package {
+    /// Name as given in the `kcl.mod`
+    pub name: String,
+    /// Path containing the `kcl.mod`
+    pub manifest_path: PathBuf,
+}
+
+impl Metadata {
+    /// [`parse`] will parse the json string into [`Metadata`].
+    fn parse(data: String) -> Result<Self> {
+        let meta = serde_json::from_str(data.as_ref())?;
+        Ok(meta)
+    }
+}
+
+/// [`fetch_metadata`] will call `kpm metadata` to obtain the metadata.
+pub fn fetch_metadata(manifest_path: PathBuf) -> Result<Metadata> {
+    let output = Command::new(kpm())
+        .arg("metadata")
+        .current_dir(manifest_path)
+        .output()
+        .unwrap();
+
+    if !output.status.success() {
+        bail!(
+            "fetch workspace failed with error: {}",
+            String::from_utf8_lossy(&output.stderr)
+        );
+    }
+
+    Ok(Metadata::parse(
+        String::from_utf8_lossy(&output.stdout).to_string(),
+    )?)
+}
+
+/// [`lookup_the_nearest_file_dir`] will start from [`from`] and search for file [`the_nearest_file`] in the parent directories.
+/// If found, it will return the [`Some`] of [`the_nearest_file`] file path. If not found, it will return [`None`]
+pub(crate) fn lookup_the_nearest_file_dir(
+    from: PathBuf,
+    the_nearest_file: &str,
+) -> Option<PathBuf> {
+    let mut current_dir = from;
+
+    loop {
+        let found_path = current_dir.join(the_nearest_file);
+        if found_path.is_file() {
+            return Some(current_dir.canonicalize().ok()?);
+        }
+
+        match current_dir.parent() {
+            Some(parent) => current_dir = parent.to_path_buf(),
+            None => return None,
+        }
+    }
+}
+
+/// [`kpm`] will return the path for executable kpm binary.
+pub fn kpm() -> PathBuf {
+    get_path_for_executable("kpm")
+}
+
+/// [`get_path_for_executable`] will return the path for [`executable_name`].
+pub fn get_path_for_executable(executable_name: &'static str) -> PathBuf {
+    // The current implementation checks $PATH for an executable to use:
+    // `<executable_name>`
+    //  example: for kpm, this tries just `kpm`, which will succeed if `kpm` is on the $PATH
+
+    if lookup_in_path(executable_name) {
+        return executable_name.into();
+    }
+
+    executable_name.into()
+}
+
+/// [`lookup_in_path`] will search for an executable file [`exec`] in the environment variable ‘PATH’.
+///  If found, return true, otherwise return false.
+fn lookup_in_path(exec: &str) -> bool {
+    let paths = env::var_os("PATH").unwrap_or_default();
+    env::split_paths(&paths)
+        .map(|path| path.join(exec))
+        .find_map(probe)
+        .is_some()
+}
+
+/// [`probe`] check if the given path points to a file.
+/// If it does, return [`Some`] of the path.
+/// If not, check if adding the current operating system's executable file extension (if any) to the path points to a file.
+/// If it does, return [`Some`] of the path with the extension added.
+/// If neither, return [`None`].
+fn probe(path: PathBuf) -> Option<PathBuf> {
+    let with_extension = match env::consts::EXE_EXTENSION {
+        "" => None,
+        it => Some(path.with_extension(it)),
+    };
+    iter::once(path)
+        .chain(with_extension)
+        .find(|it| it.is_file())
+}

kclvm/driver/src/lib.rs

@@ -1,5 +1,6 @@
 use anyhow::Result;
 pub mod arguments;
+pub mod kpm_metadata;
 pub const DEFAULT_PROJECT_FILE: &str = "project.yaml";
 
 #[cfg(test)]
@@ -12,6 +13,7 @@ use kclvm_config::{
 };
 use kclvm_parser::LoadProgramOptions;
 use kclvm_utils::path::PathPrefix;
+use kpm_metadata::fill_pkg_maps_for_k_file;
 use std::{
     fs::read_dir,
     io::{self, ErrorKind},
@@ -72,6 +74,7 @@ pub fn lookup_compile_unit(
     file: &str,
     load_pkg: bool,
 ) -> (Vec<String>, Option<LoadProgramOptions>) {
+    let compiled_file: String = file.to_string();
     match lookup_compile_unit_path(file) {
         Ok(dir) => {
             let settings_files = lookup_setting_files(&dir);
@@ -101,7 +104,7 @@ pub fn lookup_compile_unit(
                         .map(|p| p.to_string_lossy().to_string())
                         .unwrap_or_default();
 
-                    let load_opt = kclvm_parser::LoadProgramOptions {
+                    let mut load_opt = kclvm_parser::LoadProgramOptions {
                         work_dir: work_dir.clone(),
                         cmd_args: if let Some(options) = setting.clone().kcl_options {
                             options
@@ -117,27 +120,34 @@ pub fn lookup_compile_unit(
                         ..Default::default()
                     };
                     match canonicalize_input_files(&files, work_dir, true) {
-                        Ok(kcl_paths) => (kcl_paths, Some(load_opt)),
+                        Ok(kcl_paths) => {
+                            // 1. find the kcl.mod path
+                            let _ = fill_pkg_maps_for_k_file(compiled_file.into(), &mut load_opt);
+                            (kcl_paths, Some(load_opt))
+                        }
                         Err(_) => (vec![file.to_string()], None),
                     }
                 }
                 Err(_) => (vec![file.to_string()], None),
             }
         }
         Err(_) => {
+            let mut load_opt = kclvm_parser::LoadProgramOptions::default();
+            let _ = fill_pkg_maps_for_k_file(compiled_file.into(), &mut load_opt);
+
             if load_pkg {
                 let path = Path::new(file);
                 if let Some(ext) = path.extension() {
                     if ext == KCL_FILE_EXTENSION && path.is_file() {
                         if let Some(parent) = path.parent() {
                             if let Ok(files) = get_kcl_files(parent, false) {
-                                return (files, None);
+                                return (files, Some(load_opt));
                             }
                         }
                     }
                 }
             }
-            return (vec![file.to_string()], None);
+            return (vec![file.to_string()], Some(load_opt));
         }
     }
 }
@@ -185,7 +195,7 @@ pub fn lookup_kcl_yaml(dir: &PathBuf) -> io::Result<PathBuf> {
 /// | | +-- stack.yaml
 /// | +-- project.yaml
 ///
-/// If the input file is project/base/base.k, it will return Path("project/base")
+/// If the input file is project/base/base.k, it will return Path("project/prod")
 /// If the input file is project/prod/main.k or project/test/main.k, it will return
 /// Path("project/prod") or Path("project/test")
 pub fn lookup_compile_unit_path(file: &str) -> io::Result<PathBuf> {

kclvm/driver/src/test_data/kpm_metadata/kcl.mod

@@ -0,0 +1,7 @@
+[package]
+name = "kcl1"
+edition = "0.0.1"
+version = "0.0.4"
+
+[dependencies]
+kcl4 = { git = "", tag = "v0.0.1" }

kclvm/driver/src/test_data/kpm_metadata/kcl.mod.lock

@@ -0,0 +1,8 @@
+[dependencies]
+  [dependencies.kcl4]
+    name = "kcl4"
+    full_name = "kcl4_v0.0.1"
+    version = "v0.0.1"
+    sum = "cpyqJwwjqCvast6QNAiYuevgAIEH1p72OqctwGHU79Q="
+    url = ""
+    tag = "v0.0.1"

kclvm/driver/src/test_data/test_vendor/kcl4_v0.0.1/k4/main.k

@@ -0,0 +1 @@
+demo = 4

kclvm/driver/src/test_data/test_vendor/kcl4_v0.0.1/kcl.mod

@@ -0,0 +1,6 @@
+[package]
+name = "kcl4"
+edition = "0.0.1"
+version = "0.0.1"
+
+[dependencies]