Merge branch 'main' into lfx-2024

pkg/client/client.go

@@ -265,11 +265,6 @@ func (c *KpmClient) ResolvePkgDepsMetadata(kclPkg *pkg.KclPkg, update bool) erro
 			} else if d.IsFromLocal() && !utils.DirExists(d.GetLocalFullPath(kclPkg.HomePath)) {
 				return reporter.NewErrorEvent(reporter.DependencyNotFound, fmt.Errorf("dependency '%s' not found in '%s'", d.Name, searchFullPath))
 			} else if d.IsFromLocal() && utils.DirExists(d.GetLocalFullPath(kclPkg.HomePath)) {
-				sum, err := utils.HashDir(d.GetLocalFullPath(kclPkg.HomePath))
-				if err != nil {
-					return reporter.NewErrorEvent(reporter.CalSumFailed, err, fmt.Sprintf("failed to calculate checksum for '%s' in '%s'", d.Name, searchFullPath))
-				}
-				d.Sum = sum
 				depPkg, err := c.LoadPkgFromPath(d.GetLocalFullPath(kclPkg.HomePath))
 				if err != nil {
 					return err
@@ -648,18 +643,12 @@ func (c *KpmClient) AddDepToPkg(kclPkg *pkg.KclPkg, d *pkg.Dependency) error {
 	}
 
 	// download all the dependencies.
-	changedDeps, _, err := c.InitGraphAndDownloadDeps(kclPkg)
+	_, _, err := c.InitGraphAndDownloadDeps(kclPkg)
 
 	if err != nil {
 		return err
 	}
 
-	// Update kcl.mod and kcl.mod.lock
-	for k, v := range changedDeps.Deps {
-		kclPkg.ModFile.Dependencies.Deps[k] = v
-		kclPkg.Dependencies.Deps[k] = v
-	}
-
 	return err
 }
 
@@ -725,7 +714,7 @@ func (c *KpmClient) VendorDeps(kclPkg *pkg.KclPkg) error {
 		}
 		vendorFullPath := filepath.Join(vendorPath, d.FullName)
 		// If the package already exists in the 'vendor', do nothing.
-		if utils.DirExists(vendorFullPath) && check(d, vendorFullPath) {
+		if depExisted(vendorFullPath, d) {
 			continue
 		} else {
 			// If not in the 'vendor', check the global cache.
@@ -736,7 +725,7 @@ func (c *KpmClient) VendorDeps(kclPkg *pkg.KclPkg) error {
 				if err != nil {
 					return err
 				}
-			} else if utils.DirExists(d.GetLocalFullPath(kclPkg.HomePath)) && check(d, d.GetLocalFullPath(kclPkg.HomePath)) {
+			} else if depExisted(d.GetLocalFullPath(kclPkg.HomePath), d) {
 				// If there is, copy it into the 'vendor' directory.
 				err := copy.Copy(d.GetLocalFullPath(kclPkg.HomePath), vendorFullPath)
 				if err != nil {
@@ -761,6 +750,14 @@ func (c *KpmClient) VendorDeps(kclPkg *pkg.KclPkg) error {
 	return nil
 }
 
+// depExisted will check whether the dependency exists in the local path.
+// If the dep is from local, do not need to check the checksum, so return true directly if it exists.
+// If the dep is from git or oci, check the checksum, so return true if the checksum is correct and it exist.
+func depExisted(localPath string, dep pkg.Dependency) bool {
+	return (utils.DirExists(localPath) && check(dep, localPath)) ||
+		(utils.DirExists(localPath) && dep.IsFromLocal())
+}
+
 // FillDepInfo will fill registry information for a dependency.
 func (c *KpmClient) FillDepInfo(dep *pkg.Dependency, homepath string) error {
 	if dep.Source.Local != nil {
@@ -859,14 +856,16 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (*
 		dep.FromKclPkg(kpkg)
 	}
 
-	var err error
-	dep.Sum, err = utils.HashDir(dep.LocalFullPath)
-	if err != nil {
-		return nil, reporter.NewErrorEvent(
-			reporter.FailedHashPkg,
-			err,
-			fmt.Sprintf("failed to hash the kcl package '%s' in '%s'.", dep.Name, dep.LocalFullPath),
-		)
+	if dep.Source.Local == nil {
+		var err error
+		dep.Sum, err = utils.HashDir(dep.LocalFullPath)
+		if err != nil {
+			return nil, reporter.NewErrorEvent(
+				reporter.FailedHashPkg,
+				err,
+				fmt.Sprintf("failed to hash the kcl package '%s' in '%s'.", dep.Name, dep.LocalFullPath),
+			)
+		}
 	}
 
 	return dep, nil
@@ -1256,7 +1255,7 @@ func (c *KpmClient) InitGraphAndDownloadDeps(kclPkg *pkg.KclPkg) (*pkg.Dependenc
 		return nil, nil, err
 	}
 
-	changedDeps, err := c.downloadDeps(kclPkg.ModFile.Dependencies, kclPkg.Dependencies, depGraph, kclPkg.HomePath, root)
+	changedDeps, err := c.downloadDeps(&kclPkg.ModFile.Dependencies, &kclPkg.Dependencies, depGraph, kclPkg.HomePath, root)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -1300,7 +1299,7 @@ func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencie
 			return nil, errors.InvalidDependency
 		}
 
-		existDep := c.dependencyExists(&d, &lockDeps)
+		existDep := c.dependencyExists(&d, lockDeps)
 		if existDep != nil {
 			newDeps.Deps[d.Name] = *existDep
 			continue
@@ -1333,9 +1332,10 @@ func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencie
 			}
 		}
 
-		// Update kcl.mod and kcl.mod.lock
 		newDeps.Deps[d.Name] = *lockedDep
-		lockDeps.Deps[d.Name] = *lockedDep
+		// After downloading the dependency in kcl.mod, update the dep into to the kcl.mod
+		// Only the direct dependencies are updated to kcl.mod.
+		deps.Deps[d.Name] = *lockedDep
 	}
 
 	// necessary to make a copy as when we are updating kcl.mod in below for loop
@@ -1383,20 +1383,25 @@ func (c *KpmClient) downloadDeps(deps pkg.Dependencies, lockDeps pkg.Dependencie
 			return nil, err
 		}
 
-		// Download the dependencies.
-		nested, err := c.downloadDeps(deppkg.ModFile.Dependencies, lockDeps, depGraph, deppkg.HomePath, source)
+		// Download the indirect dependencies.
+		nested, err := c.downloadDeps(&deppkg.ModFile.Dependencies, lockDeps, depGraph, deppkg.HomePath, source)
 		if err != nil {
 			return nil, err
 		}
 
-		// Update kcl.mod.
 		for _, d := range nested.Deps {
 			if _, ok := newDeps.Deps[d.Name]; !ok {
 				newDeps.Deps[d.Name] = d
 			}
 		}
 	}
 
+	// After each dependency is downloaded, update all the new deps to kcl.mod.lock.
+	// No matter whether the dependency is directly or indirectly.
+	for k, v := range newDeps.Deps {
+		lockDeps.Deps[k] = v
+	}
+
 	return &newDeps, nil
 }
 

pkg/client/client_test.go

@@ -17,6 +17,7 @@ import (
 	"github.com/otiai10/copy"
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/mod/module"
+	"gopkg.in/yaml.v3"
 	"kcl-lang.io/kcl-go/pkg/kcl"
 	"kcl-lang.io/kpm/pkg/env"
 	"kcl-lang.io/kpm/pkg/git"
@@ -1385,3 +1386,44 @@ func TestRunOciWithSettingsFile(t *testing.T) {
 	_, err = kpmcli.CompileOciPkg("oci://ghcr.io/kcl-lang/helloworld", "", opts)
 	assert.Equal(t, err, nil)
 }
+
+func TestRunGitWithLocalDep(t *testing.T) {
+	kpmcli, err := NewKpmClient()
+	assert.Equal(t, err, nil)
+
+	testPath := getTestDir("test_run_git_with_local_dep")
+	defer func() {
+		_ = os.RemoveAll(filepath.Join(testPath, "catalog"))
+	}()
+
+	testCases := []struct {
+		ref        string
+		expectFile string
+	}{
+		{"8308200", "expect1.yaml"},
+		{"0b3f5ab", "expect2.yaml"},
+	}
+
+	for _, tc := range testCases {
+
+		expectPath := filepath.Join(testPath, tc.expectFile)
+		opts := opt.DefaultCompileOptions()
+		gitOpts := git.NewCloneOptions("https://github.com/kcl-lang/flask-demo-kcl-manifests.git", tc.ref, "", "", "", nil)
+
+		result, err := kpmcli.CompileGitPkg(gitOpts, opts)
+		assert.Equal(t, err, nil)
+
+		fileBytes, err := os.ReadFile(expectPath)
+		assert.Equal(t, err, nil)
+
+		var expectObj map[string]interface{}
+		err = yaml.Unmarshal(fileBytes, &expectObj)
+		assert.Equal(t, err, nil)
+
+		var gotObj map[string]interface{}
+		err = yaml.Unmarshal([]byte(result.GetRawJsonResult()), &gotObj)
+		assert.Equal(t, err, nil)
+
+		assert.Equal(t, gotObj, expectObj)
+	}
+}

pkg/client/test_data/add_with_local_path/expect/pkg/kcl.mod.lock

@@ -3,7 +3,6 @@
     name = "dep_pkg"
     full_name = "dep_pkg_0.0.1"
     version = "0.0.1"
-    sum = "C3TjaZJVdt4G8TtbiCxTUO/zAlf7fWJTAvs/CDMnlxY="
   [dependencies.helloworld]
     name = "helloworld"
     full_name = "helloworld_0.1.1"

pkg/client/test_data/test_run_git_with_local_dep/expect1.yaml

@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flask-demo
+  labels:
+    app: flask-demo
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: flask-demo
+  template:
+    metadata:
+      labels:
+        app: flask-demo
+    spec:
+      containers:
+      - name: flaskdemo
+        image: kcllang/flask_demo:8d31498e765ff67a2fa9933d4adffe067544b2fe
+        ports:
+        - protocol: TCP
+          containerPort: 5000
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: flask-demo
+  labels:
+    app: flask-demo
+  namespace: default
+spec:
+  type: NodePort
+  selector:
+    app: flask-demo
+  ports:
+  - port: 5000
+    protocol: TCP
+    targetPort: 5000

pkg/client/test_data/test_run_git_with_local_dep/expect2.yaml

@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: flask-demo
+  labels:
+    app: flask-demo
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: flask-demo
+  template:
+    metadata:
+      labels:
+        app: flask-demo
+    spec:
+      containers:
+      - name: flaskdemo
+        image: kcllang/flask_demo:8d31498e765ff67a2fa9933d4adffe067544b2fe
+        ports:
+        - protocol: TCP
+          containerPort: 5000
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: flask-demo
+  labels:
+    app: flask-demo
+  namespace: default
+spec:
+  type: NodePort
+  selector:
+    app: flask-demo
+  ports:
+  - port: 5000
+    protocol: TCP
+    targetPort: 5000

pkg/package/modfile.go

@@ -202,7 +202,7 @@ func (d Dependency) WithTheSameVersion(other Dependency) bool {
 
 // GetLocalFullPath will get the local path of a dependency.
 func (dep *Dependency) GetLocalFullPath(rootpath string) string {
-	if len(dep.LocalFullPath) == 0 && dep.IsFromLocal() {
+	if !filepath.IsAbs(dep.LocalFullPath) && dep.IsFromLocal() {
 		if filepath.IsAbs(dep.Source.Local.Path) {
 			return dep.Source.Local.Path
 		}

pkg/utils/test_data/test_link/is_link_exist/link_target_not_exist

@@ -0,0 +1 @@
+not_exist

pkg/utils/test_data/test_link/is_link_exist/test.txt

@@ -0,0 +1 @@
+test for create link

pkg/utils/utils.go

@@ -284,6 +284,45 @@ func DirExists(path string) bool {
 	return err == nil
 }
 
+// IsSymlinkValidAndExists will check whether the symlink exists and points to a valid target
+// return three values: whether the symlink exists, whether it points to a valid target, and any error encountered
+// Note: IsSymlinkValidAndExists is only useful on unix-like systems.
+func IsSymlinkValidAndExists(symlinkPath string) (bool, bool, error) {
+	// check if the symlink exists
+	info, err := os.Lstat(symlinkPath)
+	if err != nil && os.IsNotExist(err) {
+		// symlink does not exist
+		return false, false, nil
+	} else if err != nil {
+		// other error
+		return false, false, err
+	}
+
+	// check if the file is a symlink
+	if info.Mode()&os.ModeSymlink == os.ModeSymlink {
+		// get the target of the symlink
+		target, err := os.Readlink(symlinkPath)
+		if err != nil {
+			// can not read the target
+			return true, false, err
+		}
+
+		// check if the target exists
+		_, err = os.Stat(target)
+		if err == nil {
+			// target exists
+			return true, true, nil
+		}
+		if os.IsNotExist(err) {
+			// target does not exist
+			return true, false, nil
+		}
+		return true, false, err
+	}
+
+	return false, false, fmt.Errorf("%s exists but is not a symlink", symlinkPath)
+}
+
 // DefaultKpmHome create the '.kpm' in the user home and return the path of ".kpm".
 func CreateSubdirInUserHome(subdir string) (string, error) {
 	homeDir, err := os.UserHomeDir()
@@ -304,15 +343,22 @@ func CreateSubdirInUserHome(subdir string) (string, error) {
 
 // CreateSymlink will create symbolic link named 'newName' for 'oldName',
 // and if the symbolic link already exists, it will be deleted and recreated.
+// Note: CreateSymlink is only useful on unix-like systems.
 func CreateSymlink(oldName, newName string) error {
-	if DirExists(newName) {
+	symExist, _, err := IsSymlinkValidAndExists(newName)
+
+	if err != nil {
+		return err
+	}
+
+	if symExist {
 		err := os.Remove(newName)
 		if err != nil {
 			return err
 		}
 	}
 
-	err := os.Symlink(oldName, newName)
+	err = os.Symlink(oldName, newName)
 	if err != nil {
 		return err
 	}