Skip to content

Commit

Permalink
feat: support add dependencies from url (#357)
Browse files Browse the repository at this point in the history 
* feat: support add dependencies from url

Signed-off-by: zongz <[email protected]>

* fix: fix windows path case

Signed-off-by: zongz <[email protected]>

* fix: fix e2e test case

Signed-off-by: zongz <[email protected]>

---------

Signed-off-by: zongz <[email protected]>
  • Loading branch information
@zong-zhe
zong-zhe committed Jun 4, 2024
1 parent 7a35562 commit d7258bd
Show file tree
Hide file tree
Showing 31 changed files with 261 additions and 68 deletions.
8 changes: 4 additions & 4 deletions pkg/client/client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -960,7 +960,7 @@ func TestAddWithNoSumCheck(t *testing.T) {
RegistryOpts: opt.RegistryOptions{
Oci: &opt.OciOptions{
Reg: "ghcr.io",
Repo: "kcl-lang",
Repo: "kcl-lang/helloworld",
PkgName: "helloworld",
Tag: "0.1.0",
},
Expand Down Expand Up @@ -1102,7 +1102,7 @@ func TestAddWithDiffVersionNoSumCheck(t *testing.T) {
RegistryOpts: opt.RegistryOptions{
Oci: &opt.OciOptions{
Reg: "ghcr.io",
Repo: "kcl-lang",
Repo: "kcl-lang/helloworld",
PkgName: "helloworld",
Tag: "0.1.2",
},
Expand Down Expand Up @@ -1166,7 +1166,7 @@ func TestAddWithDiffVersionWithSumCheck(t *testing.T) {
RegistryOpts: opt.RegistryOptions{
Oci: &opt.OciOptions{
Reg: "ghcr.io",
Repo: "kcl-lang",
Repo: "kcl-lang/helloworld",
PkgName: "helloworld",
Tag: "0.1.2",
},
Expand Down Expand Up @@ -1346,7 +1346,7 @@ func TestAddWithLocalPath(t *testing.T) {
RegistryOpts: opt.RegistryOptions{
Oci: &opt.OciOptions{
Reg: "ghcr.io",
Repo: "kcl-lang",
Repo: "kcl-lang/helloworld",
PkgName: "helloworld",
Tag: "0.1.1",
},
Expand Down
82 changes: 24 additions & 58 deletions pkg/cmd/cmd_add.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@ import (
"fmt"
"os"
"path/filepath"
"strings"

"github.com/urfave/cli/v2"
"kcl-lang.io/kpm/pkg/client"
"kcl-lang.io/kpm/pkg/constants"
"kcl-lang.io/kpm/pkg/env"
"kcl-lang.io/kpm/pkg/errors"
"kcl-lang.io/kpm/pkg/opt"
Expand All @@ -31,7 +31,7 @@ func NewAddCmd(kpmcli *client.KpmClient) *cli.Command {
},
&cli.StringSliceFlag{
Name: "tag",
Usage: "Git repository tag",
Usage: "Oci or Git repository tag",
},
&cli.StringSliceFlag{
Name: "commit",
Expand Down Expand Up @@ -151,27 +151,30 @@ func parseAddOptions(c *cli.Context, kpmcli *client.KpmClient, localPath string)
NoSumCheck: noSumCheck,
}, nil
} else {
localPkg, err := parseLocalPathOptions(c)
if err != (*reporter.KpmEvent)(nil) {
// parse from 'kpm add xxx:0.0.1'.
ociReg, err := parseOciRegistryOptions(c, kpmcli)
if err != nil {
regOpt, err := opt.NewRegistryOptionsFrom(c.Args().First(), kpmcli.GetSettings())

if err != nil {
return nil, err
}

if regOpt.Oci != nil {
tag, err := onlyOnceOption(c, constants.Tag)

if err != (*reporter.KpmEvent)(nil) {
return nil, err
}
return &opt.AddOptions{
LocalPath: localPath,
NewPkgName: newPkgName,
RegistryOpts: *ociReg,
NoSumCheck: noSumCheck,
}, nil
} else {
return &opt.AddOptions{
LocalPath: localPath,
NewPkgName: newPkgName,
RegistryOpts: *localPkg,
NoSumCheck: noSumCheck,
}, nil

if len(tag) != 0 {
regOpt.Oci.Tag = tag
}
}

return &opt.AddOptions{
LocalPath: localPath,
NewPkgName: newPkgName,
RegistryOpts: *regOpt,
NoSumCheck: noSumCheck,
}, nil
}
}

Expand Down Expand Up @@ -215,7 +218,7 @@ func parseGitRegistryOptions(c *cli.Context) (*opt.RegistryOptions, *reporter.Kp
// parseOciRegistryOptions will parse the oci registry information from user cli inputs.
func parseOciRegistryOptions(c *cli.Context, kpmcli *client.KpmClient) (*opt.RegistryOptions, error) {
ociPkgRef := c.Args().First()
name, version, err := ParseOciPkgNameAndVersion(ociPkgRef)
name, version, err := opt.ParseOciPkgNameAndVersion(ociPkgRef)
if err != nil {
return nil, err
}
Expand All @@ -229,40 +232,3 @@ func parseOciRegistryOptions(c *cli.Context, kpmcli *client.KpmClient) (*opt.Reg
},
}, nil
}

// parseLocalPathOptions will parse the local path information from user cli inputs.
func parseLocalPathOptions(c *cli.Context) (*opt.RegistryOptions, *reporter.KpmEvent) {
localPath := c.Args().First()
if localPath == "" {
return nil, reporter.NewErrorEvent(reporter.PathIsEmpty, errors.PathIsEmpty)
}
// check if the local path exists.
if _, err := os.Stat(localPath); os.IsNotExist(err) {
return nil, reporter.NewErrorEvent(reporter.LocalPathNotExist, err)
} else {
return &opt.RegistryOptions{
Local: &opt.LocalOptions{
Path: localPath,
},
}, nil
}
}

// parseOciPkgNameAndVersion will parse package name and version
// from string "<pkg_name>:<pkg_version>".
func ParseOciPkgNameAndVersion(s string) (string, string, error) {
parts := strings.Split(s, ":")
if len(parts) == 1 {
return parts[0], "", nil
}

if len(parts) > 2 {
return "", "", reporter.NewErrorEvent(reporter.InvalidPkgRef, fmt.Errorf("invalid oci package reference '%s'", s))
}

if parts[1] == "" {
return "", "", reporter.NewErrorEvent(reporter.InvalidPkgRef, fmt.Errorf("invalid oci package reference '%s'", s))
}

return parts[0], parts[1], nil
}
3 changes: 2 additions & 1 deletion pkg/cmd/cmd_update.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"kcl-lang.io/kpm/pkg/client"
"kcl-lang.io/kpm/pkg/env"
"kcl-lang.io/kpm/pkg/mvs"
"kcl-lang.io/kpm/pkg/opt"
pkg "kcl-lang.io/kpm/pkg/package"
"kcl-lang.io/kpm/pkg/reporter"
"kcl-lang.io/kpm/pkg/semver"
Expand Down Expand Up @@ -134,7 +135,7 @@ func KpmUpdate(c *cli.Context, kpmcli *client.KpmClient) error {
// modulesToUpgrade or modulesToDowngrade will be updated.
func GetModulesToUpdate(kclPkg *pkg.KclPkg, modulesToUpgrade []module.Version, modulesToDowngrade []module.Version, pkgInfo string) error {
pkgInfo = strings.TrimSpace(pkgInfo)
pkgName, pkgVersion, err := ParseOciPkgNameAndVersion(pkgInfo)
pkgName, pkgVersion, err := opt.ParseOciPkgNameAndVersion(pkgInfo)
if err != nil {
return err
}
Expand Down
9 changes: 9 additions & 0 deletions pkg/constants/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,22 @@ const (
TarPathSuffix = ".tar"
GitPathSuffix = ".git"
OciScheme = "oci"
GitScheme = "git"
HttpScheme = "http"
HttpsScheme = "https"
SshScheme = "ssh"
FileEntry = "file"
FileWithKclModEntry = "file_with_kcl_mod"
UrlEntry = "url"
RefEntry = "ref"
TarEntry = "tar"
GitEntry = "git"

GitBranch = "branch"
GitCommit = "commit"

Tag = "tag"

KCL_MOD = "kcl.mod"
OCI_SEPARATOR = ":"
KCL_PKG_TAR = "*.tar"
Expand Down
158 changes: 158 additions & 0 deletions pkg/opt/opt.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,17 +3,22 @@
package opt

import (
"fmt"
"io"
"net/url"
"os"
"path/filepath"
"strings"

"github.com/hashicorp/go-version"
"kcl-lang.io/kcl-go/pkg/kcl"
"kcl-lang.io/kpm/pkg/constants"
"kcl-lang.io/kpm/pkg/errors"
"kcl-lang.io/kpm/pkg/path"
"kcl-lang.io/kpm/pkg/reporter"
"kcl-lang.io/kpm/pkg/settings"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/registry"
)

// CompileOptions is the input options of 'kpm run'.
Expand Down Expand Up @@ -190,6 +195,159 @@ type RegistryOptions struct {
Local *LocalOptions
}

// NewRegistryOptionsFrom will parse the registry options from oci url, oci ref and git url.
func NewRegistryOptionsFrom(rawUrlorOciRef string, settings *settings.Settings) (*RegistryOptions, error) {
parsedUrl, err := url.Parse(rawUrlorOciRef)
if err != nil {
return nil, err
}

// parse the options from the local file path
localOptions, err := NewLocalOptionsFromUrl(parsedUrl)
if localOptions != nil && err == (*reporter.KpmEvent)(nil) {
return &RegistryOptions{
Local: localOptions,
}, nil
}

// parse the options from the git url
// https, http, git and ssh are supported
gitOptions := NewGitOptionsFromUrl(parsedUrl)

if gitOptions != nil {
return &RegistryOptions{
Git: gitOptions,
}, nil
}

// parse the options from the oci url
// oci is supported
ociOptions := NewOciOptionsFromUrl(parsedUrl)
if ociOptions == nil {
ociOptions, err = NewOciOptionsFromRef(rawUrlorOciRef, settings)
if err != nil {
return nil, err
}
}

if ociOptions != nil {
return &RegistryOptions{
Oci: ociOptions,
}, nil
}

return nil, fmt.Errorf("invalid dependencies source: %s", rawUrlorOciRef)
}

// NewGitOptionsFromUrl will parse the git options from the git url.
// https, http, git and ssh are supported.
func NewGitOptionsFromUrl(parsedUrl *url.URL) *GitOptions {
if parsedUrl.Scheme != constants.GitScheme &&
parsedUrl.Scheme != constants.HttpScheme &&
parsedUrl.Scheme != constants.HttpsScheme &&
parsedUrl.Scheme != constants.SshScheme {
return nil
}

return &GitOptions{
Url: parsedUrl.Host + parsedUrl.Path,
Branch: parsedUrl.Query().Get(constants.GitBranch),
Tag: parsedUrl.Query().Get(constants.Tag),
Commit: parsedUrl.Query().Get(constants.GitCommit),
}
}

// NewOciOptionsFromUrl will parse the oci options from the oci url.
// oci is supported.
func NewOciOptionsFromUrl(parsedUrl *url.URL) *OciOptions {
if parsedUrl.Scheme != constants.OciScheme {
return nil
}

return &OciOptions{
Reg: parsedUrl.Host,
Repo: parsedUrl.Path,
Tag: parsedUrl.Query().Get(constants.Tag),
PkgName: filepath.Base(parsedUrl.Path),
}
}

// NewOciOptionsFromRef will parse the oci options from the oci ref.
// The ref should be in the format of 'repoName:repoTag'.
func NewOciOptionsFromRef(refStr string, settings *settings.Settings) (*OciOptions, error) {
reg := settings.DefaultOciRegistry()
repo := settings.DefaultOciRepo()
tag := ""

ref, err := registry.ParseReference(refStr)
if err != nil {
var pkgName string
pkgName, tag, err = ParseOciPkgNameAndVersion(refStr)
if err != nil {
return nil, err
}
if !strings.HasPrefix(pkgName, "/") {
repo = fmt.Sprintf("%s/%s", repo, pkgName)
} else {
repo = fmt.Sprintf("%s%s", repo, pkgName)
}
} else {
reg = ref.Registry
repo = ref.Repository
tag = ref.ReferenceOrDefault()
}

return &OciOptions{
Reg: reg,
Repo: repo,
Tag: tag,
PkgName: filepath.Base(repo),
}, nil
}

// NewLocalOptionsFromUrl will parse the local options from the local path.
// scheme 'file' and only path is supported.
func NewLocalOptionsFromUrl(parsedUrl *url.URL) (*LocalOptions, error) {
if parsedUrl.Scheme == "" || parsedUrl.Scheme == constants.FileEntry {
return ParseLocalPathOptions(parsedUrl.Path)
}
return nil, nil
}

// parseOciPkgNameAndVersion will parse package name and version
// from string "<pkg_name>:<pkg_version>".
func ParseOciPkgNameAndVersion(s string) (string, string, error) {
parts := strings.Split(s, ":")
if len(parts) == 1 {
return parts[0], "", nil
}

if len(parts) > 2 {
return "", "", reporter.NewErrorEvent(reporter.InvalidPkgRef, fmt.Errorf("invalid oci package reference '%s'", s))
}

if parts[1] == "" {
return "", "", reporter.NewErrorEvent(reporter.InvalidPkgRef, fmt.Errorf("invalid oci package reference '%s'", s))
}

return parts[0], parts[1], nil
}

// ParseLocalPathOptions will parse the local path information from user cli inputs.
func ParseLocalPathOptions(localPath string) (*LocalOptions, *reporter.KpmEvent) {
if localPath == "" {
return nil, reporter.NewErrorEvent(reporter.PathIsEmpty, errors.PathIsEmpty)
}
// check if the local path exists.
if _, err := os.Stat(localPath); os.IsNotExist(err) {
return nil, reporter.NewErrorEvent(reporter.LocalPathNotExist, err)
} else {
return &LocalOptions{
Path: localPath,
}, nil
}
}

type GitOptions struct {
Url string
Branch string
Expand Down

0 comments on commit d7258bd

Please sign in to comment.