Skip to content

Commit

Permalink
Merge pull request #376 from zong-zhe/unify-pull-api
Browse files Browse the repository at this point in the history
feat: add unified api for CLI pull command
  • Loading branch information
Peefy committed Jul 4, 2024
2 parents ebf591f + bb045eb commit efc7b9c
Show file tree
Hide file tree
Showing 20 changed files with 1,129 additions and 406 deletions.
3 changes: 1 addition & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ require (
github.com/distribution/reference v0.6.0
github.com/dominikbraun/graph v0.23.0
github.com/elliotchance/orderedmap/v2 v2.2.0
github.com/google/uuid v1.6.0
github.com/opencontainers/image-spec v1.1.0
github.com/otiai10/copy v1.14.0
github.com/sirupsen/logrus v1.9.3
Expand Down Expand Up @@ -74,7 +75,6 @@ require (
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/pprof v0.0.0-20240424215950-a892ee059fd6 // indirect
github.com/google/s2a-go v0.1.7 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
github.com/googleapis/gax-go/v2 v2.12.2 // indirect
github.com/gorilla/mux v1.8.1 // indirect
Expand Down Expand Up @@ -145,7 +145,6 @@ require (

require (
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/elliotchance/orderedmap v1.6.0
github.com/go-git/go-git/v5 v5.11.0
github.com/gofrs/flock v0.8.1
github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
Expand Down
2 changes: 0 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -314,8 +314,6 @@ github.com/dominikbraun/graph v0.23.0 h1:TdZB4pPqCLFxYhdyMFb1TBdFxp8XLcJfTTBQucV
github.com/dominikbraun/graph v0.23.0/go.mod h1:yOjYyogZLY1LSG9E33JWZJiq5k83Qy2C6POAuiViluc=
github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
github.com/elliotchance/orderedmap v1.6.0 h1:xjn+kbbKXeDq6v9RVE+WYwRbYfAZKvlWfcJNxM8pvEw=
github.com/elliotchance/orderedmap v1.6.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
github.com/elliotchance/orderedmap/v2 v2.2.0 h1:7/2iwO98kYT4XkOjA9mBEIwvi4KpGB4cyHeOFOnj4Vk=
github.com/elliotchance/orderedmap/v2 v2.2.0/go.mod h1:85lZyVbpGaGvHvnKa7Qhx7zncAdBIBq6u56Hb1PRU5Q=
github.com/emicklei/proto v1.13.2 h1:z/etSFO3uyXeuEsVPzfl56WNgzcvIr42aQazXaQmFZY=
Expand Down
5 changes: 3 additions & 2 deletions pkg/api/kpm_run_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/elliotchance/orderedmap/v2"
"github.com/stretchr/testify/assert"
"kcl-lang.io/kcl-go/pkg/kcl"
"kcl-lang.io/kpm/pkg/downloader"
"kcl-lang.io/kpm/pkg/opt"
pkg "kcl-lang.io/kpm/pkg/package"
"kcl-lang.io/kpm/pkg/utils"
Expand Down Expand Up @@ -266,8 +267,8 @@ func TestStoreModAndModLockFile(t *testing.T) {
Version: "0.0.1",
Sum: "sLr3e6W4RPrXYyswdOSiKqkHes1QHX2tk6SwxAPDqqo=",
LocalFullPath: filepath.Join(testPath, "dep1_0.0.1"),
Source: pkg.Source{
Oci: &pkg.Oci{
Source: downloader.Source{
Oci: &downloader.Oci{
Reg: "ghcr.io",
Repo: "kcl-lang/dep1",
Tag: "0.0.1",
Expand Down
68 changes: 63 additions & 5 deletions pkg/client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -952,7 +952,7 @@ func (c *KpmClient) FillDependenciesInfo(modFile *pkg.ModFile) error {
}

// AcquireTheLatestOciVersion will acquire the latest version of the OCI reference.
func (c *KpmClient) AcquireTheLatestOciVersion(ociSource pkg.Oci) (string, error) {
func (c *KpmClient) AcquireTheLatestOciVersion(ociSource downloader.Oci) (string, error) {
ociClient, err := oci.NewOciClient(ociSource.Reg, ociSource.Repo, &c.settings)
if err != nil {
return "", err
Expand All @@ -961,6 +961,64 @@ func (c *KpmClient) AcquireTheLatestOciVersion(ociSource pkg.Oci) (string, error
return ociClient.TheLatestTag()
}

func (c *KpmClient) downloadPkg(options ...downloader.Option) (*pkg.KclPkg, error) {
opts := downloader.DownloadOptions{}
for _, option := range options {
option(&opts)
}

localPath := opts.LocalPath
tmpDir, err := os.MkdirTemp("", "")
if err != nil {
return nil, err
}
tmpDir = filepath.Join(tmpDir, constants.GitScheme)
// clean the temp dir.
defer os.RemoveAll(tmpDir)
err = c.DepDownloader.Download(*downloader.NewDownloadOptions(
downloader.WithLocalPath(tmpDir),
downloader.WithSource(opts.Source),
downloader.WithLogWriter(c.GetLogWriter()),
downloader.WithSettings(*c.GetSettings()),
))

if err != nil {
return nil, err
}

if utils.DirExists(localPath) {
err := os.RemoveAll(localPath)
if err != nil {
return nil, err
}
}

destDir := filepath.Dir(localPath)
if !utils.DirExists(destDir) {
err = os.MkdirAll(destDir, os.ModePerm)
if err != nil {
return nil, err
}
}

err = utils.MoveFile(tmpDir, localPath)
if err != nil {
return nil, err
}

localPath, err = filepath.Abs(localPath)
if err != nil {
return nil, err
}

pkg, err := c.LoadPkgFromPath(localPath)
if err != nil {
return nil, err
}

return pkg, nil
}

// Download will download the dependency to the local path.
func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (*pkg.Dependency, error) {
if dep.Source.Git != nil {
Expand Down Expand Up @@ -992,7 +1050,7 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (*
}

if dep.Source.Oci != nil || dep.Source.Registry != nil {
var ociSource *pkg.Oci
var ociSource *downloader.Oci
if dep.Source.Oci != nil {
ociSource = dep.Source.Oci
} else if dep.Source.Registry != nil {
Expand Down Expand Up @@ -1135,7 +1193,7 @@ func (c *KpmClient) Download(dep *pkg.Dependency, homePath, localPath string) (*
}

// DownloadFromGit will download the dependency from the git repository.
func (c *KpmClient) DownloadFromGit(dep *pkg.Git, localPath string) (string, error) {
func (c *KpmClient) DownloadFromGit(dep *downloader.Git, localPath string) (string, error) {
var msg string
if len(dep.Tag) != 0 {
msg = fmt.Sprintf("with tag '%s'", dep.Tag)
Expand Down Expand Up @@ -1217,7 +1275,7 @@ func (c *KpmClient) ParseKclModFile(kclPkg *pkg.KclPkg) (map[string]map[string]s
}

// LoadPkgFromOci will download the kcl package from the oci repository and return an `KclPkg`.
func (c *KpmClient) DownloadPkgFromOci(dep *pkg.Oci, localPath string) (*pkg.KclPkg, error) {
func (c *KpmClient) DownloadPkgFromOci(dep *downloader.Oci, localPath string) (*pkg.KclPkg, error) {
ociClient, err := oci.NewOciClient(dep.Reg, dep.Repo, &c.settings)
if err != nil {
return nil, err
Expand Down Expand Up @@ -1263,7 +1321,7 @@ func (c *KpmClient) DownloadPkgFromOci(dep *pkg.Oci, localPath string) (*pkg.Kcl

// DownloadFromOci will download the dependency from the oci repository.
// Deprecated: Use the DownloadPkgFromOci instead.
func (c *KpmClient) DownloadFromOci(dep *pkg.Oci, localPath string) (string, error) {
func (c *KpmClient) DownloadFromOci(dep *downloader.Oci, localPath string) (string, error) {
ociClient, err := oci.NewOciClient(dep.Reg, dep.Repo, &c.settings)
if err != nil {
return "", err
Expand Down
22 changes: 11 additions & 11 deletions pkg/client/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,8 +67,8 @@ func TestDownloadOci(t *testing.T) {
depFromOci := pkg.Dependency{
Name: "helloworld",
Version: "0.1.2",
Source: pkg.Source{
Oci: &pkg.Oci{
Source: downloader.Source{
Oci: &downloader.Oci{
Reg: "ghcr.io",
Repo: "kcl-lang/helloworld",
Tag: "0.1.2",
Expand Down Expand Up @@ -110,8 +110,8 @@ func TestDownloadLatestOci(t *testing.T) {
depFromOci := pkg.Dependency{
Name: "helloworld",
Version: "",
Source: pkg.Source{
Oci: &pkg.Oci{
Source: downloader.Source{
Oci: &downloader.Oci{
Reg: "ghcr.io",
Repo: "kcl-lang/helloworld",
Tag: "",
Expand Down Expand Up @@ -284,8 +284,8 @@ func TestUpdateKclModAndLock(t *testing.T) {
FullName: "test_version",
Version: "test_version",
Sum: "test_sum",
Source: pkg.Source{
Git: &pkg.Git{
Source: downloader.Source{
Git: &downloader.Git{
Url: "test_url",
Tag: "test_tag",
},
Expand All @@ -297,8 +297,8 @@ func TestUpdateKclModAndLock(t *testing.T) {
FullName: "test_version",
Version: "test_version",
Sum: "test_sum",
Source: pkg.Source{
Oci: &pkg.Oci{
Source: downloader.Source{
Oci: &downloader.Oci{
Reg: "test_reg",
Repo: "test_repo",
Tag: "test_tag",
Expand Down Expand Up @@ -1305,7 +1305,7 @@ func TestLoadPkgFormOci(t *testing.T) {
assert.Equal(t, err, nil)
}()

kclpkg, err := cli.DownloadPkgFromOci(&pkg.Oci{
kclpkg, err := cli.DownloadPkgFromOci(&downloader.Oci{
Reg: tc.Reg,
Repo: tc.Repo,
Tag: tc.Tag,
Expand Down Expand Up @@ -1455,8 +1455,8 @@ func testAddWithOciDownloader(t *testing.T) {
dep := pkg.Dependency{
Name: "helloworld",
FullName: "helloworld_0.0.3",
Source: pkg.Source{
Oci: &pkg.Oci{
Source: downloader.Source{
Oci: &downloader.Oci{
Reg: "ghcr.io",
Repo: "zong-zhe/helloworld",
Tag: "0.0.3",
Expand Down
102 changes: 102 additions & 0 deletions pkg/client/pull.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package client

import (
"errors"
"fmt"
"path/filepath"

"kcl-lang.io/kpm/pkg/downloader"
pkg "kcl-lang.io/kpm/pkg/package"
"kcl-lang.io/kpm/pkg/reporter"
)

// The PullOptions struct contains the options for pulling a package from the registry.
type PullOptions struct {
// Source is the source of the package to be pulled.
// Including git, oci, local.
Source *downloader.Source
// LocalPath is the local path to download the package.
LocalPath string
}

type PullOption func(*PullOptions) error

func WithPullSourceUrl(sourceUrl string) PullOption {
return func(opts *PullOptions) error {
source, err := downloader.NewSourceFromStr(sourceUrl)
if err != nil {
return err
}
opts.Source = source
return nil
}
}

func WithPullSource(source *downloader.Source) PullOption {
return func(opts *PullOptions) error {
if source == nil {
return errors.New("source cannot be nil")
}
opts.Source = source
return nil
}
}

func WithLocalPath(path string) PullOption {
return func(opts *PullOptions) error {
opts.LocalPath = path
return nil
}
}

func NewPullOptions(opts ...PullOption) *PullOptions {
do := &PullOptions{}
for _, opt := range opts {
opt(do)
}
return do
}

func (c *KpmClient) Pull(options ...PullOption) (*pkg.KclPkg, error) {
opts := &PullOptions{}
for _, option := range options {
if err := option(opts); err != nil {
return nil, err
}
}

sourceFilePath, err := opts.Source.ToFilePath()
if err != nil {
return nil, err
}

sourceStr, err := opts.Source.ToString()
if err != nil {
return nil, err
}
reporter.ReportMsgTo(
fmt.Sprintf("start to pull %s", sourceStr),
c.GetLogWriter(),
)

kPkg, err := c.downloadPkg(
// The package pulled will be stored in the 'opts.LocalPath/sourceFilePath'
// 'opts.LocalPath' is the local path input by the user.
// 'sourceFilePath' is generated by the source.
// For example,
// kcl package from 'https://github.com/kcl-lang/kcl' will be stored in '$LOCAL_PATH/git/github.com/kcl-lang/kcl'
// kcl package from 'oci://ghcr.io/kcl-lang/kcl' will be stored in '$LOCAL_PATH/oci/ghcr.io/kcl-lang/kcl'
downloader.WithLocalPath(filepath.Join(opts.LocalPath, sourceFilePath)),
downloader.WithSource(*opts.Source),
)

if err != nil {
return nil, err
}

reporter.ReportMsgTo(
fmt.Sprintf("pulled %s %s successfully", kPkg.GetPkgName(), kPkg.GetPkgVersion()),
c.GetLogWriter(),
)
return kPkg, nil
}
53 changes: 53 additions & 0 deletions pkg/client/pull_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package client

import (
"bytes"
"os"
"path/filepath"
"testing"

"gotest.tools/v3/assert"
"kcl-lang.io/kpm/pkg/downloader"
)

func TestPull(t *testing.T) {
pulledPath := getTestDir("test_pull")

kpmcli, err := NewKpmClient()
assert.NilError(t, err)

var buf bytes.Buffer
kpmcli.SetLogWriter(&buf)

kPkg, err := kpmcli.Pull(
WithLocalPath(pulledPath),
WithPullSource(&downloader.Source{
Oci: &downloader.Oci{
Reg: "ghcr.io",
Repo: "kcl-lang/helloworld",
Tag: "0.0.1",
},
}),
)

pkgPath := filepath.Join(pulledPath, "oci", "ghcr.io", "kcl-lang", "helloworld", "0.0.1")
assert.NilError(t, err)
assert.Equal(t, kPkg.GetPkgName(), "helloworld")
assert.Equal(t, kPkg.GetPkgVersion(), "0.0.1")
assert.Equal(t, kPkg.HomePath, pkgPath)

kPkg, err = kpmcli.Pull(
WithLocalPath(pulledPath),
WithPullSourceUrl("oci://ghcr.io/kcl-lang/helloworld?tag=0.1.0"),
)
pkgPath = filepath.Join(pulledPath, "oci", "ghcr.io", "kcl-lang", "helloworld", "0.1.0")
assert.NilError(t, err)
assert.Equal(t, kPkg.GetPkgName(), "helloworld")
assert.Equal(t, kPkg.GetPkgVersion(), "0.1.0")
assert.Equal(t, kPkg.HomePath, pkgPath)

defer func() {
err = os.RemoveAll(filepath.Join(pulledPath, "oci"))
assert.NilError(t, err)
}()
}
Empty file.
2 changes: 1 addition & 1 deletion pkg/cmd/cmd_push.go
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ func pushPackage(ociUrl string, kclPkg *pkg.KclPkg, vendorMode bool, kpmcli *cli
)
}

ociOpts.Annotations, err = oci.GenOciManifestFromPkg(kclPkg)
ociOpts.Annotations, err = kclPkg.GenOciManifestFromPkg()
if err != nil {
return err
}
Expand Down
Loading

0 comments on commit efc7b9c

Please sign in to comment.