Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Handle v2 storage engine paths properly. #8

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Handle v2 storage engine paths properly. #8

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f8f9551
Handle v2 storage engine paths properly.
jwilner Dec 29, 2020
2000b2c
Don't use new style format verb
jwilner Dec 29, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
117 changes: 111 additions & 6 deletions plugin/plugin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,17 @@ package plugin

import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"path"
"strconv"
"strings"

"github.com/drone/drone-go/drone"
"github.com/drone/drone-go/plugin/secret"
"github.com/sirupsen/logrus"

"github.com/hashicorp/vault/api"
)
Expand Down Expand Up @@ -78,6 +85,8 @@ func (p *plugin) Find(ctx context.Context, req *secret.Request) (*drone.Secret,

// helper function returns the secret from vault.
func (p *plugin) find(path string) (map[string]string, error) {
isV2, path := p.rewritePath(path)

secret, err := p.client.Logical().Read(path)
if err != nil {
return nil, err
Expand All @@ -86,12 +95,18 @@ func (p *plugin) find(path string) (map[string]string, error) {
return nil, errors.New("secret not found")
}

// HACK: the vault v2 key value store is confusing
// and I could not quite figure out how to work with
// the api. This is the workaround I came up with.
v := secret.Data["data"]
if data, ok := v.(map[string]interface{}); ok {
secret.Data = data
// the V2 api includes both "data" and "metadata" fields within the top level "data" -- we only care about data.
// https://www.vaultproject.io/api-docs/secret/kv/kv-v1#sample-response
// v1 data schema:
// { properties: { data: { type: object, description: "the actual data" }}}
// https://www.vaultproject.io/api/secret/kv/kv-v2#sample-response-1
// v2 data schema:
// { properties: { data: { properties: { data: { type: object, description: "the actual data" }}}}}
if isV2 {
v := secret.Data["data"]
if data, ok := v.(map[string]interface{}); ok {
secret.Data = data
}
}

params := map[string]string{}
Expand All @@ -104,3 +119,93 @@ func (p *plugin) find(path string) (map[string]string, error) {
}
return params, err
}

// rewritePath rewrites a secret path if need be according to storage engine constraints; if it fails, it returns the
// original path.
//
// TL;DR: vault requires rewriting secret paths for the V2 engine mount points. This is most visible when you use the
// CLI to output curl strings:
// $ vault kv get \
// -output-curl-string \
// foo/versioned/bar
// curl -H "X-Vault-Request: true" \
// -H "X-Vault-Token: $(vault print token)" \
// https://vault.example.com/v1/foo/versioned/data/bar
//
// Note the addition of "data" in the output curl string. This only occurs for the v2 engine. This function
// reproduces the logic from the CLI:
// https://github.com/hashicorp/vault/blob/7aa1ffa92ee61b977efad1488b8f309b1e2136df/command/kv_get.go#L94-L110
func (p *plugin) rewritePath(path string) (bool, string) {
r := p.client.NewRequest("GET", "/v1/sys/internal/ui/mounts/"+path)
resp, err := p.client.RawRequest(r)
if err != nil {
logrus.Debugf("failed querying mount point; defaulting to original: %v", err)
return false, path
}
defer resp.Body.Close()
isV2, rewritten, err := rewritePath(resp.Body, path)
if err != nil {
logrus.Debugf("failed rewriting; defaulting to original: %v", err)
return false, path
}
logrus.Debugf("rewrote %q to %q", path, rewritten)
return isV2, rewritten
}

func rewritePath(r io.Reader, original string) (isV2 bool, rewritten string, _ error) {
defer func() {
// never permit a trailing slash, no matter what user puts in
rewritten = strings.TrimSuffix(rewritten, "/")
}()
/*
Example v2 response:
{
"request_id": "4a3a3ef6-d0a8-9a9b-d7eb-c320ef170b55",
"lease_id": "",
"renewable": false,
"lease_duration": 0,
"data": {
"accessor": "kv_f055aa7b",
"config": {
"default_lease_ttl": 0,
"force_no_cache": false,
"max_lease_ttl": 0
},
"description": "versioned encrypted key/value storage",
"local": false,
"options": {
"version": "2"
},
"path": "foo/versioned/",
"seal_wrap": false,
"type": "kv",
"uuid": "eb3b578c-a0bf-2a91-19dc-4155e8ae0116"
},
"wrap_info": null,
"warnings": null,
"auth": null
}
*/
var response struct {
Data struct {
Options struct {
Version string `json:"version"`
} `json:"options"`
Path string `json:"path"`
} `json:"data"`
}
if err := json.NewDecoder(r).Decode(&response); err != nil {
return false, original, fmt.Errorf("failed parsing response: %v", err)
}
v, err := strconv.Atoi(response.Data.Options.Version)
if err != nil || v != 2 {
return false, original, nil // we only rewrite v2
}

mountPath := response.Data.Path
if original == mountPath || original == strings.TrimSuffix(mountPath, "/") {
return true, path.Join(mountPath, "data"), nil
}

return true, path.Join(mountPath, "data", strings.TrimPrefix(original, mountPath)), nil
}
66 changes: 66 additions & 0 deletions plugin/plugin_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,12 @@ package plugin

import (
"context"
"encoding/json"
"io/ioutil"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"

"github.com/drone/drone-go/drone"
Expand Down Expand Up @@ -242,3 +245,66 @@ func TestPlugin_KeyNotFound(t *testing.T) {
return
}
}

/* Test_rewritePath establishes that the behavior of our path rewrite logic exactly parallels that of the Vault CLI;
generated with logic like (requires authentication to a vault namespace with both v1 and v2 engines mounted):

main() {
for path in mount{/v2{,/data},/v1}{/bar,}{,/}; do
jq --null-input \
--arg mount_data "$(get_mount "${path}")" \
--arg rewritten "$(get_rewritten_path "${path}")" \
--arg path "${path}" \
'{$mount_data, $rewritten, $path}'
done |
jq -s '[.[] | (.is_v2 = (.mount_data | fromjson).data.options.version == "2")]'
}

get_mount() {
local path="${1}"
curl \
--silent \
-H "X-Vault-Request: true" \
-H "X-Vault-Token: $(vault print token)" \
"https://vault.example.com/v1/sys/internal/ui/mounts/${path}"
}

get_rewritten_path() {
local path="${1}"
vault kv get -output-curl-string "${path}" | cut -d/ -f5-
}

main
*/
func Test_rewritePath(t *testing.T) {
var testCases []struct {
Path string `json:"path"`
MountData string `json:"mount_data"`
Rewritten string `json:"rewritten"`
IsV2 bool `json:"is_v2"`
}
func() {
f, err := os.Open("testdata/v2.json")
if err != nil {
t.Skipf("expected test data present at testdata/v2.json: %v", err)
}
defer f.Close()
if err := json.NewDecoder(f).Decode(&testCases); err != nil {
t.Fatalf("malformed test data: %v", err)
}
}()
for _, tc := range testCases {
t.Run(strings.ReplaceAll(tc.Path, "/", "_"), func(t *testing.T) {
isV2, rewrite, err := rewritePath(strings.NewReader(tc.MountData), tc.Path)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if rewrite != tc.Rewritten {
t.Errorf("expected %q but got %q", tc.Rewritten, rewrite)
}
if isV2 != tc.IsV2 {
t.Errorf("expected %v but got %v", tc.IsV2, isV2)
}
})
}
}
34 changes: 34 additions & 0 deletions plugin/testdata/generate_v2_json.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/usr/bin/env bash

# assumes the user has access to s a PREFIX with a V1 mount and a V2 mount

readonly _PREFIX="${PREFIX:-mount}"
readonly _V2="${V2_MOUNT:-/v2}"
readonly _V1="${V1_MOUNT:-/v1}"

main() {
for path in $_PREFIX{$_V2{,/data},$_V1}{/bar,}{,/}; do
jq --null-input \
--arg mount_data "$(get_mount "${path}")" \
--arg rewritten "$(get_rewritten_path "${path}")" \
--arg path "${path}" \
'{$mount_data, $rewritten, $path}'
done |
jq -s '[.[] | (.is_v2 = (.mount_data | fromjson).data.options.version == "2")]'
}

get_mount() {
local path="${1}"
curl \
--silent \
-H "X-Vault-Request: true" \
-H "X-Vault-Token: $(vault print token)" \
"https://vault.example.com/v1/sys/internal/ui/mounts/${path}"
}

get_rewritten_path() {
local path="${1}"
vault kv get -output-curl-string "${path}" | cut -d/ -f5-
}

main
74 changes: 74 additions & 0 deletions plugin/testdata/v2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
[
{
"mount_data": "{\"request_id\":\"b2f6799d-7a1d-8bdc-ac5f-0c1238c17969\",\"lease_id\":\"\",\"renewable\":false,\"lease_duration\":0,\"data\":{\"accessor\":\"kv_f055aa7b\",\"config\":{\"default_lease_ttl\":0,\"force_no_cache\":false,\"max_lease_ttl\":0},\"description\":\"versioned encrypted key/value storage\",\"local\":false,\"options\":{\"version\":\"2\"},\"path\":\"mount/v2/\",\"seal_wrap\":false,\"type\":\"kv\",\"uuid\":\"eb3b578c-a0bf-2a91-19dc-4155e8ae0116\"},\"wrap_info\":null,\"warnings\":null,\"auth\":null}",
"rewritten": "mount/v2/data/bar",
"path": "mount/v2/bar",
"is_v2": true
},
{
"mount_data": "{\"request_id\":\"85d5a8e6-348c-a9e5-7cf8-b0fce5d3c3af\",\"lease_id\":\"\",\"renewable\":false,\"lease_duration\":0,\"data\":{\"accessor\":\"kv_f055aa7b\",\"config\":{\"default_lease_ttl\":0,\"force_no_cache\":false,\"max_lease_ttl\":0},\"description\":\"versioned encrypted key/value storage\",\"local\":false,\"options\":{\"version\":\"2\"},\"path\":\"mount/v2/\",\"seal_wrap\":false,\"type\":\"kv\",\"uuid\":\"eb3b578c-a0bf-2a91-19dc-4155e8ae0116\"},\"wrap_info\":null,\"warnings\":null,\"auth\":null}",
"rewritten": "mount/v2/data/bar",
"path": "mount/v2/bar/",
"is_v2": true
},
{
"mount_data": "{\"request_id\":\"db9ca87a-f7a1-70f0-89aa-8ca985483793\",\"lease_id\":\"\",\"renewable\":false,\"lease_duration\":0,\"data\":{\"accessor\":\"kv_f055aa7b\",\"config\":{\"default_lease_ttl\":0,\"force_no_cache\":false,\"max_lease_ttl\":0},\"description\":\"versioned encrypted key/value storage\",\"local\":false,\"options\":{\"version\":\"2\"},\"path\":\"mount/v2/\",\"seal_wrap\":false,\"type\":\"kv\",\"uuid\":\"eb3b578c-a0bf-2a91-19dc-4155e8ae0116\"},\"wrap_info\":null,\"warnings\":null,\"auth\":null}",
"rewritten": "mount/v2/data",
"path": "mount/v2",
"is_v2": true
},
{
"mount_data": "{\"request_id\":\"2c1e9144-af1b-41a4-1c3e-980e13f0e937\",\"lease_id\":\"\",\"renewable\":false,\"lease_duration\":0,\"data\":{\"accessor\":\"kv_f055aa7b\",\"config\":{\"default_lease_ttl\":0,\"force_no_cache\":false,\"max_lease_ttl\":0},\"description\":\"versioned encrypted key/value storage\",\"local\":false,\"options\":{\"version\":\"2\"},\"path\":\"mount/v2/\",\"seal_wrap\":false,\"type\":\"kv\",\"uuid\":\"eb3b578c-a0bf-2a91-19dc-4155e8ae0116\"},\"wrap_info\":null,\"warnings\":null,\"auth\":null}",
"rewritten": "mount/v2/data",
"path": "mount/v2/",
"is_v2": true
},
{
"mount_data": "{\"request_id\":\"ccb9fbd3-ed2c-eb98-f8e6-0f5fcb570f57\",\"lease_id\":\"\",\"renewable\":false,\"lease_duration\":0,\"data\":{\"accessor\":\"kv_f055aa7b\",\"config\":{\"default_lease_ttl\":0,\"force_no_cache\":false,\"max_lease_ttl\":0},\"description\":\"versioned encrypted key/value storage\",\"local\":false,\"options\":{\"version\":\"2\"},\"path\":\"mount/v2/\",\"seal_wrap\":false,\"type\":\"kv\",\"uuid\":\"eb3b578c-a0bf-2a91-19dc-4155e8ae0116\"},\"wrap_info\":null,\"warnings\":null,\"auth\":null}",
"rewritten": "mount/v2/data/data/bar",
"path": "mount/v2/data/bar",
"is_v2": true
},
{
"mount_data": "{\"request_id\":\"a6d671b1-9029-95ac-5560-846345443f81\",\"lease_id\":\"\",\"renewable\":false,\"lease_duration\":0,\"data\":{\"accessor\":\"kv_f055aa7b\",\"config\":{\"default_lease_ttl\":0,\"force_no_cache\":false,\"max_lease_ttl\":0},\"description\":\"versioned encrypted key/value storage\",\"local\":false,\"options\":{\"version\":\"2\"},\"path\":\"mount/v2/\",\"seal_wrap\":false,\"type\":\"kv\",\"uuid\":\"eb3b578c-a0bf-2a91-19dc-4155e8ae0116\"},\"wrap_info\":null,\"warnings\":null,\"auth\":null}",
"rewritten": "mount/v2/data/data/bar",
"path": "mount/v2/data/bar/",
"is_v2": true
},
{
"mount_data": "{\"request_id\":\"23fba99b-f552-6ec5-a0d8-f6ef41d4e4c0\",\"lease_id\":\"\",\"renewable\":false,\"lease_duration\":0,\"data\":{\"accessor\":\"kv_f055aa7b\",\"config\":{\"default_lease_ttl\":0,\"force_no_cache\":false,\"max_lease_ttl\":0},\"description\":\"versioned encrypted key/value storage\",\"local\":false,\"options\":{\"version\":\"2\"},\"path\":\"mount/v2/\",\"seal_wrap\":false,\"type\":\"kv\",\"uuid\":\"eb3b578c-a0bf-2a91-19dc-4155e8ae0116\"},\"wrap_info\":null,\"warnings\":null,\"auth\":null}",
"rewritten": "mount/v2/data/data",
"path": "mount/v2/data",
"is_v2": true
},
{
"mount_data": "{\"request_id\":\"b2d1501e-f748-4c71-0258-27b36aa95f2b\",\"lease_id\":\"\",\"renewable\":false,\"lease_duration\":0,\"data\":{\"accessor\":\"kv_f055aa7b\",\"config\":{\"default_lease_ttl\":0,\"force_no_cache\":false,\"max_lease_ttl\":0},\"description\":\"versioned encrypted key/value storage\",\"local\":false,\"options\":{\"version\":\"2\"},\"path\":\"mount/v2/\",\"seal_wrap\":false,\"type\":\"kv\",\"uuid\":\"eb3b578c-a0bf-2a91-19dc-4155e8ae0116\"},\"wrap_info\":null,\"warnings\":null,\"auth\":null}",
"rewritten": "mount/v2/data/data",
"path": "mount/v2/data/",
"is_v2": true
},
{
"mount_data": "{\"request_id\":\"253551b3-a01e-1126-0dee-f5fd3470b4a4\",\"lease_id\":\"\",\"renewable\":false,\"lease_duration\":0,\"data\":{\"accessor\":\"generic_c45ac6cc\",\"config\":{\"default_lease_ttl\":86400,\"force_no_cache\":false,\"max_lease_ttl\":604800},\"description\":\"encrypted key/value storage\",\"local\":false,\"options\":{},\"path\":\"mount/v1/\",\"seal_wrap\":false,\"type\":\"generic\",\"uuid\":\"b0c47310-507d-e5d6-f186-03d3aa71cb58\"},\"wrap_info\":null,\"warnings\":null,\"auth\":null}",
"rewritten": "mount/v1/bar",
"path": "mount/v1/bar",
"is_v2": false
},
{
"mount_data": "{\"request_id\":\"1f5f8620-cdb3-0aa1-81f3-4308e3e5fd6a\",\"lease_id\":\"\",\"renewable\":false,\"lease_duration\":0,\"data\":{\"accessor\":\"generic_c45ac6cc\",\"config\":{\"default_lease_ttl\":86400,\"force_no_cache\":false,\"max_lease_ttl\":604800},\"description\":\"encrypted key/value storage\",\"local\":false,\"options\":{},\"path\":\"mount/v1/\",\"seal_wrap\":false,\"type\":\"generic\",\"uuid\":\"b0c47310-507d-e5d6-f186-03d3aa71cb58\"},\"wrap_info\":null,\"warnings\":null,\"auth\":null}",
"rewritten": "mount/v1/bar",
"path": "mount/v1/bar/",
"is_v2": false
},
{
"mount_data": "{\"request_id\":\"a9ed61f0-fc96-3f5e-188a-c52afcb8341d\",\"lease_id\":\"\",\"renewable\":false,\"lease_duration\":0,\"data\":{\"accessor\":\"generic_c45ac6cc\",\"config\":{\"default_lease_ttl\":86400,\"force_no_cache\":false,\"max_lease_ttl\":604800},\"description\":\"encrypted key/value storage\",\"local\":false,\"options\":{},\"path\":\"mount/v1/\",\"seal_wrap\":false,\"type\":\"generic\",\"uuid\":\"b0c47310-507d-e5d6-f186-03d3aa71cb58\"},\"wrap_info\":null,\"warnings\":null,\"auth\":null}",
"rewritten": "mount/v1",
"path": "mount/v1",
"is_v2": false
},
{
"mount_data": "{\"request_id\":\"fe545ea3-06a5-51d3-0236-4982e896cf8f\",\"lease_id\":\"\",\"renewable\":false,\"lease_duration\":0,\"data\":{\"accessor\":\"generic_c45ac6cc\",\"config\":{\"default_lease_ttl\":86400,\"force_no_cache\":false,\"max_lease_ttl\":604800},\"description\":\"encrypted key/value storage\",\"local\":false,\"options\":{},\"path\":\"mount/v1/\",\"seal_wrap\":false,\"type\":\"generic\",\"uuid\":\"b0c47310-507d-e5d6-f186-03d3aa71cb58\"},\"wrap_info\":null,\"warnings\":null,\"auth\":null}",
"rewritten": "mount/v1",
"path": "mount/v1/",
"is_v2": false
}
]