Skip to content

Commit

Permalink
fix v3 repo migration when store config is present, but empty (#3542)
Browse files Browse the repository at this point in the history
For some reason, `config.yaml` file in staging and demo stack have
`JobStore` and `ExecutorStore` entries, but with empty paths, and look
like:
```
...
    Compute:
        ExecutionStore:
            Type: BoltDB
            Path: ""
...
    Requester:
        JobStore:
            Type: BoltDB
            Path: ""
...
```

The v2 -> v3 migrator will skip migrating if the store configs are not
empty. This PR fixes the migrator by checking if the path is empty
instead of if the entry is empty
  • Loading branch information
wdbaruni committed Feb 28, 2024
1 parent b81ea73 commit d9d8de5
Show file tree
Hide file tree
Showing 11 changed files with 83 additions and 3 deletions.
Binary file not shown.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"JobsCompleted":0}
Binary file not shown.
9 changes: 9 additions & 0 deletions pkg/repo/migrations/testdata/v2_empty_path/config.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
node:
compute:
executionstore:
Type: BoltDB
Path: ""
requester:
jobstore:
Type: BoltDB
Path: ""
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
CAASqAkwggSkAgEAAoIBAQC9LNL/FyWbEkZ2+L4DA8i5THITIV4BB5ZLunlUGY+BxDDMMCwNOZR/k1LmfsExoiR7pT4sIlnbAIVmkKodtbOtQ1If4r4SqSPk4dvhTi5u4BEI+WfaSbWw4TutpLykUTGs/3+5uy58UBfvNqiZ0Xtvg9e1PjSqoGdJpkfMs8aZcCe5WiorLzv+P4LFlOKipKF0sJEY2O6+7XQD1nrPT8hyJqp1ZIizt9fhqMVv38W0f1i/VwVfbjy0ajnuxYCI3b6+m93Zw3z3ciP8ayefAJ9xMg0tgmZC0Au9GLYH+PV0zJM8Wt7keW/Cu8KqSTgFoUTw+qXBHPtFYa1iNpNxq0P/AgMBAAECggEACwGhoQ/Ss0wD54YfYAOHWSiWdoB5fKvZVbyatIbsL4ufnzGqTnasKiQDLb/wM5gQcVUy4dR5z+Ee4dHG5AWfV5RUIeDKWEGZAgcWCOzIjNDijNgd77hcJtdj5n1wMEs9Dair4L2APeFXEybl9lUWzm0oTrY40DiW6dUP1lwbUO/40VtiGBnc1d78e6E+WM2iqzUz5wV6VrZ39bZL9EntQ1Ja/Ygf9awgTmAevWKillwBh4SkE7UeQW3NCAmkln9ci6D03W6L1647MmVFybA4TnkRqqCVnan8s+omoodqlD96mhuOzkKJWUwqUbccTOFu7KZIj3pxw0Zo9Nz+WXbXwQKBgQDz8B4ckDQEOwPiBCqE1IsPbVHKWFgcReJ7eyOsBjgS8sYFDJcBaLsoDD9SQPqvmPhPGn0c2H4J+OByKBn5ePNiHNRBuvO1i9Tbrp0Msz6G2E0a0gIvS+9nqV/ITetHATWCVsgAlFOkkpfjj5ZmyWk+zv9ugFurW7ry3GDDCYTzoQKBgQDGh34FLjgvHp7nAifVYEMY/PjrZQjjYQUm2PqrKaVm3IZKPhhJvYUdPHD2ND2k6Xgj7Vl/xGyb1ckg4UdhuT4l6h8gJ/tml1EWlETtrAsCU+rpplf7dne+3zrOIDwziwSDIKUizowBUCF68Pn7vBGiOYh2LY2ZYZ3ZyoWAeyMTnwKBgQDfjUJLMuw4pEzeqKq+qHn4XVsDcQrufi0ctLDwSZLB6rMGlY8dv3klLnQpUVtr6WhGNiRKdyyqvuYSOCX1tp51X641vrrxuYANA2zx4qJ+uWOQHL3b0l9IRLR+wMnJfJYpKJaJ0WUGAUIpTcGr2REWemUkQ4W++IXPY59ZnjO1gQKBgHFGyh8GmkQ4go5o6nWTcXWGkjkt8v5aU2gxEgYrwxaH6Y/fEliw7I0b+ZBhinLYTBajzRdb6mSYfSRnpiUaX12e0pe0eHu3ByuM+i9qpPV1EGq04Sq4E+cy1Ajc17zJK4/uRosvsy7km9XcPabnTmBz12sm3s6PplOofLgsin0dAoGBAPGOkZWmogiJYSumviovygIyw5vzVhWOiCiqXZNzDzT6O5pnFn8zTviYiIGFj0iGcdvQgZ2CevU4txjQ5PMPBkUeai/4vV7kIntXKDzSrsI3W4VuMqoSz7G0tGZwMVHLLCtYx3zKF3nejfTmbCeOTWIcyUFC7VSNlTiB9D/S8GsZ
Empty file.
1 change: 1 addition & 0 deletions pkg/repo/migrations/testdata/v2_empty_path/repo.version
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"Version":2}
27 changes: 27 additions & 0 deletions pkg/repo/migrations/testdata/v2_empty_path/user_id.pem
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
-----BEGIN RSA PRIVATE KEY-----
MIIEowIBAAKCAQEAz6o3HJWR4HKaXbREab3Poq4FPYwnQCjESFP9WvLMBOuLqt8C
zqLCIWSm+DHABRTM9TQcDFxSK6jcw8Zgwu5rRRi4NVX1j5sE2/0ry0rWZF5xAFV+
KXzILLZDJ6gyaOew/AcKDUuCcHN6xLpqdVQn6h704bstlzFM40/q4BRc5R7Cngxa
QPwRMxy32T09xO2N2bzr4lvPHQ7uJ8usWkl4fH2wkOPW0q3C9V1clN40LeIVdKqo
hITaU5BSUmLSvYAy0lPaB1U2EQ4Ibh5M9gtAvHAn1/IZxeE6YsjNvagctyuUFqvy
B/pY9aPpgt4mU0kFyZ817cc/61p0mPY7566pzwIDAQABAoIBAGqqtPpCvZZ7Z10g
Xk+uRrv6BQkTN4oWJpQ8SBPySOdyEeCoAmofYfYDHFy6OTai85AzvwB+X1q7twH8
oPEfxAvO4bfLuoGUD8/r4BXhD/SoKgUQ5Ofsh2moCSCSnaoS8G7UMs6XbZ3TQ0lP
s6UmoAX69rAIFCfoE+VixHNrIjgxQsYe06pie/LzGgXdGjlDQ1AW5XCKupvjqMKo
lv2VaHsJ7zKGd3mkuMUj/Yxe28AyKosXJVFxjbs+VaBIEVcCyo2pri4bsDEXdL/T
f0JegSj/MowPib4tPT4YPJUzdmLKaX/+ySeD1U3r7eUoz5SQHeiVDWUOUuXomikW
bMe+VwkCgYEA4+ZjjQkNWPXiSmi9a9wGmPUIkMPGT6yuXpORazQMnT67uGPuulkb
Y8ZPWtH7UIXXEtEypUFWP8MfqblBotBO84nd+GJNXu1RWb1wFdmD0L1KUXXBxXTc
qv4R2G47LuJInMxeL0APgMGfb3MhtYzmi93w1CjaS80svb3LK2mO9k0CgYEA6UUc
imDeMSMFkRsuIuH+SpTaRukDumL3yOTV7VvFvAkPmtyCS0v236E2Rs9c1rhi5pzq
dD/5Svla8LegjhqH98l3YXmjghTrAR3G5leNNo3Ls4tG3STdUucC8Kz02kFDkzNV
ylZ6ESB8krFVHcIbALumTkjyZiJ8lzMizn89posCgYBm4S1VkJ2O6EkEUC4aj9QG
nq8EMuzGMOzyUumVnISJQ7WYcrxlMS6VJwPRuvMZkaHao8eaJFmcJSiZBMFlj2Xk
uPsUUsQcs0NDIKqFopDH3Xu8NE2KJswrANDrhC+BZbNj3TozWFbnSAECXdX80COP
RlyeWebZy0CEUzZxXZClDQKBgCsySR624W5n9dfxGJINF3HoLqOC8tXJDel9tr/F
htBTPTiILPiUkJrI16w1KhP5AReDZ0cvsZJYqqZ42pp1/0e7XwgF5mloRjxmrzZo
cBXyHA6bc/vvn6yCauD2pkDkZQcVMoXNjb0uLuNbQ0/HagiwtxM4Husa58d4CLIE
jf4RAoGBAIKjUuj1d+nUnU5xeQ2W1GY/es5dBtcpO5s7rssHPXpDnHLbiIPTkdeY
10ttnPEryOBLismjgna1fRCc+xLdZa22tWfDV9UxwgeF1cPrIbtxwI1B0SdQ9Xck
bzLOD+ng7VMq9hKasr/+XF1NvDEmht0QNTHS4MDT2rL0aOCBQ69X
-----END RSA PRIVATE KEY-----
17 changes: 14 additions & 3 deletions pkg/repo/migrations/v2_3.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,16 @@ var V2Migration = repo.NewMigration(
doWrite = true
}

emptyConfig := types.JobStoreConfig{}
if fileCfg.Node.Compute.ExecutionStore == emptyConfig {
if fileCfg.Node.Compute.ExecutionStore.Path == "" {
// persist the execution store in the repo
executionStore := resolvedCfg.Node.Compute.ExecutionStore

// handle an edge case where config.yaml has store config entry but no path,
// which will override the resolved config path and make it empty as well
if executionStore.Path == "" {
executionStore.Path = filepath.Join(repoPath, "compute_store", "executions.db")
}

// if execution store already exist with nodeID, then rename it to the new name
legacyStoreName := filepath.Join(repoPath, libp2pNodeID+"-compute")
newStorePath := filepath.Dir(executionStore.Path)
Expand All @@ -67,10 +72,16 @@ var V2Migration = repo.NewMigration(
set(types.NodeComputeExecutionStore, executionStore)
}

if fileCfg.Node.Requester.JobStore == emptyConfig {
if fileCfg.Node.Requester.JobStore.Path == "" {
// persist the job store in the repo
jobStore := resolvedCfg.Node.Requester.JobStore

// handle an edge case where config.yaml has store config entry but no path,
// which will override the resolved config path and make it empty as well
if jobStore.Path == "" {
jobStore.Path = filepath.Join(repoPath, "orchestrator_store", "jobs.db")
}

// if job store already exist with nodeID, then rename it to the new name
legacyStoreName := filepath.Join(repoPath, libp2pNodeID+"-requester")
newStorePath := filepath.Dir(jobStore.Path)
Expand Down
30 changes: 30 additions & 0 deletions pkg/repo/migrations/v2_3_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,36 @@ func (suite *V2MigrationsTestSuite) TestV2MigrationWitCustomStores() {
suite.DirExists(filepath.Join(suite.TempDir, libp2pPeerID+"-requester"))
}

// TestV2MigrationWithEmptyStorePaths test that migration with store config exist, but with empty paths
func (suite *V2MigrationsTestSuite) TestV2MigrationWithEmptyStorePaths() {
libp2pPeerID := "QmUBgU7xHKK44RuTHgrvnJfoSdZJS4fddT197iyTF5qjEV"

// Copy test data to the suite's temporary directory
testDataPath := filepath.Join("testdata", "v2_empty_path")
suite.copyRepo(testDataPath)

// verify the repo's current state
suite.verifyInitialState(libp2pPeerID)

// open the repo to trigger the migration
suite.Require().NoError(suite.repo.Open())

repoVersion, err := suite.repo.Version()
suite.Require().NoError(err)
suite.Equal(expectedRepoVersion, repoVersion)

// verify configs where updated as expected
_, cfg, err := readConfig(*suite.repo)
suite.Require().NoError(err)
suite.Equal(filepath.Join(suite.TempDir, config.ComputeExecutionsStorePath), cfg.Node.Compute.ExecutionStore.Path)
suite.Equal(filepath.Join(suite.TempDir, config.OrchestratorJobStorePath), cfg.Node.Requester.JobStore.Path)
suite.Equal(libp2pPeerID, cfg.Node.Name)

// verify the old directories were renamed
suite.NoDirExists(filepath.Join(suite.TempDir, libp2pPeerID+"-compute"))
suite.NoDirExists(filepath.Join(suite.TempDir, libp2pPeerID+"-requester"))
}

func (suite *V2MigrationsTestSuite) verifyInitialState(nodeID string) {
repoVersion, err := suite.repo.Version()
suite.Require().NoError(err)
Expand Down

0 comments on commit d9d8de5

Please sign in to comment.