-
Notifications
You must be signed in to change notification settings - Fork 86
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
compute nodes no longer reject bids based on queued capacity (#4002)
This PR not only closes #3992, but also adds a node ranker based on available and queued capacity, which we didn't have before. I had to introduce the ranker along with this PR to make sure some tests are passing that were relying on bid rejection due to exceeding resource limits, specifically `TestParallelGPU`
- Loading branch information
Showing
29 changed files
with
701 additions
and
275 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
//go:build unit || !integration | ||
|
||
package capacity | ||
|
||
import ( | ||
"context" | ||
"testing" | ||
|
||
"github.com/bacalhau-project/bacalhau/pkg/models" | ||
"github.com/stretchr/testify/suite" | ||
) | ||
|
||
type LocalUsageTrackerTestSuite struct { | ||
suite.Suite | ||
tracker *LocalUsageTracker | ||
} | ||
|
||
func (s *LocalUsageTrackerTestSuite) SetupTest() { | ||
s.tracker = NewLocalUsageTracker() | ||
} | ||
|
||
func (s *LocalUsageTrackerTestSuite) TestAddAndRemove() { | ||
ctx := context.Background() | ||
usage := models.Resources{CPU: 2, Memory: 1024, Disk: 10000, GPU: 1} | ||
|
||
// Test Add | ||
s.tracker.Add(ctx, usage) | ||
|
||
usedCapacity := s.tracker.GetUsedCapacity(ctx) | ||
s.Require().Equal(usage, usedCapacity) | ||
|
||
// Test Remove | ||
s.tracker.Remove(ctx, usage) | ||
|
||
usedCapacity = s.tracker.GetUsedCapacity(ctx) | ||
s.Require().True(usedCapacity.IsZero()) | ||
} | ||
|
||
func TestLocalUsageTrackerTestSuite(t *testing.T) { | ||
suite.Run(t, new(LocalUsageTrackerTestSuite)) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
package capacity | ||
|
||
import ( | ||
"context" | ||
"sync" | ||
|
||
"github.com/bacalhau-project/bacalhau/pkg/models" | ||
) | ||
|
||
// LocalUsageTracker keeps track of the resources used regardless of the total capacity. | ||
// It is useful when tracking jobs in the queue pending and haven't started yet. | ||
type LocalUsageTracker struct { | ||
usedCapacity models.Resources | ||
mu sync.Mutex | ||
} | ||
|
||
func NewLocalUsageTracker() *LocalUsageTracker { | ||
return &LocalUsageTracker{} | ||
} | ||
|
||
func (t *LocalUsageTracker) Add(ctx context.Context, usage models.Resources) { | ||
t.mu.Lock() | ||
defer t.mu.Unlock() | ||
t.usedCapacity = *t.usedCapacity.Add(usage) | ||
} | ||
|
||
func (t *LocalUsageTracker) Remove(ctx context.Context, usage models.Resources) { | ||
t.mu.Lock() | ||
defer t.mu.Unlock() | ||
t.usedCapacity = *t.usedCapacity.Sub(usage) | ||
} | ||
|
||
func (t *LocalUsageTracker) GetUsedCapacity(ctx context.Context) models.Resources { | ||
return t.usedCapacity | ||
} | ||
|
||
// compile-time check that LocalUsageTracker implements Tracker | ||
var _ UsageTracker = (*LocalUsageTracker)(nil) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.