Skip to content

Commit

Permalink
Merge pull request #37868 from awsaxeman/f-aws-fsx-lustre_file_system…
Browse files Browse the repository at this point in the history 
…-metadata

[new feature] fsx lustre file system metadata
  • Loading branch information
@gdavison
gdavison committed Jun 7, 2024
2 parents 4110403 + 5a2755b commit 798d151
Show file tree
Hide file tree
Showing 6 changed files with 310 additions and 5 deletions.
3 changes: 3 additions & 0 deletions .changelog/37868.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:enhancement
resource/aws_fsx_lustre_file_system: Add `metadata_configuration` argument
```
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ require (
github.com/ProtonMail/go-crypto v1.1.0-alpha.2
github.com/YakDriver/go-version v0.1.0
github.com/YakDriver/regexache v0.23.0
github.com/aws/aws-sdk-go v1.53.17
github.com/aws/aws-sdk-go v1.53.18
github.com/aws/aws-sdk-go-v2 v1.27.1
github.com/aws/aws-sdk-go-v2/config v1.27.17
github.com/aws/aws-sdk-go-v2/credentials v1.17.17
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew
github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
github.com/aws/aws-sdk-go v1.53.17 h1:TwtYMzVBTaqPVj/pcemHRIgk01OycWEcEUyUUX0tpCI=
github.com/aws/aws-sdk-go v1.53.17/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/aws/aws-sdk-go v1.53.18 h1:BeMeCK5e3bDGJj675FhnO94zRci8O35ombWXRvYomJs=
github.com/aws/aws-sdk-go v1.53.18/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
github.com/aws/aws-sdk-go-v2 v1.27.1 h1:xypCL2owhog46iFxBKKpBcw+bPTX/RJzwNj8uSilENw=
github.com/aws/aws-sdk-go-v2 v1.27.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to=
Expand Down
134 changes: 132 additions & 2 deletions internal/service/fsx/lustre_file_system.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,28 @@ func resourceLustreFileSystem() *schema.Resource {
},
},
},
"metadata_configuration": {
Type: schema.TypeList,
Optional: true,
Computed: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
names.AttrMode: {
Type: schema.TypeString,
Optional: true,
Computed: true,
ValidateDiagFunc: validation.ToDiagFunc(validation.StringInSlice(fsx.MetadataConfigurationMode_Values(), false)),
},
names.AttrIOPS: {
Type: schema.TypeInt,
Optional: true,
Computed: true,
ValidateDiagFunc: validation.ToDiagFunc(validation.IntInSlice([]int{1500, 3000, 6000, 12000, 24000, 36000, 48000, 60000, 72000, 84000, 96000, 108000, 120000, 132000, 144000, 156000, 168000, 180000, 192000})),
},
},
},
},
"mount_name": {
Type: schema.TypeString,
Computed: true,
Expand Down Expand Up @@ -277,12 +299,13 @@ func resourceLustreFileSystem() *schema.Resource {

CustomizeDiff: customdiff.Sequence(
verify.SetTagsDiff,
resourceLustreFileSystemSchemaCustomizeDiff,
resourceLustreFileSystemStorageCapacityCustomizeDiff,
resourceLustreFileSystemMetadataConfigCustomizeDiff,
),
}
}

func resourceLustreFileSystemSchemaCustomizeDiff(_ context.Context, d *schema.ResourceDiff, meta interface{}) error {
func resourceLustreFileSystemStorageCapacityCustomizeDiff(_ context.Context, d *schema.ResourceDiff, meta any) error {
// we want to force a new resource if the new storage capacity is less than the old one
if d.HasChange("storage_capacity") {
o, n := d.GetChange("storage_capacity")
Expand All @@ -296,6 +319,51 @@ func resourceLustreFileSystemSchemaCustomizeDiff(_ context.Context, d *schema.Re
return nil
}

func resourceLustreFileSystemMetadataConfigCustomizeDiff(_ context.Context, d *schema.ResourceDiff, meta any) error {
//metadata_configuration is only supported when deployment_type is persistent2
if v, ok := d.GetOk("metadata_configuration"); ok {
if len(v.([]any)) > 0 {
deploymentType := d.Get("deployment_type").(string)
if deploymentType != fsx.LustreDeploymentTypePersistent2 {
return fmt.Errorf("metadata_configuration can only be set when deployment type is " + fsx.LustreDeploymentTypePersistent2)
}
}
}

// we want to force a new resource if the new Iops is less than the old one
if d.HasChange("metadata_configuration") {
if v, ok := d.GetOk("metadata_configuration"); ok && len(v.([]interface{})) > 0 && v.([]interface{})[0] != nil {
if mode := d.Get("metadata_configuration.0.mode"); mode == fsx.MetadataConfigurationModeUserProvisioned {
o, n := d.GetChange("metadata_configuration")

oldV := o.([]interface{})
newV := n.([]interface{})
var metaOld map[string]interface{}
var metaNew map[string]interface{}

for _, v := range oldV {
metaOld = v.(map[string]interface{})
}

for _, v := range newV {
metaNew = v.(map[string]interface{})
}

if len(metaNew) > 0 && len(metaOld) > 0 {
if metaNew[names.AttrIOPS].(int) < metaOld[names.AttrIOPS].(int) {
log.Printf("[DEBUG] Forcing new due to metadata iops decrease. old iops: %d new iops: %d", metaOld[names.AttrIOPS].(int), metaNew[names.AttrIOPS].(int))
if err := d.ForceNew("metadata_configuration.0.iops"); err != nil {
return err
}
}
}
}
}
}

return nil
}

func resourceLustreFileSystemCreate(ctx context.Context, d *schema.ResourceData, meta interface{}) diag.Diagnostics {
var diags diag.Diagnostics
conn := meta.(*conns.AWSClient).FSxConn(ctx)
Expand Down Expand Up @@ -382,6 +450,11 @@ func resourceLustreFileSystemCreate(ctx context.Context, d *schema.ResourceData,
inputB.LustreConfiguration.LogConfiguration = expandLustreLogCreateConfiguration(v.([]interface{}))
}

if v, ok := d.GetOk("metadata_configuration"); ok && len(v.([]interface{})) > 0 {
inputC.LustreConfiguration.MetadataConfiguration = expandLustreMetadataCreateConfiguration(v.([]interface{}))
inputB.LustreConfiguration.MetadataConfiguration = expandLustreMetadataCreateConfiguration(v.([]interface{}))
}

if v, ok := d.GetOk("per_unit_storage_throughput"); ok {
inputC.LustreConfiguration.PerUnitStorageThroughput = aws.Int64(int64(v.(int)))
inputB.LustreConfiguration.PerUnitStorageThroughput = aws.Int64(int64(v.(int)))
Expand Down Expand Up @@ -469,6 +542,9 @@ func resourceLustreFileSystemRead(ctx context.Context, d *schema.ResourceData, m
if err := d.Set("log_configuration", flattenLustreLogConfiguration(lustreConfig.LogConfiguration)); err != nil {
return sdkdiag.AppendErrorf(diags, "setting log_configuration: %s", err)
}
if err := d.Set("metadata_configuration", flattenLustreMetadataConfiguration(lustreConfig.MetadataConfiguration)); err != nil {
return sdkdiag.AppendErrorf(diags, "setting metadata_configuration: %s", err)
}
d.Set("mount_name", lustreConfig.MountName)
d.Set("network_interface_ids", aws.StringValueSlice(filesystem.NetworkInterfaceIds))
d.Set(names.AttrOwnerID, filesystem.OwnerId)
Expand Down Expand Up @@ -518,6 +594,10 @@ func resourceLustreFileSystemUpdate(ctx context.Context, d *schema.ResourceData,
input.LustreConfiguration.LogConfiguration = expandLustreLogCreateConfiguration(d.Get("log_configuration").([]interface{}))
}

if d.HasChange("metadata_configuration") {
input.LustreConfiguration.MetadataConfiguration = expandLustreMetadataUpdateConfiguration(d.Get("metadata_configuration").([]interface{}))
}

if d.HasChange("per_unit_storage_throughput") {
input.LustreConfiguration.PerUnitStorageThroughput = aws.Int64(int64(d.Get("per_unit_storage_throughput").(int)))
}
Expand Down Expand Up @@ -647,6 +727,56 @@ func flattenLustreLogConfiguration(adopts *fsx.LustreLogConfiguration) []map[str
return []map[string]interface{}{m}
}

func expandLustreMetadataCreateConfiguration(l []interface{}) *fsx.CreateFileSystemLustreMetadataConfiguration {
if len(l) == 0 || l[0] == nil {
return nil
}

data := l[0].(map[string]interface{})
req := &fsx.CreateFileSystemLustreMetadataConfiguration{
Mode: aws.String(data[names.AttrMode].(string)),
}

if v, ok := data[names.AttrIOPS].(int); ok && v != 0 {
req.Iops = aws.Int64(int64(v))
}

return req
}

func expandLustreMetadataUpdateConfiguration(l []interface{}) *fsx.UpdateFileSystemLustreMetadataConfiguration {
if len(l) == 0 || l[0] == nil {
return nil
}

data := l[0].(map[string]interface{})
req := &fsx.UpdateFileSystemLustreMetadataConfiguration{
Mode: aws.String(data[names.AttrMode].(string)),
}

if v, ok := data[names.AttrIOPS].(int); ok && v != 0 {
req.Iops = aws.Int64(int64(v))
}

return req
}

func flattenLustreMetadataConfiguration(adopts *fsx.FileSystemLustreMetadataConfiguration) []map[string]interface{} {
if adopts == nil {
return []map[string]interface{}{}
}

m := map[string]interface{}{
names.AttrMode: aws.StringValue(adopts.Mode),
}

if adopts.Iops != nil {
m[names.AttrIOPS] = aws.Int64Value(adopts.Iops)
}

return []map[string]interface{}{m}
}

func logStateFunc(v interface{}) string {
value := v.(string)
// API returns the specific log stream arn instead of provided log group
Expand Down
164 changes: 164 additions & 0 deletions internal/service/fsx/lustre_file_system_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ func TestAccFSxLustreFileSystem_basic(t *testing.T) {
resource.TestCheckResourceAttr(resourceName, "imported_file_chunk_size", acctest.Ct0),
resource.TestCheckResourceAttr(resourceName, "log_configuration.#", acctest.Ct1),
resource.TestCheckResourceAttr(resourceName, "log_configuration.0.level", "DISABLED"),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct0),
resource.TestCheckResourceAttrSet(resourceName, "mount_name"),
resource.TestCheckResourceAttr(resourceName, "network_interface_ids.#", acctest.Ct2),
acctest.CheckResourceAttrAccountID(resourceName, names.AttrOwnerID),
Expand Down Expand Up @@ -768,6 +769,130 @@ func TestAccFSxLustreFileSystem_logConfig(t *testing.T) {
})
}

func TestAccFSxLustreFileSystem_metadataConfig(t *testing.T) {
ctx := acctest.Context(t)
var filesystem1, filesystem2 fsx.FileSystem
resourceName := "aws_fsx_lustre_file_system.test"
rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix)

resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { acctest.PreCheck(ctx, t); acctest.PreCheckPartitionHasService(t, fsx.EndpointsID) },
ErrorCheck: acctest.ErrorCheck(t, names.FSxServiceID),
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories,
CheckDestroy: testAccCheckLustreFileSystemDestroy(ctx),
Steps: []resource.TestStep{
{
Config: testAccLustreFileSystemConfig_metadata(rName, "AUTOMATIC"),
Check: resource.ComposeTestCheckFunc(
testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem1),
testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem1),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct1),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.mode", "AUTOMATIC"),
resource.TestCheckResourceAttrSet(resourceName, "metadata_configuration.0.iops"),
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{names.AttrSecurityGroupIDs},
},
{
Config: testAccLustreFileSystemConfig_metadata_iops(rName, "USER_PROVISIONED", 1500),
Check: resource.ComposeTestCheckFunc(
testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem2),
testAccCheckLustreFileSystemNotRecreated(&filesystem1, &filesystem2),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct1),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.mode", "USER_PROVISIONED"),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.iops", "1500"),
),
},
},
})
}

func TestAccFSxLustreFileSystem_metadataConfig_increase(t *testing.T) {
ctx := acctest.Context(t)
var filesystem1, filesystem2 fsx.FileSystem
resourceName := "aws_fsx_lustre_file_system.test"
rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix)

resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { acctest.PreCheck(ctx, t); acctest.PreCheckPartitionHasService(t, fsx.EndpointsID) },
ErrorCheck: acctest.ErrorCheck(t, names.FSxServiceID),
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories,
CheckDestroy: testAccCheckLustreFileSystemDestroy(ctx),
Steps: []resource.TestStep{
{
Config: testAccLustreFileSystemConfig_metadata_iops(rName, "USER_PROVISIONED", 1500),
Check: resource.ComposeTestCheckFunc(
testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem1),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct1),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.mode", "USER_PROVISIONED"),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.iops", "1500"),
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{names.AttrSecurityGroupIDs},
},
{
Config: testAccLustreFileSystemConfig_metadata_iops(rName, "USER_PROVISIONED", 3000),
Check: resource.ComposeTestCheckFunc(
testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem2),
testAccCheckLustreFileSystemNotRecreated(&filesystem1, &filesystem2),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct1),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.mode", "USER_PROVISIONED"),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.iops", "3000"),
),
},
},
})
}

func TestAccFSxLustreFileSystem_metadataConfig_decrease(t *testing.T) {
ctx := acctest.Context(t)
var filesystem1, filesystem2 fsx.FileSystem
resourceName := "aws_fsx_lustre_file_system.test"
rName := sdkacctest.RandomWithPrefix(acctest.ResourcePrefix)

resource.ParallelTest(t, resource.TestCase{
PreCheck: func() { acctest.PreCheck(ctx, t); acctest.PreCheckPartitionHasService(t, fsx.EndpointsID) },
ErrorCheck: acctest.ErrorCheck(t, names.FSxServiceID),
ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories,
CheckDestroy: testAccCheckLustreFileSystemDestroy(ctx),
Steps: []resource.TestStep{
{
Config: testAccLustreFileSystemConfig_metadata_iops(rName, "USER_PROVISIONED", 3000),
Check: resource.ComposeTestCheckFunc(
testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem1),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct1),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.mode", "USER_PROVISIONED"),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.iops", "3000"),
),
},
{
ResourceName: resourceName,
ImportState: true,
ImportStateVerify: true,
ImportStateVerifyIgnore: []string{names.AttrSecurityGroupIDs},
},
{
Config: testAccLustreFileSystemConfig_metadata_iops(rName, "USER_PROVISIONED", 1500),
Check: resource.ComposeTestCheckFunc(
testAccCheckLustreFileSystemExists(ctx, resourceName, &filesystem2),
testAccCheckLustreFileSystemRecreated(&filesystem1, &filesystem2),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.#", acctest.Ct1),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.mode", "USER_PROVISIONED"),
resource.TestCheckResourceAttr(resourceName, "metadata_configuration.0.iops", "1500"),
),
},
},
})
}

func TestAccFSxLustreFileSystem_rootSquashConfig(t *testing.T) {
ctx := acctest.Context(t)
var filesystem fsx.FileSystem
Expand Down Expand Up @@ -1608,6 +1733,45 @@ resource "aws_fsx_lustre_file_system" "test" {
`, rName, status))
}

func testAccLustreFileSystemConfig_metadata(rName, mode string) string {
return acctest.ConfigCompose(testAccLustreFileSystemConfig_base(rName), fmt.Sprintf(`
resource "aws_fsx_lustre_file_system" "test" {
storage_capacity = 1200
subnet_ids = aws_subnet.test[*].id
deployment_type = "PERSISTENT_2"
per_unit_storage_throughput = 125
metadata_configuration {
mode = %[2]q
}
tags = {
Name = %[1]q
}
}
`, rName, mode))
}

func testAccLustreFileSystemConfig_metadata_iops(rName, mode string, iops int) string {
return acctest.ConfigCompose(testAccLustreFileSystemConfig_base(rName), fmt.Sprintf(`
resource "aws_fsx_lustre_file_system" "test" {
storage_capacity = 1200
subnet_ids = aws_subnet.test[*].id
deployment_type = "PERSISTENT_2"
per_unit_storage_throughput = 125
metadata_configuration {
mode = %[2]q
iops = %[3]d
}
tags = {
Name = %[1]q
}
}
`, rName, mode, iops))
}

func testAccLustreFileSystemConfig_rootSquash(rName, uid string) string {
return acctest.ConfigCompose(testAccLustreFileSystemConfig_base(rName), fmt.Sprintf(`
resource "aws_fsx_lustre_file_system" "test" {
Expand Down
Loading

0 comments on commit 798d151

Please sign in to comment.