Add support for S3 package sync

src/SquirrelCli/Options.cs

@@ -206,11 +206,47 @@ public override void Validate()
         }
     }
 
+    internal class SyncS3Options : BaseOptions
+    {
+        public string key { get; private set; }
+        public string secret { get; private set; }
+        public string region { get; private set; }
+        public string endpointUrl { get; private set; }
+        public string bucket { get; private set; }
+        public string pathPrefix { get; private set; }
+        public bool overwrite { get; private set; }
+
+        public SyncS3Options() 
+        {
+            Add("key=", "Authentication {IDENTIFIER} or access key", v => key = v);
+            Add("secret=", "Authentication secret {KEY}", v => secret = v);
+            Add("region=", "AWS service {REGION} (eg. us-west-1)", v => region = v);
+            Add("endpointUrl=", "Custom service {URL} (from backblaze, digital ocean, etc)", v => endpointUrl = v);
+            Add("bucket=", "{NAME} of the S3 bucket to access", v => bucket = v);
+            Add("pathPrefix=", "A sub-folder {PATH} to read and write files in", v => pathPrefix = v);
+            Add("overwrite", "Replace any mismatched remote files with files in local directory", v => overwrite = true);
+        }
+
+        public override void Validate()
+        {
+            IsRequired(nameof(secret), nameof(key), nameof(bucket));
+            IsValidUrl(nameof(endpointUrl));
+
+            if ((region == null) == (endpointUrl == null)) {
+                throw new OptionValidationException("One of 'region' and 'endpoint' arguments is required and are also mutually exclusive. Specify one of these. ");
+            }
+
+            if (region != null) {
+                var r = Amazon.RegionEndpoint.GetBySystemName(region);
+                if (r.DisplayName == "Unknown")
+                    Log.Warn($"Region '{region}' lookup failed, is this a valid AWS region?");
+            }
+        }
+    }
+
     internal class SyncHttpOptions : BaseOptions
     {
         public string url { get; private set; }
-        public string token { get; private set; }
-
         public SyncHttpOptions()
         {
             Add("url=", "Base url to the http location with hosted releases", v => url = v);

src/SquirrelCli/Program.cs

@@ -49,6 +49,8 @@ public static int Main(string[] args)
                 //{ "http-up", "sync", new SyncHttpOptions(), o => new SimpleWebRepository(o).UploadMissingPackages().Wait() },
                 { "github-down", "Download recent releases from GitHub", new SyncGithubOptions(), o => new GitHubRepository(o).DownloadRecentPackages().Wait() },
                 //{ "github-up", "sync", new SyncGithubOptions(), o => new GitHubRepository(o).UploadMissingPackages().Wait() },
+                { "s3-down", "Download recent releases from a S3 bucket", new SyncS3Options(), o => new S3Repository(o).DownloadRecentPackages().Wait() },
+                { "s3-up", "Upload recent releases to a S3 bucket", new SyncS3Options(), o => new S3Repository(o).UploadMissingPackages().Wait() },
                 //"",
                 //"Examples:",
                 //$"    {exeName} pack ",

src/SquirrelCli/SquirrelCli.csproj

@@ -20,6 +20,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="AWSSDK.S3" Version="3.7.7.17" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
     <PackageReference Include="System.IO" Version="4.3.0" />
     <PackageReference Include="System.Net.Http" Version="4.3.4" />

src/SquirrelCli/Sync/S3Repository.cs

@@ -0,0 +1,131 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Amazon;
+using Amazon.S3;
+using Amazon.S3.Model;
+using Squirrel;
+using Squirrel.SimpleSplat;
+
+namespace SquirrelCli.Sources
+{
+    internal class S3Repository : IPackageRepository
+    {
+        private readonly SyncS3Options _options;
+        private readonly AmazonS3Client _client;
+        private readonly string _prefix;
+
+        private readonly static IFullLogger Log = SquirrelLocator.Current.GetService<ILogManager>().GetLogger(typeof(S3Repository));
+
+        public S3Repository(SyncS3Options options)
+        {
+            _options = options;
+            if (options.region != null) {
+                var r = RegionEndpoint.GetBySystemName(options.region);
+                _client = new AmazonS3Client(_options.key, _options.secret, r);
+            } else if (options.endpointUrl != null) {
+                var config = new AmazonS3Config() { ServiceURL = _options.endpointUrl };
+                _client = new AmazonS3Client(_options.key, _options.secret, config);
+            } else {
+                throw new InvalidOperationException("Missing endpoint");
+            }
+
+            var prefix = _options.pathPrefix?.Replace('\\', '/') ?? "";
+            if (!String.IsNullOrWhiteSpace(prefix) && !prefix.EndsWith("/")) prefix += "/";
+            _prefix = prefix;
+        }
+
+        public async Task DownloadRecentPackages()
+        {
+            var releasesDir = new DirectoryInfo(_options.releaseDir);
+            if (!releasesDir.Exists)
+                releasesDir.Create();
+
+            var releasesPath = Path.Combine(releasesDir.FullName, "RELEASES");
+            Log.Info("Downloading RELEASES");
+            using (var obj = await _client.GetObjectAsync(_options.bucket, _prefix + "RELEASES"))
+                await obj.WriteResponseStreamToFileAsync(releasesPath, false, CancellationToken.None);
+
+            var releasesToDownload = ReleaseEntry.ParseReleaseFile(File.ReadAllText(releasesPath))
+                .Where(x => !x.IsDelta)
+                .OrderByDescending(x => x.Version)
+                .Take(1)
+                .Select(x => new {
+                    LocalPath = Path.Combine(releasesDir.FullName, x.Filename),
+                    Filename = x.Filename,
+                });
+
+            foreach (var releaseToDownload in releasesToDownload) {
+                Log.Info("Downloading " + releaseToDownload.Filename);
+                using (var pkgobj = await _client.GetObjectAsync(_options.bucket, _prefix + releaseToDownload.Filename))
+                    await pkgobj.WriteResponseStreamToFileAsync(releaseToDownload.LocalPath, false, CancellationToken.None);
+            }
+        }
+
+        public async Task UploadMissingPackages()
+        {
+            var releasesDir = new DirectoryInfo(_options.releaseDir);
+
+            var files = releasesDir.GetFiles();
+            var setupFile = files.Where(f => f.FullName.EndsWith("Setup.exe")).SingleOrDefault();
+            var releasesFile = files.Where(f => f.Name == "RELEASES").SingleOrDefault();
+            var filesWithoutSpecial = files.Except(new[] { setupFile, releasesFile });
+
+            foreach (var f in filesWithoutSpecial) {
+                string key = _prefix + f.Name;
+                string deleteOldVersionId = null;
+
+                try {
+                    var metadata = await _client.GetObjectMetadataAsync(_options.bucket, key);
+                    var md5 = GetFileMD5Checksum(f.FullName);
+                    var stored = metadata?.ETag?.Trim().Trim('"');
+
+                    if (stored != null) {
+                        if (stored.Equals(md5, StringComparison.InvariantCultureIgnoreCase)) {
+                            Log.Info($"Skipping '{f.FullName}', matching file exists in remote.");
+                            continue;
+                        } else if (_options.overwrite) {
+                            Log.Info($"File '{f.FullName}' exists in remote, replacing...");
+                            deleteOldVersionId = metadata.VersionId;
+                        } else {
+                            Log.Warn($"File '{f.FullName}' exists in remote and checksum does not match. Use 'overwrite' argument to replace remote file.");
+                            continue;
+                        }
+                    }
+                } catch (AmazonS3Exception ex) when (ex.StatusCode == HttpStatusCode.NotFound) {
+                    // we don't care if the file does not exist, we're uploading!
+                }
+
+                var req = new PutObjectRequest {
+                    BucketName = _options.bucket,
+                    FilePath = f.FullName,
+                    Key = key,
+                };
+
+                Log.Info("Uploading " + f.Name);
+                var resp = await _client.PutObjectAsync(req);
+                if ((int) resp.HttpStatusCode >= 300 || (int) resp.HttpStatusCode < 200)
+                    throw new Exception("Failed to upload with status code " + resp.HttpStatusCode);
+
+                if (deleteOldVersionId != null) {
+                    Log.Info("Deleting old version of " + f.Name);
+                    await _client.DeleteObjectAsync(_options.bucket, key, deleteOldVersionId);
+                }
+            }
+        }
+
+        private static string GetFileMD5Checksum(string filePath)
+        {
+            var sha = System.Security.Cryptography.MD5.Create();
+            byte[] checksum;
+            using (var fs = File.OpenRead(filePath))
+                checksum = sha.ComputeHash(fs);
+            return BitConverter.ToString(checksum).Replace("-", String.Empty);
+        }
+    }
+}