Skip to content
This repository has been archived by the owner on Jul 5, 2024. It is now read-only.

Commit

Permalink
Do not zip/unzip needlessly when applying delta's
Browse files Browse the repository at this point in the history
  • Loading branch information
@caesay
caesay committed Jun 8, 2022
1 parent 8fe1988 commit b3a2eca
Show file tree
Hide file tree
Showing 10 changed files with 186 additions and 164 deletions.
46 changes: 0 additions & 46 deletions src/Squirrel/Internal/ApplyReleasesProgress.cs
View file

This file was deleted.

61 changes: 60 additions & 1 deletion src/Squirrel/Internal/DeltaPackage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ public DeltaPackage(string baseTempDir = null)
{
_baseTempDir = baseTempDir ?? Utility.GetDefaultTempBaseDirectory();
}

public string ApplyDeltaPackage(string basePackageZip, string deltaPackageZip, string outputFile, Action<int> progress = null)
{
progress = progress ?? (x => { });
Expand Down Expand Up @@ -84,6 +84,64 @@ public string ApplyDeltaPackage(string basePackageZip, string deltaPackageZip, s
return outputFile;
}

public void ApplyDeltaPackageFast(string workingPath, string deltaPackageZip, Action<int> progress = null)
{
progress = progress ?? (x => { });

Contract.Requires(deltaPackageZip != null);

using var _1 = Utility.GetTempDirectory(out var deltaPath, _baseTempDir);
EasyZip.ExtractZipToDirectory(deltaPackageZip, deltaPath);
progress(10);

var pathsVisited = new List<string>();

var deltaPathRelativePaths = new DirectoryInfo(deltaPath).GetAllFilesRecursively()
.Select(x => x.FullName.Replace(deltaPath + Path.DirectorySeparatorChar, ""))
.ToArray();

// Apply all of the .diff files
var files = deltaPathRelativePaths
.Where(x => x.StartsWith("lib", StringComparison.InvariantCultureIgnoreCase))
.Where(x => !x.EndsWith(".shasum", StringComparison.InvariantCultureIgnoreCase))
.Where(x => !x.EndsWith(".diff", StringComparison.InvariantCultureIgnoreCase) ||
!deltaPathRelativePaths.Contains(x.Replace(".diff", ".bsdiff")))
.ToArray();

for (var index = 0; index < files.Length; index++) {
var file = files[index];
pathsVisited.Add(Regex.Replace(file, @"\.(bs)?diff$", "").ToLowerInvariant());
applyDiffToFile(deltaPath, file, workingPath);
var perc = (index + 1) / (double)files.Length * 100;
Utility.CalculateProgress((int)perc, 10, 90);
}

progress(90);

// Delete all of the files that were in the old package but
// not in the new one.
new DirectoryInfo(workingPath).GetAllFilesRecursively()
.Select(x => x.FullName.Replace(workingPath + Path.DirectorySeparatorChar, "").ToLowerInvariant())
.Where(x => x.StartsWith("lib", StringComparison.InvariantCultureIgnoreCase) && !pathsVisited.Contains(x))
.ForEach(x => {
this.Log().Info("{0} was in old package but not in new one, deleting", x);
File.Delete(Path.Combine(workingPath, x));
});

progress(95);

// Update all the files that aren't in 'lib' with the delta
// package's versions (i.e. the nuspec file, etc etc).
deltaPathRelativePaths
.Where(x => !x.StartsWith("lib", StringComparison.InvariantCultureIgnoreCase))
.ForEach(x => {
this.Log().Info("Updating metadata file: {0}", x);
File.Copy(Path.Combine(deltaPath, x), Path.Combine(workingPath, x), true);
});

progress(100);
}

void applyDiffToFile(string deltaPath, string relativeFilePath, string workingDirectory)
{
var inputFile = Path.Combine(deltaPath, relativeFilePath);
Expand Down Expand Up @@ -113,6 +171,7 @@ void applyDiffToFile(string deltaPath, string relativeFilePath, string workingDi
} else {
throw new InvalidOperationException("msdiff is not supported on non-windows platforms.");
}

verifyPatchedFile(relativeFilePath, inputFile, tempTargetFile);
} else {
using (var of = File.OpenWrite(tempTargetFile))
Expand Down
6 changes: 3 additions & 3 deletions src/Squirrel/Internal/EasyZip.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,17 +32,17 @@ public static void ExtractZipToDirectory(string inputFile, string outputDirector
});
}

public static void CreateZipFromDirectory(string outputFile, string directoryToCompress, bool nestDirectory = false)
public static void CreateZipFromDirectory(string outputFile, string directoryToCompress, bool nestDirectory = false, CompressionLevel level = CompressionLevel.BestSpeed)
{
Log.Info($"Compressing '{directoryToCompress}' to '{outputFile}' using SharpCompress (DEFLATE)...");
using var archive = ZipArchive.Create();
archive.DeflateCompressionLevel = CompressionLevel.BestSpeed;
archive.DeflateCompressionLevel = level;
if (nestDirectory) {
AddAllFromDirectoryInNestedDir(archive, directoryToCompress);
} else {
archive.AddAllFromDirectory(directoryToCompress);
}
archive.SaveTo(outputFile, CompressionType.Deflate);
archive.SaveTo(outputFile, level == CompressionLevel.None ? CompressionType.None : CompressionType.Deflate);
}

private static void AddAllFromDirectoryInNestedDir(
Expand Down
21 changes: 21 additions & 0 deletions src/Squirrel/Internal/Utility.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,27 @@ namespace Squirrel
{
internal static class Utility
{
/// <summary>
/// Calculates the total percentage of a specific step that should report within a specific range.
/// <para />
/// If a step needs to report between 50 -> 75 %, this method should be used as CalculateProgress(percentage, 50, 75).
/// </summary>
/// <param name="percentageOfCurrentStep">The percentage of the current step, a value between 0 and 100.</param>
/// <param name="stepStartPercentage">The start percentage of the range the current step represents.</param>
/// <param name="stepEndPercentage">The end percentage of the range the current step represents.</param>
/// <returns>The calculated percentage that can be reported about the total progress.</returns>
public static int CalculateProgress(int percentageOfCurrentStep, int stepStartPercentage, int stepEndPercentage)
{
// Ensure we are between 0 and 100
percentageOfCurrentStep = Math.Max(Math.Min(percentageOfCurrentStep, 100), 0);

var range = stepEndPercentage - stepStartPercentage;
var singleValue = range / 100d;
var totalPercentage = (singleValue * percentageOfCurrentStep) + stepStartPercentage;

return (int) totalPercentage;
}

public static string RemoveByteOrderMarkerIfPresent(string content)
{
return string.IsNullOrEmpty(content)
Expand Down
99 changes: 54 additions & 45 deletions src/Squirrel/UpdateManager.ApplyReleases.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using Squirrel.SimpleSplat;
using System.Threading;
using System.Runtime.Versioning;
using SharpCompress.Compressors.Deflate;
using Squirrel.NuGet;

namespace Squirrel
Expand All @@ -25,7 +26,10 @@ protected async Task<string> ApplyReleases(UpdateInfo updateInfo, bool silentIns

// Progress range: 00 -> 60
// takes the latest local package, applies a series of delta to get the full package to update to
var release = await createFullPackagesFromDeltas(updateInfo.ReleasesToApply, updateInfo.LatestLocalReleaseEntry, new ApplyReleasesProgress(updateInfo.ReleasesToApply.Count, x => progress(CalculateProgress(x, 0, 60)))).ConfigureAwait(false);
var release = await Task.Run(() => {
return createFullPackagesFromDeltas(updateInfo.ReleasesToApply, updateInfo.LatestLocalReleaseEntry,
x => progress(Utility.CalculateProgress(x, 0, 60)));
}).ConfigureAwait(false);

progress(60);

Expand All @@ -40,7 +44,7 @@ protected async Task<string> ApplyReleases(UpdateInfo updateInfo, bool silentIns

// Progress range: 60 -> 80
// extracts the new package to a version dir (app-{ver}) inside VersionStagingDir
var newVersionDir = await this.ErrorIfThrows(() => installPackageToStagingDir(updateInfo, release, x => progress(CalculateProgress(x, 60, 80))),
var newVersionDir = await this.ErrorIfThrows(() => installPackageToStagingDir(updateInfo, release, x => progress(Utility.CalculateProgress(x, 60, 80))),
"Failed to install package to app dir").ConfigureAwait(false);

progress(80);
Expand Down Expand Up @@ -156,57 +160,61 @@ await ZipPackage.ExtractZipReleaseForInstall(
});
}

async Task<ReleaseEntry> createFullPackagesFromDeltas(IEnumerable<ReleaseEntry> releasesToApply, ReleaseEntry currentVersion, ApplyReleasesProgress progress)
internal ReleaseEntry createFullPackagesFromDeltas(IEnumerable<ReleaseEntry> releasesToApply, ReleaseEntry currentVersion, Action<int> progress = null)
{
Contract.Requires(releasesToApply != null);

progress = progress ?? new ApplyReleasesProgress(releasesToApply.Count(), x => { });
var releases = releasesToApply?.ToArray() ?? new ReleaseEntry[0];
progress ??= (x => { });

// If there are no remote releases at all, bail
if (!releasesToApply.Any()) {
if (!releases.Any()) {
return null;
}

// If there are no deltas in our list, we're already done
if (releasesToApply.All(x => !x.IsDelta)) {
return releasesToApply.MaxBy(x => x.Version).FirstOrDefault();
if (releases.All(x => !x.IsDelta)) {
return releases.MaxBy(x => x.Version).FirstOrDefault();
}

if (!releasesToApply.All(x => x.IsDelta)) {
if (!releases.All(x => x.IsDelta)) {
throw new Exception("Cannot apply combinations of delta and full packages");
}

// Progress calculation is "complex" here. We need to known how many releases, and then give each release a similar amount of
// progress. For example, when applying 5 releases:
//
// release 1: 00 => 20
// release 2: 20 => 40
// release 3: 40 => 60
// release 4: 60 => 80
// release 5: 80 => 100
//

// Smash together our base full package and the nearest delta
var outputPackageZip = await Task.Run(() => {
var basePkg = Path.Combine(_config.PackagesDir, currentVersion.Filename);
var deltaPkg = Path.Combine(_config.PackagesDir, releasesToApply.First().Filename);
return new DeltaPackage(_config.AppTempDir).ApplyDeltaPackage(basePkg, deltaPkg,
Regex.Replace(deltaPkg, @"-delta.nupkg$", ".nupkg", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant),
x => progress.ReportReleaseProgress(x));
}).ConfigureAwait(false);
using var _1 = Utility.GetTempDirectory(out var workingPath, _config.AppTempDir);

// extract the base package (this version) to a directory
var basePkg = Path.Combine(_config.PackagesDir, currentVersion.Filename);
EasyZip.ExtractZipToDirectory(basePkg, workingPath);
progress(10);

progress.FinishRelease();
var dp = new DeltaPackage(_config.AppTempDir);

if (releasesToApply.Count() == 1) {
return ReleaseEntry.GenerateFromFile(outputPackageZip);
// apply every delta to the working directory
double progressStepSize = 100d / releases.Length;
for (var index = 0; index < releases.Length; index++) {
var r = releases[index];
double baseProgress = index * progressStepSize;

dp.ApplyDeltaPackageFast(workingPath, Path.Combine(_config.PackagesDir, r.Filename), x => {
double totalProgress = baseProgress + (progressStepSize * (x / 100d));
progress(Utility.CalculateProgress((int) totalProgress, 10, 90));
});
}

var fi = new FileInfo(outputPackageZip);
var entry = ReleaseEntry.GenerateFromFile(fi.OpenRead(), fi.Name);
var outputFile = Path.Combine(
_config.PackagesDir,
Regex.Replace(releases.Last().Filename, @"-delta.nupkg$", "-full.nupkg", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant));

progress(90);

// Recursively combine the rest of them
return await createFullPackagesFromDeltas(releasesToApply.Skip(1), entry, progress).ConfigureAwait(false);
// re-package working directory into full page
this.Log().Info("Repacking into full package: {0}", outputFile);
EasyZip.CreateZipFromDirectory(outputFile, workingPath, level: CompressionLevel.None);

progress(100);

return ReleaseEntry.GenerateFromFile(outputFile);
}

[SupportedOSPlatform("windows")]
Expand Down Expand Up @@ -243,17 +251,18 @@ async Task<string> invokePostInstall(string targetDir, bool isInitialInstall, bo
this.Log().Info("Squirrel Enabled Apps: [{0}]", String.Join(",", squirrelApps));

// For each app, run the install command in-order and wait
if (!firstRunOnly) await squirrelApps.ForEachAsync(async exe => {
using (var cts = new CancellationTokenSource()) {
cts.CancelAfter(30 * 1000);
try {
await PlatformUtil.InvokeProcessAsync(exe, args, Path.GetDirectoryName(exe), cts.Token).ConfigureAwait(false);
} catch (Exception ex) {
this.Log().ErrorException("Couldn't run Squirrel hook, continuing: " + exe, ex);
if (!firstRunOnly)
await squirrelApps.ForEachAsync(async exe => {
using (var cts = new CancellationTokenSource()) {
cts.CancelAfter(30 * 1000);
try {
await PlatformUtil.InvokeProcessAsync(exe, args, Path.GetDirectoryName(exe), cts.Token).ConfigureAwait(false);
} catch (Exception ex) {
this.Log().ErrorException("Couldn't run Squirrel hook, continuing: " + exe, ex);
}
}
}
}, 1 /* at a time */).ConfigureAwait(false);
}, 1 /* at a time */).ConfigureAwait(false);

// If this is the first run, we run the apps with first-run and
// *don't* wait for them, since they're probably the main EXE
Expand Down Expand Up @@ -366,4 +375,4 @@ static bool isAppFolderDead(string appFolderPath)
return File.Exists(Path.Combine(appFolderPath, ".dead"));
}
}
}
}
27 changes: 3 additions & 24 deletions src/Squirrel/UpdateManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ public async Task<ReleaseEntry> UpdateApp(Action<int> progress = null)
var currentVersion = CurrentlyInstalledVersion();

// 0 -> 10%
updateInfo = await this.ErrorIfThrows(() => CheckForUpdate(ignoreDeltaUpdates, x => progress(CalculateProgress(x, 0, 10))),
updateInfo = await this.ErrorIfThrows(() => CheckForUpdate(ignoreDeltaUpdates, x => progress(Utility.CalculateProgress(x, 0, 10))),
"Failed to check for updates").ConfigureAwait(false);

if (updateInfo == null || updateInfo.FutureReleaseEntry == null) {
Expand All @@ -133,12 +133,12 @@ public async Task<ReleaseEntry> UpdateApp(Action<int> progress = null)

// 10 -> 50%
await this.ErrorIfThrows(() =>
DownloadReleases(updateInfo.ReleasesToApply, x => progress(CalculateProgress(x, 10, 50))),
DownloadReleases(updateInfo.ReleasesToApply, x => progress(Utility.CalculateProgress(x, 10, 50))),
"Failed to download updates").ConfigureAwait(false);

// 50 -> 100%
await this.ErrorIfThrows(() =>
ApplyReleases(updateInfo, x => progress(CalculateProgress(x, 50, 100))),
ApplyReleases(updateInfo, x => progress(Utility.CalculateProgress(x, 50, 100))),
"Failed to apply updates").ConfigureAwait(false);

if (SquirrelRuntimeInfo.IsWindows) {
Expand Down Expand Up @@ -254,26 +254,5 @@ private Task<IDisposable> acquireUpdateLock()
return ret;
});
}

/// <summary>
/// Calculates the total percentage of a specific step that should report within a specific range.
/// <para />
/// If a step needs to report between 50 -> 75 %, this method should be used as CalculateProgress(percentage, 50, 75).
/// </summary>
/// <param name="percentageOfCurrentStep">The percentage of the current step, a value between 0 and 100.</param>
/// <param name="stepStartPercentage">The start percentage of the range the current step represents.</param>
/// <param name="stepEndPercentage">The end percentage of the range the current step represents.</param>
/// <returns>The calculated percentage that can be reported about the total progress.</returns>
internal static int CalculateProgress(int percentageOfCurrentStep, int stepStartPercentage, int stepEndPercentage)
{
// Ensure we are between 0 and 100
percentageOfCurrentStep = Math.Max(Math.Min(percentageOfCurrentStep, 100), 0);

var range = stepEndPercentage - stepStartPercentage;
var singleValue = range / 100d;
var totalPercentage = (singleValue * percentageOfCurrentStep) + stepStartPercentage;

return (int) totalPercentage;
}
}
}

0 comments on commit b3a2eca

Please sign in to comment.