-
Notifications
You must be signed in to change notification settings - Fork 38
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Improve decompression memory management #80
Labels
Milestone
Comments
yallie
added a commit
to yallie/Zyan
that referenced
this issue
Sep 25, 2020
Copied from the project: https://github.com/Aethon/SmallBlockMemoryStream
yallie
added a commit
to yallie/Zyan
that referenced
this issue
Sep 25, 2020
Benchmark code: //
// Compile server using:
// csc test.cs /r:Zyan.Communication.dll /platform:anycpu /debug /out:server.exe
//
// Compile client using:
// csc.exe test.cs /r:Zyan.Communication.dll /platform:x86 /debug /out:client.exe
//
// Start up server.exe, then run client.exe.
//
using System;
using System.Collections.Concurrent;
using System.Diagnostics;
using System.Linq;
using Zyan.Communication;
using Zyan.Communication.ChannelSinks.Compression;
using Zyan.Communication.Protocols.Tcp;
using Zyan.Communication.Toolbox;
class CanceledSubscriptionTest
{
static void Main()
{
try
{
StartServer();
}
catch
{
StartClient();
}
}
// ------------ Server code --------------
static void StartServer()
{
Console.Title = $"Server, PID: {Process.GetCurrentProcess().Id}";
var proto = new TcpDuplexServerProtocolSetup(4321)
{
CompressionMethod = CompressionMethod.Default
};
using (var host = new ZyanComponentHost(nameof(CanceledSubscriptionTest), proto))
{
host.RegisterComponent<IService, Service>();
Console.WriteLine("Server started. Press ENTER to quit.");
Console.ReadLine();
}
}
public class Service : IService
{
public byte[][] GetPayload(int count)
{
var chunkSize = 8192;
var arraySize = count / chunkSize;
Console.WriteLine($"Generating {count} bytes of payload, that is, byte[{chunkSize}][{arraySize}].");
var result = Enumerable.Range(1, arraySize).Select(i => new byte[chunkSize]).ToArray();
Console.WriteLine($"Returning approx. {chunkSize * arraySize} bytes.");
GC.Collect();
GC.WaitForPendingFinalizers();
GC.Collect();
return result;
}
}
// ------------ Shared code --------------
public interface IService
{
byte[][] GetPayload(int count);
}
// ------------ Client code --------------
static void StartClient()
{
Console.Title = $"Client, PID: {Process.GetCurrentProcess().Id}";
var proto = new TcpDuplexClientProtocolSetup()
{
CompressionMethod = CompressionMethod.Default
};
var url = proto.FormatUrl("localhost", 4321, nameof(CanceledSubscriptionTest));
using (var conn = new ZyanConnection(url))
{
Console.WriteLine("Client started. Hit ENTER for 10 runs, ^C to quit.");
var proxy = conn.CreateProxy<IService>();
while (true)
{
var command = Console.ReadLine();
var count = 0;
if (!int.TryParse(command, out count))
{
count = 10;
}
try
{
var size = 2_343_750;
for (var i = 0; i < count; i++)
{
Console.Write($"Get payload: {size} bytes...");
var data = proxy.GetPayload(size);
Console.WriteLine($"done!");
size *= 2;
}
}
catch (Exception ex)
{
Console.WriteLine($"Exception: {ex}");
throw;
}
}
}
}
} Plain Get payload: 2343750 bytes...done!
Get payload: 4687500 bytes...done!
Get payload: 9375000 bytes...done!
Get payload: 18750000 bytes...done!
Get payload: 37500000 bytes...done!
Get payload: 75000000 bytes...done!
Get payload: 150000000 bytes...done!
Get payload: 300000000 bytes...Exception: System.OutOfMemoryException Custom Get payload: 2343750 bytes...done!
Get payload: 4687500 bytes...done!
Get payload: 9375000 bytes...done!
Get payload: 18750000 bytes...done!
Get payload: 37500000 bytes...done!
Get payload: 75000000 bytes...done!
Get payload: 150000000 bytes...done!
Get payload: 300000000 bytes...done!
Get payload: 600000000 bytes...done!
Get payload: 1200000000 bytes...Exception: System.OutOfMemoryException Custom
|
Implemented in 1554ef0. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Zyan unpacks the compressed payload using
MemoryStream
.When the buffer overflows,
MemoryStream
allocates the larger block and moves the data:https://referencesource.microsoft.com/#mscorlib/system/io/memorystream.cs,283
Reallocation can be avoided if the required capacity is specified upfront:
https://referencesource.microsoft.com/#mscorlib/system/io/memorystream.cs,70
Or we can use chunked memory stream instead. Something like this perhaps:
https://github.com/Aethon/SmallBlockMemoryStream
The text was updated successfully, but these errors were encountered: