I am trying to create memory-mapped IO files for having sized ( 1-100 GB ) files. These files will contain random data that will be generated during the process.
So, I am trying to generate partitions dynamically by considering the available memory of the system.
Following is the source code.
public class RandomLinesGenerator
{
private static string GenerateRandomTextLine(int minValue, int maxValue)
{
const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
Random random = new Random();
int length = random.Next(minValue, maxValue);
return new string(Enumerable.Repeat(chars, length)
.Select(s => s[random.Next(s.Length)]).ToArray());
}
public static byte[] GetRandomBytes(long desiredByteCount, int prefixIntMinValue, int prefixIntMaxValue)
{
Random random = new Random();
long currentByteCount = 0;
StringBuilder builder = new StringBuilder();
while (currentByteCount < desiredByteCount)
{
// Generate a random integer
int randomNumber = random.Next(prefixIntMinValue, prefixIntMaxValue + 1);
// Generate a random line
string randomLine = $"{randomNumber} {GenerateRandomTextLine(1, 150)}";
// Calculate the byte count of the line
long lineByteCount = Encoding.UTF8.GetByteCount(randomLine) + Environment.NewLine.Length;
// If adding the line exceeds the desired byte count, break the loop
if (currentByteCount + lineByteCount > desiredByteCount)
{
break;
}
builder.AppendLine(randomLine); // ==> This is the line where the exception is occurring of Insufficient memory..
currentByteCount += lineByteCount;
}
var bytes = Encoding.UTF8.GetBytes(builder.ToString());
builder.Clear();
return bytes;
}
}
public class MemoryMappedIOTechnique
{
private static readonly string _fileLocation = Path.Combine("Resources", "files");
private long GetPartitionSize(long totalFileSize)
{
var partitionSize = (long)Math.Ceiling((long)new PerformanceCounter("Memory", "Available Bytes").NextValue() * 0.8);
return Math.Min(partitionSize ,totalFileSize);
}
public void Execute(int sizeInGb)
{
try
{
long totalBytesToWrite = sizeInGb * Convert.ToInt64(1024 * 1024 * 1024);
string filePath = Path.Combine(_fileLocation, $"file_{sizeInGb}-GB.txt");
long partitionSize = GetPartitionSize(totalBytesToWrite);
using (MemoryMappedFile mmf = MemoryMappedFile.CreateFromFile(filePath, FileMode.Create, "mmf", totalBytesToWrite))
{
long offset = 0;
long partitions = (totalBytesToWrite / partitionSize);
for (var partition = 0; partition < partitions; partition++)
{
using (var accessor = mmf.CreateViewAccessor(offset, partitionSize))
{
var randomBytes = RandomLinesGenerator.GetRandomBytes(partitionSize, 10, 2500);
accessor.WriteArray(0, randomBytes, 0, randomBytes.Length);
offset += partitionSize;
}
}
}
}
catch(Exception ex)
{
}
}
Now I am getting an exception while I just called the function for providing 2 GB. First, it is taking more amount of time and the other it causes the exception of 'Insufficient memory I could see that on Task Manager, almost 3 GB is still free.
Can anybody guide me on how to change the source code to handle safely all types of cases without having insufficient memory exception cases? Need to generate ( 1, 2, ...-100 GB ) files.
I am completely new to Memory-mapped IO and have very little knowledge about this whole topic.