Facing too much Insufficient Memory exception while working with memory mapped-IO files

37 views Asked by At

I am trying to create memory-mapped IO files for having sized ( 1-100 GB ) files. These files will contain random data that will be generated during the process.

So, I am trying to generate partitions dynamically by considering the available memory of the system.

Following is the source code.

public class RandomLinesGenerator
{
    private static string GenerateRandomTextLine(int minValue, int maxValue)
    {
        const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
        Random random = new Random();
        int length = random.Next(minValue, maxValue);

        return new string(Enumerable.Repeat(chars, length)
          .Select(s => s[random.Next(s.Length)]).ToArray());
    }

    public static byte[] GetRandomBytes(long desiredByteCount, int prefixIntMinValue, int prefixIntMaxValue)
    {
        Random random = new Random();

        long currentByteCount = 0;

        StringBuilder builder = new StringBuilder();

        while (currentByteCount < desiredByteCount)
        {
            // Generate a random integer
            int randomNumber = random.Next(prefixIntMinValue, prefixIntMaxValue + 1);

            // Generate a random line
            string randomLine = $"{randomNumber} {GenerateRandomTextLine(1, 150)}";

            // Calculate the byte count of the line
            long lineByteCount = Encoding.UTF8.GetByteCount(randomLine) + Environment.NewLine.Length;

            // If adding the line exceeds the desired byte count, break the loop
            if (currentByteCount + lineByteCount > desiredByteCount)
            {
                break;
            }

            builder.AppendLine(randomLine); // ==> This is the line where the exception is occurring of Insufficient memory..

            currentByteCount += lineByteCount;
        }

        var bytes =  Encoding.UTF8.GetBytes(builder.ToString());
        builder.Clear();

        return bytes;
    }
}

public class MemoryMappedIOTechnique
{
    private static readonly string _fileLocation = Path.Combine("Resources", "files");
    private long GetPartitionSize(long totalFileSize)
    {
        var partitionSize = (long)Math.Ceiling((long)new PerformanceCounter("Memory", "Available Bytes").NextValue() * 0.8);
        return Math.Min(partitionSize ,totalFileSize);
    }

    public void Execute(int sizeInGb)
    {
        try
        {    

            long totalBytesToWrite = sizeInGb * Convert.ToInt64(1024 * 1024 * 1024);

            string filePath = Path.Combine(_fileLocation, $"file_{sizeInGb}-GB.txt");
            long partitionSize = GetPartitionSize(totalBytesToWrite);

            using (MemoryMappedFile mmf = MemoryMappedFile.CreateFromFile(filePath, FileMode.Create, "mmf", totalBytesToWrite))
            {
                long offset = 0;
                long partitions = (totalBytesToWrite / partitionSize);

                for (var partition = 0; partition < partitions; partition++)
                {
                    using (var accessor = mmf.CreateViewAccessor(offset, partitionSize))
                    {
                        var randomBytes = RandomLinesGenerator.GetRandomBytes(partitionSize, 10, 2500);
                        accessor.WriteArray(0, randomBytes, 0, randomBytes.Length);
                        offset += partitionSize;
                    }
                }
            }
        }
        catch(Exception ex)
        {

        }
    }

Now I am getting an exception while I just called the function for providing 2 GB. First, it is taking more amount of time and the other it causes the exception of 'Insufficient memory I could see that on Task Manager, almost 3 GB is still free.

Can anybody guide me on how to change the source code to handle safely all types of cases without having insufficient memory exception cases? Need to generate ( 1, 2, ...-100 GB ) files.

I am completely new to Memory-mapped IO and have very little knowledge about this whole topic.

0

There are 0 answers