I have a data loading process that load a big amount of data into DataTable then do some data process, but every time when the job finished the DataLoader.exe(32bit, has a 1.5G memory limit) does not release all the memory being used.
I tried 3 ways to release memory:
- DataTable.Clear() then call DataTable.Dispose() (Release about 800 MB memory but still increase 200 MB memory every time data loading job finish, after 3 or 4 times of data loading, out of memory exception thrown because it exceeds 1.5 G memory in total)
- Set DataTable to null (No memory released, and if choose load more data, out of memory exception thrown)
- call DataTable.Dispose() directly (No memory released, and if choose load more data, out of memory exception thrown)
Following is the code I tried for testing(In the real program it is not called recursively, it is triggered by some directory watching logic. This code is just for testing. Sorry for the confusion.):
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Data;
namespace DataTable_Memory_test
{
class Program
{
static void Main(string[] args)
{
try
{
LoadData();
Console.ReadKey();
}
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
Console.ReadKey();
}
}
private static void LoadData()
{
DataTable table = new DataTable();
table.Columns.Add("Dosage", typeof(int));
table.Columns.Add("Drug", typeof(string));
table.Columns.Add("Patient", typeof(string));
table.Columns.Add("Date", typeof(DateTime));
// Fill the data table to make it take about 1 G memory.
for (int i = 0; i < 1677700; i++)
{
table.Rows.Add(25, "Indocin", "David", DateTime.Now);
table.Rows.Add(50, "Enebrel", "Sam", DateTime.Now);
table.Rows.Add(10, "Hydralazine", "Christoff", DateTime.Now);
table.Rows.Add(21, "Combivent", "Janet", DateTime.Now);
table.Rows.Add(100, "Dilantin", "Melanie", DateTime.Now);
}
Console.WriteLine("Data table load finish: please check memory.");
Console.WriteLine("Press 0 to clear and dispose datatable, press 1 to set datatable to null, press 2 to dispose datatable directly");
string key = Console.ReadLine();
if (key == "0")
{
table.Clear();
table.Dispose();
Console.WriteLine("Datatable disposed, data table row count is {0}", table.Rows.Count);
GC.Collect();
long lMemoryMB = GC.GetTotalMemory(true/* true = Collect garbage before measuring */) / 1024 / 1024; // memory in megabytes
Console.WriteLine(lMemoryMB);
}
else if (key == "1")
{
table = null;
GC.Collect();
long lMemoryMB = GC.GetTotalMemory(true/* true = Collect garbage before measuring */) / 1024 / 1024; // memory in megabytes
Console.WriteLine(lMemoryMB);
}
else if (key == "2")
{
table.Dispose();
GC.Collect();
long lMemoryMB = GC.GetTotalMemory(true/* true = Collect garbage before measuring */) / 1024 / 1024; // memory in megabytes
Console.WriteLine(lMemoryMB);
}
Console.WriteLine("Job finish, please check memory");
Console.WriteLine("Press 0 to exit, press 1 to load more data and check if throw out of memory exception");
key = Console.ReadLine();
if (key == "0")
{
Environment.Exit(0);
}
else if (key == "1")
{
LoadData();
}
}
}
}
Eventually I found this Data table not release memory bug was caused by Oracle bulk copy. Just in case some one got the same problem. Please see following post for reference
OracleBulkCopy Memory Leak(OutOfMemory Exception)