C#: Opening and processing very large GeoJSON file

360 views Asked by At

I'm trying to process a 25GB GeoJSON file using GeoJSON.net

The accepted answer here works on a small test file but is causing Memory Exception errors with a large 25GB file

There isn't a huge amount of info about how to process the FeatureCollection so I'm just looping through

Can anyone advise what I'm doing wrong?

CODE

    try
    {
        

        JsonSerializer serializer = new JsonSerializer();

        using (FileStream s = File.Open(jsonFile, FileMode.Open))
        using (StreamReader sr = new StreamReader(s, Encoding.UTF8))
        using (JsonReader reader = new JsonTextReader(sr))
        {

            while (reader.Read())
            {
                // deserialize only when there's "{" character in the stream
                if (reader.TokenType == JsonToken.StartObject)
                {
                    FeatureCollection FC = serializer.Deserialize<FeatureCollection>(reader);
                    // Errors Here
                    foreach (var Feature in FC.Features)
                    {
                        if (Feature.Properties.ContainsKey("place"))
                        {
                            foreach (var p in Feature.Properties)
                            {
                                var Key = p.Key;
                                var Value = p.Value;
                                Console.WriteLine("Tags K: {0} Value: {1} ", Key, Value);
                            }
                        }
                    }

                }
            }
        }
    }
    catch (Exception e)
    {
        Console.WriteLine("Err: " + e.Message);
        
    }

The second answer on the same page isn't doing anything BTW JsonReaderExtensions is in a separate file copied from that page

        Regex regex = new Regex(@"^\[\d+\]$");


            using (FileStream s = File.Open(jsonFile, FileMode.Open))
            using (StreamReader sr = new StreamReader(s, Encoding.UTF8))
            using (JsonReader reader = new JsonTextReader(sr))
            {
                IEnumerable<FeatureCollection> objects = reader.SelectTokensWithRegex<FeatureCollection>(regex);

                foreach (var Feature in objects)
                {
                    Console.WriteLine("Hello");
                    // Doesn't get here
                }

Update:

I think the problem is with GeoJSON.net not Newtonsoft.Json as I've used the same method above to open bigger json files using dynamic jsonFeatures = serializer.Deserialize<ExpandoObject>(reader);

Following the comment by @Martin Costello

I've come up with opening the file using a standard StreamReader line by line then convert the filtered lines back into valid GeoJSON I'm sure there must be a better way to do this?

            string Start = @"{
""type"": ""FeatureCollection"",
""name"": ""points"",
""crs"": { ""type"": ""name"", ""properties"": { ""name"": ""urn:ogc:def:crs:OGC:1.3:CRS84"" } },
""features"": [";

            String End = @"]
}
";
            try
            {
                string line;
                StreamReader INfile = new StreamReader(jsonFile);
                while ((line = INfile.ReadLine()) != null)
                {
                        // for debugging
                    CurrentLine = line;
                      // Filter only the line with "place"
                    if (Regex.Match(line, @"\bplace\b", RegexOptions.IgnoreCase).Success)
                    {
                        // rebuild to valid GeoJSON 
                        string json = Start + line + End;
                        FeatureCollection FC = JsonConvert.DeserializeObject<FeatureCollection>(json);
                        foreach (var Feature in FC.Features)
                        {
                            foreach (var p in Feature.Properties)
                            {
                                var Key = p.Key;
                                var Value = p.Value;
                                switch (Key.ToLower())
                                {
                                    //  Do Stuff
                                }
                            }
                        }
                    }
                }
                        }
            catch (Exception e)
            {
                Console.OutputEncoding = System.Text.Encoding.UTF8;
                Console.WriteLine("Err: " + e.Message + "\n" + CurrentLine);
                
                
            }
0

There are 0 answers