Python - Opening and changing large text files

Question

I have a ~600MB Roblox type .mesh file, which reads like a text file in any text editor. I have the following code below:

mesh = open("file.mesh", "r").read()
mesh = mesh.replace("[", "{").replace("]", "}").replace("}{", "},{")
mesh = "{"+mesh+"}"
f = open("p2t.txt", "w")
f.write(mesh)

It returns:

Traceback (most recent call last):
  File "C:\TheDirectoryToMyFile\p2t2.py", line 2, in <module>
    mesh = mesh.replace("[", "{").replace("]", "}").replace("}{", "},{")
MemoryError

Here is a sample of my file:

[-0.00599, 0.001466, 0.006][0.16903, 0.84515, 0.50709][0.00000, 0.00000, 0][-0.00598, 0.001472, 0.00599][0.09943, 0.79220, 0.60211][0.00000, 0.00000, 0]

What can I do?

Edit:

I'm not sure what the head, follow, and tail commands are in that other thread that marked this as a duplicate. I tried to use it, but couldn't get it to work. The file is also one giant line, it isn't split into lines.

Answer 1

You need to read one bite per iteration, analyze it and then write to another file or to sys.stdout. Try this code:

mesh = open("file.mesh", "r")
mesh_out = open("file-1.mesh", "w")

c = mesh.read(1)

if c:
    mesh_out.write("{")
else:
    exit(0)
while True:
    c = mesh.read(1)
    if c == "":
        break

    if c == "[":
        mesh_out.write(",{")
    elif c == "]":
        mesh_out.write("}")
    else:
        mesh_out.write©

UPD:

It works really slow (thanks to jamylak). So I've changed it:

import sys
import re


def process_char(c, stream, is_first=False):
    if c == '':
        return False
    if c == '[':
        stream.write('{' if is_first else ',{')
        return True
    if c == ']':
        stream.write('}')
        return True


def process_file(fname):
    with open(fname, "r") as mesh:
        c = mesh.read(1)
        if c == '':
            return
        sys.stdout.write('{')

        while True:
            c = mesh.read(8192)
            if c == '':
                return

            c = re.sub(r'\[', ',{', c)
            c = re.sub(r'\]', '}', c)
            sys.stdout.write(c)


if __name__ == '__main__':
    process_file(sys.argv[1])

So now it's working ~15 sec on 1.4G file. To run it:

$ python mesh.py file.mesh > file-1.mesh

Answer 2

You could do it line by line:

mesh = open("file.mesh", "r")
with open("p2t.txt", "w") as f:
   for line in mesh:
      line= line.replace("[", "{").replace("]", "}").replace("}{", "},{")
      line = "{"+line +"}"
      f.write(line)

Answer 3

import os
f = open('p2f.txt','w')
with open("file.mesh") as mesh:
  while True:
    c = mesh.read(1)
    if not c:
      f.seek(-1,os.SEEK_END)
      f.truncate()
      break
    elif c == '[':
        f.write('{')
    elif c == ']':
        f.write('},')
   else:
       f.write(c)

p2f.txt:

{-0.00599, 0.001466, 0.006},{0.16903, 0.84515, 0.50709},{0.00000, 0.00000, 0},{-0.00598, 0.001472, 0.00599},{0.09943, 0.79220, 0.60211},{0.00000, 0.00000, 0}

Answer 4

def read(afilename):
    with open("afilename", "r") as file
       lines = file.readlines()
       lines.replace("[", "{")
       #place reset of code here in 

Answer 5

BLOCK_SIZE = 1 << 15
with open(input_file, 'rb') as fin, open(output_file, 'wb') as fout:
    for block in iter(lambda: fin.read(BLOCK_SIZE), b''):
        # do your replace
        fout.write(block)