I am trying to find a possible memory leak in my python script. I am using memory profiler but I am not sure how to interpret the memory profile output to detect the memory leak.
Filename: vc_query.py
Line # Mem usage Increment Occurrences Line Contents
=============================================================
818 79.36 MiB 79.36 MiB 1 @profile(stream=open("mem_profile.log", "w"), precision=2)
819 def run_query_n(args):
820 79.36 MiB 0.00 MiB 1 def download_notifs(url, start, size, update):
821 payload.update({"start": start, "length": size})
822 partial = download_report(url=url, payload=payload, silent=True, client=vc_, method='POST', repeat=args.max_retry, repeat_after=1).json().get('data')
823 if partial:
824 for ea in partial:
825 ea.update(update)
826 outputs.extend(partial)
827
828 79.38 MiB 0.02 MiB 5 started_range = list(map(lambda x: parser.parse(x), args.notif_between.split('->')))
829 79.38 MiB 0.00 MiB 1 ranges = split_date(started_range[0], started_range[1], 'd')
865 79.45 MiB 0.07 MiB 1 db, outputs = sqlite3.connect(":memory:"), []
866 292.77 MiB 0.00 MiB 2 for dates in ranges:
880 81.61 MiB 0.55 MiB 3 roof = download_report(url=f"{base_url}reports/notifications2.json?status={statuses.index(status)}&{cat}{query}&report_type[]={n_type}", silent=True, client=vc_
881 81.61 MiB 1.60 MiB 2 , method='POST', payload=payload).json().get('recordsFiltered')
882 81.61 MiB 0.00 MiB 1 if is_number(roof):
883 292.77 MiB 211.08 MiB 2 with concurrent.futures.ThreadPoolExecutor() as executor:
884 81.70 MiB 0.00 MiB 2 for i in range((int(roof) // int(args.page_size))+1):
885 81.70 MiB 0.08 MiB 2 executor.submit(download_notifs, f"{base_url}reports/notifications2.json?status={statuses.index(status)}&{cat}{query}&report_type[]={n_type}"
886 81.62 MiB 0.00 MiB 1 , i*int(args.page_size), int(args.page_size), {'report_type': n_type})
887 292.77 MiB 0.00 MiB 1 tse = datetime.now()
888 292.77 MiB 0.00 MiB 1 if args.output:
889 292.77 MiB 0.00 MiB 1 if 'return' not in args.output:
890 292.77 MiB 0.00 MiB 1 print(f'completed\tduration: {(tse - tss).seconds} seconds')
891 292.77 MiB 0.00 MiB 1 sql = "CREATE TABLE IF NOT EXISTS notif ("
892 292.77 MiB 0.00 MiB 22 for field in notif_headers:
893 292.77 MiB 0.00 MiB 21 sql = sql + field + " TEXT,"
894 292.77 MiB 0.00 MiB 1 sql = sql[:-1] + ")"
895 293.40 MiB 0.63 MiB 1 db.cursor().execute(sql)
896 293.40 MiB 0.00 MiB 1 unique_id = set()
897 293.40 MiB 0.00 MiB 1 if len(outputs) > 0:
898 305.77 MiB 0.00 MiB 28639 for row in outputs:
899 305.77 MiB 0.00 MiB 28638 if row.get('id') in unique_id:
900 continue
901 305.77 MiB 2.66 MiB 28638 unique_id.add(row.get('id'))
902 305.77 MiB 0.00 MiB 28638 sql = "Insert INTO notif VALUES ("
903 305.77 MiB 0.00 MiB 630036 for key in notif_keys:
904 305.77 MiB 0.00 MiB 601398 try:
905 305.77 MiB 0.00 MiB 601398 value = get_json_value(row, key)
906 except Exception:
907 value = ''
908 305.77 MiB 0.02 MiB 601398 sql = sql + "\"" + str(value).replace("\"", '') + "\","
909 305.77 MiB 0.00 MiB 28638 sql = sql[:-1] + ")"
910 305.77 MiB 9.68 MiB 28638 db.cursor().execute(sql)
911 186.11 MiB -119.66 MiB 1 outputs = []
912 186.11 MiB 0.00 MiB 1 split_categories = set()
913 186.11 MiB 0.00 MiB 1 db.row_factory = sqlite3.Row
914 187.64 MiB 1.53 MiB 1 query = db.cursor().execute(f"select * from notif" if args.sql == '' else args.sql).fetchall()
915 187.64 MiB 0.00 MiB 1 found = 0
916 187.64 MiB 0.00 MiB 1 limit = args.limit if args.limit else -1
917 203.81 MiB -12335.21 MiB 28639 for r in query:
918 203.81 MiB -12324.48 MiB 28638 if found == limit:
919 break
920 203.81 MiB -12324.48 MiB 28638 dt = dict(r)
927 203.81 MiB -12324.48 MiB 28638 found += 1
928 203.81 MiB -12319.86 MiB 28638 dt.update(to_timestamp(dt))
929 203.81 MiB -12335.22 MiB 28638 ot = {}
930 203.81 MiB -12335.22 MiB 28638 if '@ALL' not in args.columns:
931 ot.update({key: val for key, val in dt.items() if key in args.columns})
932 else:
933 203.81 MiB -12335.22 MiB 28638 ot = dt.copy()
934 203.81 MiB -12334.39 MiB 28638 outputs.append(ot)
935 203.81 MiB -12335.21 MiB 28638 if hasattr(args, 'split'):
936 split_categories.add(ot.get(args.split[0]))
937 183.94 MiB -19.87 MiB 1 db.close()
938 183.95 MiB 0.00 MiB 1 return outputs
I expect the memory usage return to where it began at line 820 and since I run this as a service (it is an infinite loop) the memory usage keeps growing.