I'm using Python's watchdog module to listen for created events on a certain directory. Where I do some processing on newly created .csv files. When I test my code with nothing in the handler the watchdog fires correctly for all pasted/created files, but when I add my code/logic inside the handler it fires less than expected (52/60 files for example). OS used: Windows 10, code is expected to work on a Windows server. Python version: 3.7.3
Code:
import time
from watchdog.observers import Observer
from watchdog.events import FileSystemEventHandler
import pandas as pd
import numpy as np
from error_handling import ErrorHandler
import os
from timeloop import Timeloop
from datetime import timedelta
import json
from fill_data import OnMyWatch2
class OnMyWatch:
# Set the directory on watch
watchDirectory = "."
def __init__(self):
self.observer1 = Observer()
def run(self):
self.observer1.schedule(Handler() , self.watchDirectory, recursive = True)
self.observer1.start()
try:
while True:
time.sleep(1)
except:
self.observer1.stop()
print("Observer Stopped")
self.observer1.join()
class Handler(FileSystemEventHandler):
@staticmethod
def on_any_event(event):
if event.is_directory:
return None
elif event.event_type == 'created':
try:
# Event is created, you can process it now
print("Watchdog received created event - % s." % event.src_path)
pathlower = str(event.src_path).lower()
if ".csv" in pathlower:
print("File is csv file")
# Once any code is added here the problem happens
# Example:
# df = pd.read_csv(event.src_path, names= ALL_INI.columnlistbundle.split("|"), sep="|", encoding='latin-1', engine='python')
# arraySelectedColumnsBundle = ALL_INI.selectedcolumnsbundle.split(",")
# bundle_df = df[np.array(arraySelectedColumnsBundle)]
else:
print("File is not csv file")
except Exception as e:
ErrorHandler(0, 'In Observer ', '-7', 'Exception ' + str(e), '', 1, '')
if __name__ == '__main__':
if os.path.isdir("."):
watch = OnMyWatch()
watch.run()