From 8ad51ab63aa5f4b6b3e68cf14fd1b99e343e18a5 Mon Sep 17 00:00:00 2001 From: Arne van Iterson Date: Sun, 1 Oct 2023 15:28:32 +0200 Subject: [PATCH 1/2] CSV export in suite --- src/suite.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/suite.py b/src/suite.py index 7538dab..1e314a2 100644 --- a/src/suite.py +++ b/src/suite.py @@ -66,12 +66,17 @@ class MainApp: # Load values from config after UI has been initialised self.img_path.set(config_json["path"]) self.img_size.set(config_json["size"]) + + now = datetime.datetime.now() + self.log = open(pathlib.Path(config_json["out"], F"result-{now.strftime('%Y-%m-%dT%H.%M.%S')}.csv"), "x") + self.log.write("test") def on_quit(self, event=None): ''' Close PLT windows on main app quit ''' plt.close() + self.log.close() self.mainwindow.quit(); def run(self): @@ -358,6 +363,12 @@ class MainApp: if not part_update: self.drawCannyHM(img, 1, 1) + # Write results to CSV file + if not part_update: + # if self.log. + # Only do this if the image has changed + self.log.write(F"{self.img_name[1]}, test\n") + # Show all data plt.show(block=False) ## Graphs self.draw_output(size) ## Images From a41d57f45137a507dbb519bfdf20758e4ea08019 Mon Sep 17 00:00:00 2001 From: Arne van Iterson Date: Sun, 1 Oct 2023 18:09:26 +0200 Subject: [PATCH 2/2] Add csv logging component --- src/helpers/logger.py | 47 ++++++ src/suite.py | 325 ++++++++++++++++++++++++------------------ 2 files changed, 237 insertions(+), 135 deletions(-) create mode 100644 src/helpers/logger.py diff --git a/src/helpers/logger.py b/src/helpers/logger.py new file mode 100644 index 0000000..c624726 --- /dev/null +++ b/src/helpers/logger.py @@ -0,0 +1,47 @@ +import pathlib +import datetime + +now = datetime.datetime.now() + + +class Logger: + def __init__(self, path): + self.fileName = pathlib.Path( + path, f"result-{now.strftime('%Y-%m-%dT%H.%M.%S')}.csv" + ) + self.file = open(self.fileName, "x") + + self.first = True + self.index = [] + self.data = [] + + def add(self, name: str, value): + self.index.append(name) + self.data.append(value) + + def csv(self, input): + result = "" + + for idx, item in enumerate(input): + result += str(item) + if idx != (len(input) - 1): + result += ", " + result += "\n" + print(result) + + return result + + def update(self): + if self.first: + self.first = False + self.file.write(self.csv(self.index)) + + self.file.write(self.csv(self.data)) + + # Clear data + self.index = [] + self.data = [] + + def __del__(self): + print("Log file closed!") + self.file.close() diff --git a/src/suite.py b/src/suite.py index 3b3bf0c..cb0cab4 100644 --- a/src/suite.py +++ b/src/suite.py @@ -9,11 +9,13 @@ import cv2 import time import matplotlib.pyplot as plt import json -from helpers.statistics import imgStats -import datetime +import datetime import os import copy +from helpers.statistics import imgStats +from helpers.logger import Logger + ## UI config load PROJECT_PATH = pathlib.Path(__file__).parent PROJECT_UI = "./src/helpers/gui/main.ui" @@ -23,26 +25,29 @@ CONFIG_PATH = "./src/config/config.json" config_file = open(CONFIG_PATH) config_json = json.load(config_file) +log = Logger(config_json["out"]) + + ## UI class setup class MainApp: def __init__(self, master=None): self.builder = builder = pygubu.Builder() builder.add_resource_path(PROJECT_PATH) builder.add_from_file(PROJECT_UI) - + # Main widget self.mainwindow = builder.get_object("main", master) # Canvas for output images self.canvas = builder.get_object("output_canvas") - self.tk_imgs = [] # Required or python will forget - self.output = [[] for x in range(2)] + self.tk_imgs = [] # Required or python will forget self.meta = builder.get_object("dataset") + self.output = [[] for x in range(2)] # Keep track of images in dataset self.img_current = 0 self.img_name = "" - self.img_old = -1 ## minus 1 to enforce full update on start + self.img_old = -1 ## minus 1 to enforce full update on start self.img_max = 0 # Plots @@ -53,42 +58,54 @@ class MainApp: self.canny_thr2 = None self.img_path = None self.contrast = None - + self.img_size = None - self.img_size_old = 0 ## Check if the rendering size has changed, if it has the analysis has to be run - + self.img_size_old = 0 ## Check if the rendering size has changed, if it has the analysis has to be run + self.sobel_select = None self.export_id = None self.brightness = None - builder.import_variables(self,['canny_thr1','canny_thr2','img_path','contrast','img_size','sobel_select','export_id','brightness']) + builder.import_variables( + self, + [ + "canny_thr1", + "canny_thr2", + "img_path", + "contrast", + "img_size", + "sobel_select", + "export_id", + "brightness", + ], + ) builder.connect_callbacks(self) - + # Load values from config after UI has been initialised self.img_path.set(config_json["path"]) self.img_size.set(config_json["size"]) - - now = datetime.datetime.now() - self.log = open(pathlib.Path(config_json["out"], F"result-{now.strftime('%Y-%m-%dT%H.%M.%S')}.csv"), "x") - self.log.write("test") def on_quit(self, event=None): - ''' - Close PLT windows on main app quit - ''' - plt.close() - self.log.close() - self.mainwindow.quit(); + """ + Close PLT windows on main app quit + """ + # TODO: This function runs multiple times for some reason + + if plt is not None: + plt.close() # Close graph vies + log.file.close() # Close log files + + self.mainwindow.quit() # Close main def run(self): - ''' - Run loop - ''' + """ + Run loop + """ self.mainwindow.mainloop() - + def img_prev(self, event=None): - ''' - Open previous image from path - ''' + """ + Open previous image from path + """ if self.img_current == 0: self.img_current = self.img_max - 1 else: @@ -96,9 +113,9 @@ class MainApp: self.update(self) def img_next(self, event=None): - ''' - Open next image from path - ''' + """ + Open next image from path + """ if self.img_current == (self.img_max - 1): self.img_current = 0 else: @@ -106,111 +123,114 @@ class MainApp: self.update(self) def apply(self, event=None, path=None): - ''' - Export current dataset - ''' + """ + Export current dataset + """ # Get export settings img_arr = self.tk_imgs img_id = self.export_id.get() if path == None: path = config_json["out"] else: - print(F"Using path: {path}") - - if (img_id >= 0 and img_id < len(img_arr)): + print(f"Using path: {path}") + + if img_id >= 0 and img_id < len(img_arr): # Create file now = datetime.datetime.now() - new_file_name = F"{self.img_current}-{self.output[1][img_id]}-{now.strftime('%Y-%m-%dT%H.%M.%S')}.png" - + new_file_name = f"{self.img_current}-{self.output[1][img_id]}-{now.strftime('%Y-%m-%dT%H.%M.%S')}.png" + # Put data file_path = pathlib.Path(path, new_file_name) # print(file_path) - + imgpil = ImageTk.getimage(self.tk_imgs[img_id]) - imgpil.save(file_path, "PNG" ) + imgpil.save(file_path, "PNG") imgpil.close() - + print(f"Exported Image ID {img_id} to {os.path.join(path, new_file_name)}") else: print("Nothing to export!") - + def apply_all(self, event=None): - ''' - Export given preprocess id for every image in the dataset folder - ''' + """ + Export given preprocess id for every image in the dataset folder + """ img_id = self.export_id.get() img_current = copy.deepcopy(self.img_current) - + now = datetime.datetime.now() - path = pathlib.Path(config_json["out"], F"{self.output[1][img_id]}-all-{now.strftime('%Y-%m-%dT%H.%M.%S')}/") + path = pathlib.Path( + config_json["out"], + f"{self.output[1][img_id]}-all-{now.strftime('%Y-%m-%dT%H.%M.%S')}/", + ) os.mkdir(path) - + while True: self.img_next() - self.update(part_update=True) # Enforce partial update since we don't need the histograms etc. + self.update( + part_update=True + ) # Enforce partial update since we don't need the histograms etc. self.apply(path=path) - - if (self.img_current == img_current): + + if self.img_current == img_current: break ## Ensure display is always correct with image self.update() def add_output(self, data, name: str): - ''' - Add CV2 image to canvas output - ''' + """ + Add CV2 image to canvas output + """ self.output[0].append(data) self.output[1].append(name) - + def draw_output(self, size): # Check if size of canvas has updated drawW = self.canvas.winfo_width() - + # Reset drawing position drawX = 0 drawY = 0 - + # Clear previously printed images self.tk_imgs = [] - + self.meta.config(state=NORMAL) self.meta.delete(1.0, END) - self.meta.insert(END, f"{self.img_name[1]}\n") - + self.meta.insert(END, f"{self.img_name}\n") + # Draw all output images for idx, data in enumerate(self.output[0]): # Create ui image tk_img = cv2.cvtColor(data, cv2.COLOR_BGR2RGB) tk_img = ImageTk.PhotoImage(image=Image.fromarray(tk_img)) self.tk_imgs.append(tk_img) - + ## Check if next item will be out of range - if (drawX + size >= drawW): + if drawX + size >= drawW: drawY += size drawX = 0 - self.canvas.configure(height=(drawY+size)) - - self.canvas.create_image(drawX,drawY,anchor=NW,image=self.tk_imgs[idx],tags="og") + self.canvas.configure(height=(drawY + size)) + + self.canvas.create_image( + drawX, drawY, anchor=NW, image=self.tk_imgs[idx], tags="og" + ) drawX += size - + # Add name to text box - self.meta.insert(END, F"{idx}: {self.output[1][idx]}\n") - + self.meta.insert(END, f"{idx}: {self.output[1][idx]}\n") + # Clear output self.meta.config(state=DISABLED) - - # Draw canvas - # TODO IDK volgens mij moet je deze wel callen maar het programma doet het nog (geen vragen stellen) - # self.canvas.draw() def createPlot(self, columns, rows): fig, axs = plt.subplots(columns, rows) return axs - + def drawHist(self, image, labels, column, row): self.axs[column, row].clear() - for i,lab in enumerate(labels): + for i, lab in enumerate(labels): hist = cv2.calcHist( [image], [i], @@ -221,7 +241,7 @@ class MainApp: self.axs[column, row].plot(hist, label=lab) self.axs[column, row].grid() self.axs[column, row].legend() - + def drawCannyHM(self, img, column, row): self.axs[column, row].clear() canny_max = 500 @@ -233,41 +253,71 @@ class MainApp: for th2 in range(0, canny_max, canny_step): # Canny Edge Detection edges = cv2.Canny(image=img, threshold1=th1, threshold2=th2) - + w_res = cv2.countNonZero(edges) y_ind = (int)(th1 / canny_step) x_ind = (int)(th2 / canny_step) - + results[y_ind].append(w_res) - + # print(f"Result at thres {th1}, {th2}; \tIndex {y_ind}, {x_ind} \t= {w_res}") # print(results[y_ind]) - - + func = np.diag(results) - self.axs[column, row-1].clear() - self.axs[column, row-1].title.set_text("Canny F U N C") - self.axs[column, row-1].plot(func) - self.axs[column, row-1].plot(np.diff(func)) - - self.axs[column, row].title.set_text(F"Mean: {np.matrix(results).mean()}\nStd: {np.matrix(results).std()}") + self.axs[column, row - 1].clear() + self.axs[column, row - 1].title.set_text("Canny F U N C") + self.axs[column, row - 1].plot(func) + self.axs[column, row - 1].plot(np.diff(func)) + + self.axs[column, row].title.set_text( + f"Mean: {np.matrix(results).mean()}\nStd: {np.matrix(results).std()}" + ) self.axs[column, row].imshow(results) - self.axs[column, row].xaxis.set_major_formatter(lambda x, pos: str(x*canny_step)) - self.axs[column, row].yaxis.set_major_formatter(lambda x, pos: str(x*canny_step)) + self.axs[column, row].xaxis.set_major_formatter( + lambda x, pos: str(x * canny_step) + ) + self.axs[column, row].yaxis.set_major_formatter( + lambda x, pos: str(x * canny_step) + ) + + log.add("Canny Mean", np.matrix(results).mean()) + log.add("Canny Std", np.matrix(results).std()) + log.add("Canny Min", np.matrix(results).min()) + log.add("Canny Max", np.matrix(results).max()) + # log.add("Canny Diff max", np.diff(func)) def writeStats(self, img, labels, column, row): mean, std = imgStats(img) self.axs[column, row].title.set_text( "Mean: %c:%d %c:%d %c:%d \nStd: %c:%d %c:%d %c:%d" - %(labels[0], mean[0], labels[1], mean[1], labels[2], mean[2], - labels[0], std[0], labels[1], std[1], labels[2], std[2])) - + % ( + labels[0], + mean[0], + labels[1], + mean[1], + labels[2], + mean[2], + labels[0], + std[0], + labels[1], + std[1], + labels[2], + std[2], + ) + ) + + for idx, label in enumerate(labels): + log.add(f"Mean {label}", mean[idx]) + log.add(f"Std {label}", std[idx]) + def update(self, event=None, part_update=False): path = self.img_path.get() - + ## Check if hist and canny hm have to be rerendered - if not part_update: ## If partial update has not been forced, check if full update is required - if (self.img_current != self.img_old or self.img_size != self.img_size_old): + if ( + not part_update + ): ## If partial update has not been forced, check if full update is required + if self.img_current != self.img_old or self.img_size != self.img_size_old: part_update = False self.img_old = self.img_current self.img_size_old = self.img_size @@ -275,7 +325,7 @@ class MainApp: part_update = True else: print("Partial update forced!") - + if path != None and path != "": # Get all images at current path images = [] @@ -284,61 +334,67 @@ class MainApp: for file in glob.glob(path + "/*.png"): images.append(file) - + self.img_max = len(images) - self.img_name = os.path.split(images[self.img_current]) - + self.img_name = os.path.split(images[self.img_current])[1] + log.add("Img", self.img_name) + # Get all user vars ct1 = self.canny_thr1.get() - ct2 = self.canny_thr2.get() + ct2 = self.canny_thr2.get() sxy = self.sobel_select.get() size = self.img_size.get() contrast = self.contrast.get() bright = self.brightness.get() - + # Clear output self.output = [[] for x in range(2)] - + # Import and resize image img = cv2.imread(images[self.img_current]) - img = cv2.resize(img, (size, size), interpolation = cv2.INTER_AREA) + img = cv2.resize(img, (size, size), interpolation=cv2.INTER_AREA) self.add_output(img, "Original") - + # Set grayscale img_gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) self.add_output(img_gray, "Grayscale") - + # Contrast / brightness boost contrast_val = contrast / 100 bright_val = bright / 100 - img_contrast = np.clip(contrast_val * (img_gray + bright_val), 0, 255).astype(np.uint8) + img_contrast = np.clip( + contrast_val * (img_gray + bright_val), 0, 255 + ).astype(np.uint8) # self.add_output(img_contrast, F"Contrast / Brightness\n c+{contrast_val} b+{bright_val}") - self.add_output(img_contrast, F"BCG") - + self.add_output(img_contrast, f"BCG") + # Blurred edition img_blur = cv2.GaussianBlur(img_gray, (3, 3), 0) self.add_output(img_blur, "Blurred_k3") - + # Sobel edge - if sxy in ['x', 'y', 'both']: - if sxy == 'x': + if sxy in ["x", "y", "both"]: + if sxy == "x": dx = 1 dy = 0 - elif sxy == 'y': + elif sxy == "y": dx = 0 dy = 1 - elif sxy == 'both': + elif sxy == "both": dx = 1 dy = 1 - - img_sobel = cv2.Sobel(src=img_blur, ddepth=cv2.CV_8U, dx=dx, dy=dy, ksize=5) + + img_sobel = cv2.Sobel( + src=img_blur, ddepth=cv2.CV_8U, dx=dx, dy=dy, ksize=5 + ) else: - img_sobel = img_gray + img_sobel = img_gray + self.add_output(img_sobel, "Sobel_edge") - # self.add_output(img_sobel, F"Sobel Edge\n nz={cv2.countNonZero(img_sobel)}") - + log.add("Sobel nonzero", cv2.countNonZero(img_sobel)) + # Canny edge - img_canny = cv2.Canny(image=img_blur,threshold1=ct1,threshold2=ct2) + img_canny = cv2.Canny(image=img_blur, threshold1=ct1, threshold2=ct2) self.add_output(img_canny, "Canny_edge") # BGR @@ -347,35 +403,34 @@ class MainApp: self.add_output(img[:, :, 2], "BGR_R") if img is not None: - self.drawHist(img, ('B', 'G', 'R'), 0, 0) - self.writeStats(img, ('B', 'G', 'R'), 0, 0) - + self.drawHist(img, ("B", "G", "R"), 0, 0) + self.writeStats(img, ("B", "G", "R"), 0, 0) + # HSV img_hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV) self.add_output(img_hsv, "HSV") - self.add_output(img_hsv[:, :, 0], "HSV_H") # H - self.add_output(img_hsv[:, :, 1], "HSV_S") # S - self.add_output(img_hsv[:, :, 2], "HSV_V") # V - + self.add_output(img_hsv[:, :, 0], "HSV_H") # H + self.add_output(img_hsv[:, :, 1], "HSV_S") # S + self.add_output(img_hsv[:, :, 2], "HSV_V") # V + if not part_update: if img_hsv is not None: - self.drawHist(img_hsv, ('H', 'S', 'V'), 0, 1) - self.writeStats(img_hsv, ('H', 'S', 'V'), 0, 1) - + self.drawHist(img_hsv, ("H", "S", "V"), 0, 1) + self.writeStats(img_hsv, ("H", "S", "V"), 0, 1) + # Canny Heatmap if not part_update: self.drawCannyHM(img, 1, 1) - + # Write results to CSV file if not part_update: - # if self.log. - # Only do this if the image has changed - self.log.write(F"{self.img_name[1]}, test\n") - + log.update() + # Show all data - plt.show(block=False) ## Graphs + plt.show(block=False) ## Graphs self.draw_output(size) ## Images + if __name__ == "__main__": app = MainApp() app.run()