Fixed double code and scaling for dectree
This commit is contained in:
parent
4aa97d72a5
commit
49fd6576a0
|
@ -53,6 +53,12 @@ $ ./.venv/Scripts/activate(.bat/.ps1)
|
||||||
$ pip install -r ./requirements.txt
|
$ pip install -r ./requirements.txt
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### Fix relative imports
|
||||||
|
1. Install the local package as editable using `pip`:
|
||||||
|
```sh
|
||||||
|
$ pip install -e .
|
||||||
|
```
|
||||||
|
|
||||||
### Create a dataset
|
### Create a dataset
|
||||||
1. Rename all images to include a tag and unique id, seperated by an underscore '_'
|
1. Rename all images to include a tag and unique id, seperated by an underscore '_'
|
||||||
- e.g. `accasia_1210262`
|
- e.g. `accasia_1210262`
|
||||||
|
@ -84,6 +90,9 @@ $ python ./src/helpers/test/knn.py -i ./out/result-(date/time).csv -o ./out/mode
|
||||||
```
|
```
|
||||||
4. Edit your `config.json` to include the newly created model
|
4. Edit your `config.json` to include the newly created model
|
||||||
|
|
||||||
|
> :memo: **Please note:**<br>
|
||||||
|
> The KNN Training script also generates the scaler required to make the decision tree model
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
Arne van Iterson<br>
|
Arne van Iterson<br>
|
||||||
|
|
BIN
requirements.txt
BIN
requirements.txt
Binary file not shown.
|
@ -0,0 +1,2 @@
|
||||||
|
from setuptools import setup, find_packages
|
||||||
|
setup(name='CVSuite', version='1.0', packages=find_packages())
|
|
@ -0,0 +1,2 @@
|
||||||
|
from .logger import *
|
||||||
|
from .tags import *
|
|
@ -7,27 +7,14 @@ import csv
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
from ..tags import Tree
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(prog='DecisionTree CLI')
|
parser = argparse.ArgumentParser(prog='DecisionTree CLI')
|
||||||
parser.add_argument('-i', '--input', help='Input CSV file', required=True)
|
parser.add_argument('-i', '--input', help='Input CSV file', required=True)
|
||||||
parser.add_argument('-o', '--output', help='Output model folder', required=True)
|
parser.add_argument('-o', '--output', help='Output model folder', required=True)
|
||||||
parser.add_argument(
|
parser.add_argument('-m', '--model', help='Chosen model (\'dectree\', \'randforest\' or \'extratree\')', required=True)
|
||||||
'-m',
|
|
||||||
'--model',
|
|
||||||
help='Chosen model (\'dectree\', \'randforest\' or \'extratree\')',
|
|
||||||
required=True
|
|
||||||
)
|
|
||||||
parser.add_argument('-s', '--scaler', help='Scaler preprocesser', required=True)
|
parser.add_argument('-s', '--scaler', help='Scaler preprocesser', required=True)
|
||||||
|
|
||||||
class Tree(Enum):
|
|
||||||
ACCASIA = 0
|
|
||||||
BERK = 1
|
|
||||||
EIK = 2
|
|
||||||
ELS = 3
|
|
||||||
ESDOORN = 4
|
|
||||||
ES = 5
|
|
||||||
LINDE = 6
|
|
||||||
PLATAAN = 7
|
|
||||||
|
|
||||||
class CVSuiteTestTree:
|
class CVSuiteTestTree:
|
||||||
def __init__(self, model_path = None):
|
def __init__(self, model_path = None):
|
||||||
self.model_path = model_path
|
self.model_path = model_path
|
||||||
|
@ -40,12 +27,11 @@ class CVSuiteTestTree:
|
||||||
reader = csv.reader(file, delimiter= ',')
|
reader = csv.reader(file, delimiter= ',')
|
||||||
matrix = list(reader)
|
matrix = list(reader)
|
||||||
|
|
||||||
i = 0
|
|
||||||
labels = []
|
labels = []
|
||||||
data = [[] for x in range(len(matrix)-1)]
|
data = [[] for x in range(len(matrix)-1)]
|
||||||
|
|
||||||
# Load all but the headers
|
# Load all but the headers
|
||||||
for row in matrix[1:]:
|
for ridx, row in enumerate(matrix[1:]):
|
||||||
## append data to lists
|
## append data to lists
|
||||||
label = row.pop(0).upper()
|
label = row.pop(0).upper()
|
||||||
labels.append(Tree[label].value)
|
labels.append(Tree[label].value)
|
||||||
|
@ -54,12 +40,12 @@ class CVSuiteTestTree:
|
||||||
row.pop(0)
|
row.pop(0)
|
||||||
|
|
||||||
# append all but ID and tree
|
# append all but ID and tree
|
||||||
for element in row:
|
for idx, element in enumerate(row):
|
||||||
data[i].append(float(element))
|
value = self.scaler[idx].transform([[float(element)]])
|
||||||
i += 1
|
data[ridx].append(value[0][0])
|
||||||
|
|
||||||
# normalize data
|
# normalize data has been included in code above :D
|
||||||
#TODO: Arne help
|
#TODO: Check if data is normalized correctly
|
||||||
|
|
||||||
# train model
|
# train model
|
||||||
self.train(data, labels, output)
|
self.train(data, labels, output)
|
||||||
|
|
|
@ -9,20 +9,13 @@ import joblib
|
||||||
import datetime
|
import datetime
|
||||||
import os
|
import os
|
||||||
|
|
||||||
|
from ..logger import C_DBUG
|
||||||
|
from ..tags import Tree
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(prog='KNN Train CLI')
|
parser = argparse.ArgumentParser(prog='KNN Train CLI')
|
||||||
parser.add_argument('-i', '--input', help='Input CSV file', required=True)
|
parser.add_argument('-i', '--input', help='Input CSV file', required=True)
|
||||||
parser.add_argument('-o', '--output', help='Output model file', required=True)
|
parser.add_argument('-o', '--output', help='Output model file', required=True)
|
||||||
|
|
||||||
class Tree(Enum):
|
|
||||||
ACCASIA = 0
|
|
||||||
BERK = 1
|
|
||||||
EIK = 2
|
|
||||||
ELS = 3
|
|
||||||
ESDOORN = 4
|
|
||||||
ES = 5
|
|
||||||
LINDE = 6
|
|
||||||
PLATAAN = 7
|
|
||||||
|
|
||||||
class CVSuiteTestKNN:
|
class CVSuiteTestKNN:
|
||||||
def __init__(self, model = None):
|
def __init__(self, model = None):
|
||||||
self.scale = []
|
self.scale = []
|
||||||
|
@ -113,7 +106,12 @@ class CVSuiteTestKNN:
|
||||||
self.knn.save(os.path.join(output, F"model_knn_{now.strftime('%Y-%m-%dT%H.%M.%S')}.yaml"))
|
self.knn.save(os.path.join(output, F"model_knn_{now.strftime('%Y-%m-%dT%H.%M.%S')}.yaml"))
|
||||||
|
|
||||||
def predict(self, data, nr = 3):
|
def predict(self, data, nr = 3):
|
||||||
return self.knn.findNearest(data, nr)
|
ret, results, neighbours ,dist = self.knn.findNearest(data, nr)
|
||||||
|
print(C_DBUG, "KNN Raw:")
|
||||||
|
print("\t\tresult: \t{}".format(results) )
|
||||||
|
print("\t\tneighbours:\t{}".format(neighbours) )
|
||||||
|
print("\t\tdistance:\t{}".format(dist) )
|
||||||
|
return neighbours[0]
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
105
src/suite.py
105
src/suite.py
|
@ -28,14 +28,13 @@ import matplotlib.pyplot as plt
|
||||||
|
|
||||||
# Helpers
|
# Helpers
|
||||||
from helpers.statistics import imgStats
|
from helpers.statistics import imgStats
|
||||||
from helpers.logger import CVSuiteLogger, C_DBUG, C_WARN
|
from helpers.logger import CVSuiteLogger, C_INFO, C_DBUG, C_WARN, C_DONE
|
||||||
from helpers.canvas import CVSuiteCanvas
|
from helpers.canvas import CVSuiteCanvas
|
||||||
from helpers.sift import getSiftData
|
from helpers.sift import getSiftData
|
||||||
from helpers.tags import Tree
|
from helpers.tags import Tree
|
||||||
|
|
||||||
# Tests
|
# Tests
|
||||||
from helpers.test.knn import CVSuiteTestKNN
|
from helpers.test.knn import CVSuiteTestKNN
|
||||||
|
|
||||||
from helpers.test.decision_tree import CVSuiteTestDecisionTree
|
from helpers.test.decision_tree import CVSuiteTestDecisionTree
|
||||||
|
|
||||||
## UI config load
|
## UI config load
|
||||||
|
@ -48,10 +47,18 @@ CONFIG_PATH = "./src/config/config.json"
|
||||||
config_file = open(CONFIG_PATH, encoding="utf-8")
|
config_file = open(CONFIG_PATH, encoding="utf-8")
|
||||||
config_json = json.load(config_file)
|
config_json = json.load(config_file)
|
||||||
|
|
||||||
|
print(
|
||||||
|
f"""
|
||||||
|
Welcome to CVSuite aka {TITLE}!
|
||||||
|
Powered by ARNweb.nl and TomSelier.com
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
## UI class setup
|
## UI class setup
|
||||||
class CVSuite:
|
class CVSuite:
|
||||||
def __init__(self, master=None):
|
def __init__(self, master=None):
|
||||||
|
### UI setup ###
|
||||||
# Pygubu builder
|
# Pygubu builder
|
||||||
self.builder = builder = pygubu.Builder()
|
self.builder = builder = pygubu.Builder()
|
||||||
builder.add_resource_path(PROJECT_PATH)
|
builder.add_resource_path(PROJECT_PATH)
|
||||||
|
@ -102,28 +109,45 @@ class CVSuite:
|
||||||
)
|
)
|
||||||
builder.connect_callbacks(self)
|
builder.connect_callbacks(self)
|
||||||
|
|
||||||
|
# Load values from config after UI has been initialised
|
||||||
|
self.img_path.set(config_json["path"])
|
||||||
|
self.img_size.set(config_json["size"])
|
||||||
|
|
||||||
|
### End of UI setup ###
|
||||||
|
|
||||||
|
### Model tests setup ###
|
||||||
# Attempt to load scaler
|
# Attempt to load scaler
|
||||||
if config_json["scaler"] != "":
|
if config_json["scaler"] != "":
|
||||||
self.scaler = joblib.load(config_json["scaler"])
|
self.scaler = joblib.load(config_json["scaler"])
|
||||||
else:
|
else:
|
||||||
self.scaler = None
|
self.scaler = None
|
||||||
|
|
||||||
# Model tests
|
self.models = []
|
||||||
if self.scaler is not None and config_json["models"]["knn"] != "":
|
if self.scaler is not None:
|
||||||
self.test_knn = CVSuiteTestKNN(config_json["models"]["knn"])
|
for model in config_json["models"]:
|
||||||
else:
|
if config_json["models"][model] != "":
|
||||||
self.test_knn = None
|
print(C_INFO, f"Loading model {model}")
|
||||||
|
if model == "knn":
|
||||||
|
self.models.append(
|
||||||
|
("KNN", CVSuiteTestKNN(config_json["models"]["knn"]))
|
||||||
|
)
|
||||||
|
elif model == "dectree":
|
||||||
|
self.models.append(
|
||||||
|
(
|
||||||
|
"Decision Tree",
|
||||||
|
CVSuiteTestDecisionTree(
|
||||||
|
config_json["models"]["dectree"]
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
print(C_WARN, f"Model {model} does not exist!")
|
||||||
|
else:
|
||||||
|
print(C_WARN, f"Model {model} not configured!")
|
||||||
|
print(C_DONE, f"{len(self.models)} models loaded!")
|
||||||
|
|
||||||
if config_json["models"]["dectree"] != "":
|
print(C_DONE, "CVSuite initialised!\n")
|
||||||
self.test_dectree = CVSuiteTestDecisionTree(
|
### End of model tests setup ###
|
||||||
config_json["models"]["dectree"]
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
self.test_dectree = None
|
|
||||||
|
|
||||||
# Load values from config after UI has been initialised
|
|
||||||
self.img_path.set(config_json["path"])
|
|
||||||
self.img_size.set(config_json["size"])
|
|
||||||
|
|
||||||
def on_quit(self, event=None):
|
def on_quit(self, event=None):
|
||||||
"""
|
"""
|
||||||
|
@ -311,7 +335,7 @@ class CVSuite:
|
||||||
# Don't run the test if there's no scaler
|
# Don't run the test if there's no scaler
|
||||||
if self.scaler is None:
|
if self.scaler is None:
|
||||||
return
|
return
|
||||||
|
|
||||||
output = self.builder.get_object("testdata")
|
output = self.builder.get_object("testdata")
|
||||||
output.configure(state="normal")
|
output.configure(state="normal")
|
||||||
output.delete(1.0, "end")
|
output.delete(1.0, "end")
|
||||||
|
@ -319,42 +343,26 @@ class CVSuite:
|
||||||
# Remove tag and photoId
|
# Remove tag and photoId
|
||||||
tag = data.pop(0)
|
tag = data.pop(0)
|
||||||
photoId = data.pop(1)
|
photoId = data.pop(1)
|
||||||
|
|
||||||
# Add actual name
|
# Add actual name
|
||||||
output.insert("end", f"Actual:\n\t{tag.upper()}\n")
|
output.insert("end", f"Actual type:\n\t{tag.upper()}\n")
|
||||||
|
|
||||||
# Normalise data using loaded scalers
|
# Normalise data using loaded scalers
|
||||||
for idx, value in enumerate(data):
|
for idx, value in enumerate(data):
|
||||||
d = np.array(value)
|
d = np.array(value)
|
||||||
data[idx] = self.scaler[idx].transform(d.astype(np.float32).reshape(1, -1))[0][0]
|
data[idx] = self.scaler[idx].transform(d.astype(np.float32).reshape(1, -1))[0][0]
|
||||||
|
|
||||||
data = np.array([data], dtype=np.float32)
|
data = np.array([data], dtype=np.float32)
|
||||||
|
|
||||||
if self.test_knn is not None:
|
|
||||||
# Do knn test
|
|
||||||
output.insert("end", "KNN Result:\n")
|
|
||||||
|
|
||||||
ret, results, neighbours ,dist = self.test_knn.predict(data)
|
for name, ins in self.models:
|
||||||
|
output.insert("end", f"{name} Result:\n")
|
||||||
|
|
||||||
for idx, res_id in enumerate(neighbours[0]):
|
# Predict result using model instance
|
||||||
output.insert("end", f" {idx}:\t{Tree(res_id).name}\n")
|
result = ins.predict(data)
|
||||||
|
|
||||||
print(C_DBUG, "KNN Result:")
|
|
||||||
print("\t\tresult: \t{}".format(results) )
|
|
||||||
print("\t\tneighbours:\t{}".format(neighbours) )
|
|
||||||
print("\t\tdistance:\t{}".format(dist) )
|
|
||||||
else:
|
|
||||||
print(C_WARN, "KNN Model not configured!")
|
|
||||||
|
|
||||||
if self.test_dectree is not None:
|
# Prediciton result should be an array
|
||||||
result = self.test_dectree.predict(data)
|
for idx, value in enumerate(result):
|
||||||
output.insert("end", "Decision Tree Result:\n")
|
output.insert("end", f" [{idx + 1}]\t{Tree(value).name}\n")
|
||||||
output.insert("end", f"\t{Tree(result).name}\n")
|
|
||||||
|
|
||||||
print(C_DBUG, "Decision Tree Result:")
|
|
||||||
print("\t\t result: \t{}".format(result))
|
|
||||||
else:
|
|
||||||
print(C_WARN, "Decison Tree Model not configured!")
|
|
||||||
|
|
||||||
output.configure(state="disabled")
|
output.configure(state="disabled")
|
||||||
|
|
||||||
|
@ -381,9 +389,8 @@ class CVSuite:
|
||||||
|
|
||||||
def update(self, event=None, part_update=None):
|
def update(self, event=None, part_update=None):
|
||||||
## Check if hist and canny hm have to be rerendered
|
## Check if hist and canny hm have to be rerendered
|
||||||
if (
|
if part_update is None:
|
||||||
part_update is None
|
## If partial update has not been forced, check if full update is required
|
||||||
): ## If partial update has not been forced, check if full update is required
|
|
||||||
if self.img_current != self.img_old or self.img_size != self.img_size_old:
|
if self.img_current != self.img_old or self.img_size != self.img_size_old:
|
||||||
part_update = False
|
part_update = False
|
||||||
self.img_old = self.img_current
|
self.img_old = self.img_current
|
||||||
|
@ -397,7 +404,7 @@ class CVSuite:
|
||||||
print("Full update forced!")
|
print("Full update forced!")
|
||||||
|
|
||||||
if self.updatePath():
|
if self.updatePath():
|
||||||
print(C_DBUG, f"Processing {self.img_name}")
|
print(C_INFO, f"Processing {self.img_name}")
|
||||||
|
|
||||||
self.mainwindow.title(f"{TITLE} - {self.img_name}")
|
self.mainwindow.title(f"{TITLE} - {self.img_name}")
|
||||||
self.log.add("Tree", self.img_name.split("_")[0])
|
self.log.add("Tree", self.img_name.split("_")[0])
|
||||||
|
|
Loading…
Reference in New Issue