Commit 07f64a57 authored by sjjsmuel's avatar sjjsmuel

general cleanup

parent 90da0f19
from helpers.AnnotationLocationLoader import AnnotationLocationLoader
'''
This script is able to load the annotations for each part of the dataset separately and report a few stats about them.
'''
def get_stats(annotation_loader):
bilder = annotation_loader.get_all_annotated_images()
bilder_count = len(annotation_loader.get_all_available_images())
bilder_mit_annotation = 0
summe_der_annotationen = 0
for image in bilder:
annot_per_image = len(annotation_loader.get_annotations(image, ['Caries']))
summe_der_annotationen += annot_per_image
if annot_per_image > 0:
bilder_mit_annotation += 1
print("Anzahl Bilder: {}".format(bilder_count))
print("Bilder mit Caries Annotationen: {}".format(bilder_mit_annotation))
print("Summe der Annotationen: {}".format(summe_der_annotationen))
print()
if __name__ == '__main__':
print('Test')
annotLoader_test = AnnotationLocationLoader(annotation_file='input/caries_dataset_annotation.json',
images_base_folder='input/test_data',
mouth_annotations_folder='input/mouth_annotations')
get_stats(annotLoader_test)
print('Training')
annotLoader_train = AnnotationLocationLoader(annotation_file='input/caries_dataset_annotation.json',
images_base_folder='input/training_data',
mouth_annotations_folder='input/mouth_annotations')
get_stats(annotLoader_train)
print('Evaluation')
annotLoader_eval = AnnotationLocationLoader(annotation_file='input/caries_dataset_annotation.json',
images_base_folder='input/evaluation_data',
mouth_annotations_folder='input/mouth_annotations')
get_stats(annotLoader_eval)
import tensorflow as tf
from pathlib import Path
import numpy as np
from PIL import Image
import cv2
from helpers.AnnotationLocationLoader import AnnotationLocationLoader
from helpers.PredictionLocationLoader import PredictionLocationLoader
from optparse import OptionParser
def run_inference_for_single_image(model, image):
'''
Inference function for single image based on function from TF Object Detection API
https://github.com/tensorflow/models/blob/master/research/object_detection/colab_tutorials/object_detection_tutorial.ipynb
'''
image = np.asarray(image)
# The input needs to be a tensor, convert it using `tf.convert_to_tensor`.
input_tensor = tf.convert_to_tensor(image)
# The model expects a batch of images, so add an axis with `tf.newaxis`.
input_tensor = input_tensor[tf.newaxis, ...]
# Run inference
model_fn = model.signatures['serving_default']
output_dict = model_fn(input_tensor)
# All outputs are batches tensors.
# Convert to numpy arrays, and take index [0] to remove the batch dimension.
# We're only interested in the first num_detections.
num_detections = int(output_dict.pop('num_detections'))
output_dict = {key: value[0, :num_detections].numpy()
for key, value in output_dict.items()}
output_dict['num_detections'] = num_detections
# detection_classes should be ints.
output_dict['detection_classes'] = output_dict['detection_classes'].astype(np.int64)
return output_dict
def save_inference(model, image_path, output_path, annotation_loader):
"""
Run the inference for an image, predict localizations, create visualisations and save everything to file
:param model: Faster R-CNN model
:param image_path: path the image for which to run the inference
:param output_path: path to base output folder
:param annotation_loader: helper to get the corresponding ground truth
:return: None - save results to disk
"""
# get image as numpy array
filename = image_path.name
image_np = np.array(Image.open(image_path))
# run inference
output_dict = run_inference_for_single_image(model, image_np)
selected_indices = tf.image.non_max_suppression(output_dict['detection_boxes'], output_dict['detection_scores'], 20, 0.3)
img = cv2.imread(str(image_path), cv2.IMREAD_IGNORE_ORIENTATION | cv2.IMREAD_COLOR)
height, width, _ = img.shape
# draw the annotated labels to the original picture
annotations = annotation_loader.get_annotations(filename, ['caries'])
for annotation in annotations:
cv2.rectangle(img, annotation[1][0], annotation[1][1], (0, 255, 0), 3)
prepared_boxes = []
for i, detection_score in enumerate(output_dict['detection_scores']):
if i not in selected_indices:
continue
if detection_score < 0.5:
continue
# ymin, xmin, ymax, xmax as relative to the image.
bbox = output_dict['detection_boxes'][i]
ymin = int(round(bbox[0] * height))
xmin = int(round(bbox[1] * width))
ymax = int(round(bbox[2] * height))
xmax = int(round(bbox[3] * width))
cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (255, 255, 255), 3)
class_prediction = output_dict['detection_classes'][i]
class_prediction = index_class_map[class_prediction]
class_score = output_dict['detection_scores'][i]
label = "{} {:.2f}%".format(class_prediction, (class_score*100))
cv2.putText(img, label, (xmin, (ymin - 5)), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2)
box = (class_prediction, [(xmin, ymin), (xmax, ymax), class_score])
prepared_boxes.append(box)
with open(str(output_path.parent/"predictions.txt"), "a") as predictions_file:
predictions_file.write('{}; {}\n'.format(filename, str(prepared_boxes)))
cv2.imwrite(str(output_path/filename), img)
parser = OptionParser()
parser.add_option("-i", "--path_evaluation", dest="evaluation_path", help="Path to evaluation input.", default="./input/evaluation_data")
parser.add_option("--model_folder", dest="model_folder", help="Path to the model to evaluate.", default="./out/trained_models/frcnn_model/exported_model")
parser.add_option("-o", "--path_output", dest="output_path", help="Path to base folder for output data.", default='./out')
(options, args) = parser.parse_args()
evaluation_folder = Path(options.evaluation_path)
frcnn_checkpoint_folder = Path(options.model_folder)
frcnn_output_folder = Path(options.output_path) / 'evaluation_frcnn'
if not frcnn_output_folder.exists():
frcnn_output_folder.mkdir()
class_index_map = {'caries': 1, 'no_caries': 2}
index_class_map = {}
for element in class_index_map:
index_class_map[class_index_map[element]] = element
annotation_loader = AnnotationLocationLoader(images_base_folder=evaluation_folder)
# clean up predicted boxes from last run
with open(str(frcnn_output_folder/"predictions.txt"), "w") as predictions_file:
predictions_file.write("")
model_dir = frcnn_checkpoint_folder / "saved_model"
model = tf.saved_model.load(str(model_dir))
# run inference for all images in the evaluation folder
for path in [path for path in evaluation_folder.iterdir() if path.is_dir()]:
# define an output folder for each class
out_path = Path(frcnn_output_folder) / path.name
if not out_path.exists():
out_path.mkdir()
# get the filenames of all test-images in the current folder
filenames = [item for item in path.glob('*') if item.name != '.DS_Store']
for img in filenames:
save_inference(model, img, out_path, annotation_loader)
print("[INFO] Finished {}/{}".format(img.parent.name, img.name))
# reload currently saved predictions for evaluation the classification
prediction_loader = PredictionLocationLoader(prediction_file=str(frcnn_output_folder/'predictions.txt'), images_base_folder=evaluation_folder)
tp = 0
fp = 0
fn = 0
tn = 0
fp_list = []
fn_list = []
# evaluate the classification capabilities of the Faster R-CNN model
for path in [path for path in evaluation_folder.iterdir() if path.is_dir()]:
for image in [item for item in path.glob('*') if item.name != '.DS_Store']:
predictions = prediction_loader.get_annotations(str(image.name))
if len(predictions) > 0: #prediction is positive
if path.name == 'caries':
tp += 1
else:
fp += 1
fp_list.append(str(image.name))
else: # prediction is negative
if path.name == 'caries':
fn += 1
else:
tn += 1
fn_list.append(str(image.name))
p = float(tp) / (tp + fp)
r = float(tp) / (tp + fn)
f1 = (2 * p * r) / (p + r)
# save and print the classification results
with open(str(frcnn_output_folder/'classification_results.txt'), 'w') as out_file:
out_file.write("Klassifikationsergebnisse für " + str(frcnn_checkpoint_folder)+"\n")
out_file.write("\n")
out_file.write("tp " + str(tp)+"\n")
out_file.write("fp " + str(fp)+"\n")
out_file.write("fn " + str(fn)+"\n")
out_file.write("tn " + str(tn)+"\n")
out_file.write("Precision " + str(p)+"\n")
out_file.write("Recall " + str(r)+"\n")
out_file.write("F1-Measure " + str(f1)+"\n")
out_file.write("\n")
out_file.write("False Positive Bilder:\n")
out_file.write(str(fp_list))
out_file.write("\n")
out_file.write("False Negative Bilder:\n")
out_file.write(str(fn_list))
print('Results for the Faster R-CNN as Classifier:')
print("Precision " + str(p))
print("Recall " + str(r))
print("F1-Measure " + str(f1))
print(tp, fn)
print(fp, tn)
print("[INFO] Evaluation finished.")
\ No newline at end of file
...@@ -2,37 +2,39 @@ import json ...@@ -2,37 +2,39 @@ import json
from pathlib import Path, PosixPath from pathlib import Path, PosixPath
class AnnotationLocationLoader: class AnnotationLocationLoader:
_annotation_file = None def __init__(self, annotation_file='input/caries_dataset_annotation.json', images_base_folder=Path('input/test_data/'), mouth_annotations_folder=None):
_img_path = None
_mouth_annotations_folder = None
_annotated_images = set()
_available_annotations = set()
_available_images = None
_data = {}
def __init__(self, annotation_file='input/caries_dataset_annotation.json', images_base_folder=Path('input/test_data/'), mouth_annotations_folder=Path('input/mouth_annotations/')):
self._annotation_file = annotation_file self._annotation_file = annotation_file
self._img_path = None
self._annotated_images = set()
self._available_annotations = set()
self._available_images = None
self._data = {}
if not type(images_base_folder) == PosixPath: if not type(images_base_folder) == PosixPath:
images_base_folder = Path(images_base_folder) images_base_folder = Path(images_base_folder)
self._img_path = images_base_folder self._img_path = images_base_folder
if not type(mouth_annotations_folder) == PosixPath:
mouth_annotations_folder = Path(mouth_annotations_folder)
self._mouth_annotations_folder = mouth_annotations_folder
# get the names of the images witch are available as files # get the names of the images witch are available as files
self._available_images = self._get_names_from_available_images() self._available_images = self._get_names_from_available_images()
self._load_annotations() self._load_annotations()
self._load_mouth_annotation_additon()
if not mouth_annotations_folder == None:
if not type(mouth_annotations_folder) == PosixPath:
mouth_annotations_folder = Path(mouth_annotations_folder)
self._mouth_annotations_folder = mouth_annotations_folder
self._load_mouth_annotation_additon()
self._annotated_images = list(self._annotated_images) self._annotated_images = list(self._annotated_images)
self._available_annotations = list(self._available_annotations) self._available_annotations = list(self._available_annotations)
def _get_names_from_available_images(self): def _get_names_from_available_images(self):
"""
Finds the names for all pictures available in the image base folder.
:return:
"""
names_from_available_images = [] names_from_available_images = []
for path in [path for path in self._img_path.iterdir() if path.is_dir()]: for path in [path for path in self._img_path.iterdir() if path.is_dir()]:
names_from_available_images.extend([filename.name for filename in path.iterdir() if filename.is_file() and not filename.name.startswith('.')]) names_from_available_images.extend([filename.name for filename in path.iterdir() if filename.is_file() and not filename.name.startswith('.')])
...@@ -40,6 +42,9 @@ class AnnotationLocationLoader: ...@@ -40,6 +42,9 @@ class AnnotationLocationLoader:
def _load_annotations(self): def _load_annotations(self):
"""
Loads all annotations for the available images from the json file to an internal structure.
"""
with open(self._annotation_file) as file: with open(self._annotation_file) as file:
json_data = json.load(file) json_data = json.load(file)
...@@ -71,6 +76,10 @@ class AnnotationLocationLoader: ...@@ -71,6 +76,10 @@ class AnnotationLocationLoader:
def _load_mouth_annotation_additon(self): def _load_mouth_annotation_additon(self):
"""
Loads the additional annotations for the mouth for all available images if there is a corresponding folder given
This represents the local constrains for the advanced training.
"""
annotation_files = [file for file in self._mouth_annotations_folder.iterdir() if file.is_file() and not file.name.startswith('.')] annotation_files = [file for file in self._mouth_annotations_folder.iterdir() if file.is_file() and not file.name.startswith('.')]
counter_number_of_annotated_but_missing_files = 0 counter_number_of_annotated_but_missing_files = 0
for annotation_file in annotation_files: for annotation_file in annotation_files:
...@@ -110,6 +119,12 @@ class AnnotationLocationLoader: ...@@ -110,6 +119,12 @@ class AnnotationLocationLoader:
""" """
return self._annotated_images return self._annotated_images
def get_all_available_images(self):
"""
:return: list of the names of all images witch have at least one annotation
"""
return self._available_images
def is_annotated(self, image_name): def is_annotated(self, image_name):
""" """
Should check weather for the given filename an annotation exists Should check weather for the given filename an annotation exists
......
...@@ -2,7 +2,6 @@ from tensorflow.keras import Model ...@@ -2,7 +2,6 @@ from tensorflow.keras import Model
from tensorflow import GradientTape, cast, reduce_mean, reduce_sum, multiply, newaxis, reshape, transpose, squeeze from tensorflow import GradientTape, cast, reduce_mean, reduce_sum, multiply, newaxis, reshape, transpose, squeeze
from tensorflow.image import resize from tensorflow.image import resize
from tensorflow.math import multiply, reduce_min, reduce_max, divide, add, l2_normalize from tensorflow.math import multiply, reduce_min, reduce_max, divide, add, l2_normalize
from tensorflow.linalg import matmul
import tensorflow as tf import tensorflow as tf
from tensorflow.keras.losses import categorical_crossentropy from tensorflow.keras.losses import categorical_crossentropy
from tensorflow.keras.metrics import CategoricalAccuracy, Mean from tensorflow.keras.metrics import CategoricalAccuracy, Mean
......
from helpers.AnnotationLocationLoader import AnnotationLocationLoader from helpers.AnnotationLocationLoader import AnnotationLocationLoader
from helpers.PredictionLocationLoader import PredictionLocationLoader from helpers.PredictionLocationLoader import PredictionLocationLoader
from pathlib import Path from pathlib import Path
from optparse import OptionParser
import shutil
def convertGroundtruths(out_path): def convertGroundtruths(out_path):
# create folder for gt # create folder for gt
...@@ -8,22 +10,24 @@ def convertGroundtruths(out_path): ...@@ -8,22 +10,24 @@ def convertGroundtruths(out_path):
if not out_path.exists(): if not out_path.exists():
out_path.mkdir(parents=True) out_path.mkdir(parents=True)
annotLoader = AnnotationLocationLoader(annotation_file='../input/caries_dataset_annotation.json', images_base_folder='../input/test_data', mouth_annotations_folder='../input/mouth_annotations') annotLoader = AnnotationLocationLoader(annotation_file='../input/caries_dataset_annotation.json', images_base_folder='../input/evaluation_data')
for image in annotLoader.get_all_annotated_images(): for image in annotLoader.get_all_available_images():
#create file #create file
file = image[:-3] + 'txt' file = image[:-3] + 'txt'
with open(out_path / file, "w") as groundtruth_file: with open(out_path / file, "w") as groundtruth_file:
for annotation in annotLoader.get_annotations(image): for annotation in annotLoader.get_annotations(image):
# format: <class_name> <left> <top> <right> <bottom> # format: <class_name> <left> <top> <right> <bottom>
if not str(annotation[0]) == 'caries':
continue
groundtruth_file.write(str(annotation[0]) + " " + str(annotation[1][0][0]) + " " + str(annotation[1][0][1]) + " " + str(annotation[1][1][0]) + " " + str(annotation[1][1][1]) + "\n") groundtruth_file.write(str(annotation[0]) + " " + str(annotation[1][0][0]) + " " + str(annotation[1][0][1]) + " " + str(annotation[1][1][0]) + " " + str(annotation[1][1][1]) + "\n")
def convertDetections(out_path): def convertDetections(out_path, predictions_file):
# create folder for detections # create folder for detections
out_path = out_path / 'detections' out_path = out_path / 'detections'
if not out_path.exists(): if not out_path.exists():
out_path.mkdir(parents=True) out_path.mkdir(parents=True)
predLoader = PredictionLocationLoader(prediction_file='../out/evaluation/predictions.txt', images_base_folder='../input/test_data') predLoader = PredictionLocationLoader(prediction_file=predictions_file, images_base_folder='../input/evaluation_data')
for image in predLoader.get_all_annotated_images(): for image in predLoader.get_all_annotated_images():
file = image[:-3] + 'txt' file = image[:-3] + 'txt'
with open(out_path / file, "w") as detection_file: with open(out_path / file, "w") as detection_file:
...@@ -33,14 +37,30 @@ def convertDetections(out_path): ...@@ -33,14 +37,30 @@ def convertDetections(out_path):
str(annotation[0]) + " " + str(annotation[1][2]) + " " + str(annotation[1][0][0]) + " " + str(annotation[1][0][1]) + " " + str( str(annotation[0]) + " " + str(annotation[1][2]) + " " + str(annotation[1][0][0]) + " " + str(annotation[1][0][1]) + " " + str(
annotation[1][1][0]) + " " + str(annotation[1][1][1]) + "\n") annotation[1][1][0]) + " " + str(annotation[1][1][1]) + "\n")
if __name__ == '__main__':
''' '''
Please run the Pascal VOC Metrics implemented by Rafael Padilla on the output of this Script. Please run the Pascal VOC Metrics implemented by Rafael Padilla on the output of this Script.
https://github.com/rafaelpadilla/Object-Detection-Metrics https://github.com/rafaelpadilla/Object-Detection-Metrics
''' '''
output_path = Path('../out/convertedForObjectDetectionMetrics')
convertGroundtruths(output_path) parser = OptionParser()
convertDetections(output_path)
print('[INFO] Finished conversion for ObjectDetectionMetrics.') parser.add_option("-i", "--predictions_file", dest="predictions_file", help="Path to predictions file as input.", default="../out/evaluation_ws_base/predictions.txt")
parser.add_option("-o", "--path_output", dest="output_path", help="Path to output folder.", default='../out/convertedForObjectDetectionMetrics_ws_base')
(options, args) = parser.parse_args()
if not Path(options.predictions_file).is_file():
parser.error('Error: Could not find the prediction file. Check the path passed to -i or --prediction_file')
output_path = Path(options.output_path)
shutil.rmtree(str(output_path))
predictions_file = options.predictions_file
convertGroundtruths(output_path)
convertDetections(output_path, predictions_file)
print('[INFO] Finished conversion for ObjectDetectionMetrics.')
print('[INFO] Saved to folder {}'.format(str(output_path)))
from tensorflow.keras import Model from tensorflow.keras import Model
from tensorflow import GradientTape, cast, reduce_mean, reduce_sum, multiply, newaxis, reshape, transpose, squeeze from tensorflow import GradientTape, cast, reduce_mean, reduce_sum, multiply, newaxis, reshape, transpose, squeeze
from tensorflow.image import resize
from tensorflow.math import multiply, reduce_min, reduce_max, divide, add, l2_normalize
from tensorflow.linalg import matmul
import tensorflow as tf
from tensorflow.keras.losses import categorical_crossentropy from tensorflow.keras.losses import categorical_crossentropy
from tensorflow.keras.metrics import CategoricalAccuracy, Mean from tensorflow.keras.metrics import CategoricalAccuracy, Mean
...@@ -16,6 +12,7 @@ metric_tracker = CategoricalAccuracy() ...@@ -16,6 +12,7 @@ metric_tracker = CategoricalAccuracy()
loss_tracker = Mean(name='loss') loss_tracker = Mean(name='loss')
class NoCAMModel(Model): class NoCAMModel(Model):
class_index_dict = {'caries': 0, 'no_caries': 1} # hardcoded classes
def train_step(self, data): def train_step(self, data):
# Unpack the data. Its structure depends on your model and # Unpack the data. Its structure depends on your model and
...@@ -23,7 +20,6 @@ class NoCAMModel(Model): ...@@ -23,7 +20,6 @@ class NoCAMModel(Model):
x, y = data x, y = data
img = x['img'] img = x['img']
mouth_filter = x['mouth']
with GradientTape() as tape: with GradientTape() as tape:
y_pred, conv_out = self(img, training=True) # Forward pass y_pred, conv_out = self(img, training=True) # Forward pass
......
import json
from pathlib import Path
import ast import ast
from pathlib import Path, PosixPath
class PredictionLocationLoader: class PredictionLocationLoader:
_prediction_file = None
_img_path = None
_annotated_images = set()
_available_annotations = set()
_available_images = None
_data = {}
def __init__(self, prediction_file='out/predictions.txt', images_base_folder='input/test_data/'): def __init__(self, prediction_file='out/predictions.txt', images_base_folder='input/test_data/'):
self._prediction_file = prediction_file self._prediction_file = prediction_file
self._img_path = None
self._annotated_images = set()
self._available_annotations = set()
self._available_images = None
self._data = {}
if not type(images_base_folder) == PosixPath:
images_base_folder = Path(images_base_folder)
self._img_path = images_base_folder self._img_path = images_base_folder
# get the names of the images witch are available as files # get the names of the images witch are available as files
...@@ -22,6 +21,9 @@ class PredictionLocationLoader: ...@@ -22,6 +21,9 @@ class PredictionLocationLoader:
self._load_predictions() self._load_predictions()
def _get_names_from_available_images(self): def _get_names_from_available_images(self):
"""
Finds the names for all pictures available in the image base folder.
"""
names_from_available_images = [] names_from_available_images = []
for path in [path for path in Path(self._img_path).iterdir() if path.is_dir()]: for path in [path for path in Path(self._img_path).iterdir() if path.is_dir()]:
names_from_available_images.extend([filename.name for filename in path.iterdir() if filename.is_file() and not filename.name.startswith('.')]) names_from_available_images.extend([filename.name for filename in path.iterdir() if filename.is_file() and not filename.name.startswith('.')])
...@@ -29,6 +31,9 @@ class PredictionLocationLoader: ...@@ -29,6 +31,9 @@ class PredictionLocationLoader:
def _load_predictions(self): def _load_predictions(self):
"""
Loads the predictions from a network based on the prediction file.
"""
with open(self._prediction_file) as file: with open(self._prediction_file) as file:
for line in file.readlines(): for line in file.readlines():
filename, predictions = line.split(';') filename, predictions = line.split(';')
......
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utility functions for creating TFRecord data sets."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow as tf
def int64_feature(value):
return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))