Commit 07f64a57 authored by sjjsmuel's avatar sjjsmuel

general cleanup

parent 90da0f19
from helpers.AnnotationLocationLoader import AnnotationLocationLoader
'''
This script is able to load the annotations for each part of the dataset separately and report a few stats about them.
'''
def get_stats(annotation_loader):
bilder = annotation_loader.get_all_annotated_images()
bilder_count = len(annotation_loader.get_all_available_images())
bilder_mit_annotation = 0
summe_der_annotationen = 0
for image in bilder:
annot_per_image = len(annotation_loader.get_annotations(image, ['Caries']))
summe_der_annotationen += annot_per_image
if annot_per_image > 0:
bilder_mit_annotation += 1
print("Anzahl Bilder: {}".format(bilder_count))
print("Bilder mit Caries Annotationen: {}".format(bilder_mit_annotation))
print("Summe der Annotationen: {}".format(summe_der_annotationen))
print()
if __name__ == '__main__':
print('Test')
annotLoader_test = AnnotationLocationLoader(annotation_file='input/caries_dataset_annotation.json',
images_base_folder='input/test_data',
mouth_annotations_folder='input/mouth_annotations')
get_stats(annotLoader_test)
print('Training')
annotLoader_train = AnnotationLocationLoader(annotation_file='input/caries_dataset_annotation.json',
images_base_folder='input/training_data',
mouth_annotations_folder='input/mouth_annotations')
get_stats(annotLoader_train)
print('Evaluation')
annotLoader_eval = AnnotationLocationLoader(annotation_file='input/caries_dataset_annotation.json',
images_base_folder='input/evaluation_data',
mouth_annotations_folder='input/mouth_annotations')
get_stats(annotLoader_eval)
import tensorflow as tf
from pathlib import Path
import numpy as np
from PIL import Image
import cv2
from helpers.AnnotationLocationLoader import AnnotationLocationLoader
from helpers.PredictionLocationLoader import PredictionLocationLoader
from optparse import OptionParser
def run_inference_for_single_image(model, image):
'''
Inference function for single image based on function from TF Object Detection API
https://github.com/tensorflow/models/blob/master/research/object_detection/colab_tutorials/object_detection_tutorial.ipynb
'''
image = np.asarray(image)
# The input needs to be a tensor, convert it using `tf.convert_to_tensor`.
input_tensor = tf.convert_to_tensor(image)
# The model expects a batch of images, so add an axis with `tf.newaxis`.
input_tensor = input_tensor[tf.newaxis, ...]
# Run inference
model_fn = model.signatures['serving_default']
output_dict = model_fn(input_tensor)
# All outputs are batches tensors.
# Convert to numpy arrays, and take index [0] to remove the batch dimension.
# We're only interested in the first num_detections.
num_detections = int(output_dict.pop('num_detections'))
output_dict = {key: value[0, :num_detections].numpy()
for key, value in output_dict.items()}
output_dict['num_detections'] = num_detections
# detection_classes should be ints.
output_dict['detection_classes'] = output_dict['detection_classes'].astype(np.int64)
return output_dict
def save_inference(model, image_path, output_path, annotation_loader):
"""
Run the inference for an image, predict localizations, create visualisations and save everything to file
:param model: Faster R-CNN model
:param image_path: path the image for which to run the inference
:param output_path: path to base output folder
:param annotation_loader: helper to get the corresponding ground truth
:return: None - save results to disk
"""
# get image as numpy array
filename = image_path.name
image_np = np.array(Image.open(image_path))
# run inference
output_dict = run_inference_for_single_image(model, image_np)
selected_indices = tf.image.non_max_suppression(output_dict['detection_boxes'], output_dict['detection_scores'], 20, 0.3)
img = cv2.imread(str(image_path), cv2.IMREAD_IGNORE_ORIENTATION | cv2.IMREAD_COLOR)
height, width, _ = img.shape
# draw the annotated labels to the original picture
annotations = annotation_loader.get_annotations(filename, ['caries'])
for annotation in annotations:
cv2.rectangle(img, annotation[1][0], annotation[1][1], (0, 255, 0), 3)
prepared_boxes = []
for i, detection_score in enumerate(output_dict['detection_scores']):
if i not in selected_indices:
continue
if detection_score < 0.5:
continue
# ymin, xmin, ymax, xmax as relative to the image.
bbox = output_dict['detection_boxes'][i]
ymin = int(round(bbox[0] * height))
xmin = int(round(bbox[1] * width))
ymax = int(round(bbox[2] * height))
xmax = int(round(bbox[3] * width))
cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (255, 255, 255), 3)
class_prediction = output_dict['detection_classes'][i]
class_prediction = index_class_map[class_prediction]
class_score = output_dict['detection_scores'][i]
label = "{} {:.2f}%".format(class_prediction, (class_score*100))
cv2.putText(img, label, (xmin, (ymin - 5)), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2)
box = (class_prediction, [(xmin, ymin), (xmax, ymax), class_score])
prepared_boxes.append(box)
with open(str(output_path.parent/"predictions.txt"), "a") as predictions_file:
predictions_file.write('{}; {}\n'.format(filename, str(prepared_boxes)))
cv2.imwrite(str(output_path/filename), img)
parser = OptionParser()
parser.add_option("-i", "--path_evaluation", dest="evaluation_path", help="Path to evaluation input.", default="./input/evaluation_data")
parser.add_option("--model_folder", dest="model_folder", help="Path to the model to evaluate.", default="./out/trained_models/frcnn_model/exported_model")
parser.add_option("-o", "--path_output", dest="output_path", help="Path to base folder for output data.", default='./out')
(options, args) = parser.parse_args()
evaluation_folder = Path(options.evaluation_path)
frcnn_checkpoint_folder = Path(options.model_folder)
frcnn_output_folder = Path(options.output_path) / 'evaluation_frcnn'
if not frcnn_output_folder.exists():
frcnn_output_folder.mkdir()
class_index_map = {'caries': 1, 'no_caries': 2}
index_class_map = {}
for element in class_index_map:
index_class_map[class_index_map[element]] = element
annotation_loader = AnnotationLocationLoader(images_base_folder=evaluation_folder)
# clean up predicted boxes from last run
with open(str(frcnn_output_folder/"predictions.txt"), "w") as predictions_file:
predictions_file.write("")
model_dir = frcnn_checkpoint_folder / "saved_model"
model = tf.saved_model.load(str(model_dir))
# run inference for all images in the evaluation folder
for path in [path for path in evaluation_folder.iterdir() if path.is_dir()]:
# define an output folder for each class
out_path = Path(frcnn_output_folder) / path.name
if not out_path.exists():
out_path.mkdir()
# get the filenames of all test-images in the current folder
filenames = [item for item in path.glob('*') if item.name != '.DS_Store']
for img in filenames:
save_inference(model, img, out_path, annotation_loader)
print("[INFO] Finished {}/{}".format(img.parent.name, img.name))
# reload currently saved predictions for evaluation the classification
prediction_loader = PredictionLocationLoader(prediction_file=str(frcnn_output_folder/'predictions.txt'), images_base_folder=evaluation_folder)
tp = 0
fp = 0
fn = 0
tn = 0
fp_list = []
fn_list = []
# evaluate the classification capabilities of the Faster R-CNN model
for path in [path for path in evaluation_folder.iterdir() if path.is_dir()]:
for image in [item for item in path.glob('*') if item.name != '.DS_Store']:
predictions = prediction_loader.get_annotations(str(image.name))
if len(predictions) > 0: #prediction is positive
if path.name == 'caries':
tp += 1
else:
fp += 1
fp_list.append(str(image.name))
else: # prediction is negative
if path.name == 'caries':
fn += 1
else:
tn += 1
fn_list.append(str(image.name))
p = float(tp) / (tp + fp)
r = float(tp) / (tp + fn)
f1 = (2 * p * r) / (p + r)
# save and print the classification results
with open(str(frcnn_output_folder/'classification_results.txt'), 'w') as out_file:
out_file.write("Klassifikationsergebnisse für " + str(frcnn_checkpoint_folder)+"\n")
out_file.write("\n")
out_file.write("tp " + str(tp)+"\n")
out_file.write("fp " + str(fp)+"\n")
out_file.write("fn " + str(fn)+"\n")
out_file.write("tn " + str(tn)+"\n")
out_file.write("Precision " + str(p)+"\n")
out_file.write("Recall " + str(r)+"\n")
out_file.write("F1-Measure " + str(f1)+"\n")
out_file.write("\n")
out_file.write("False Positive Bilder:\n")
out_file.write(str(fp_list))
out_file.write("\n")
out_file.write("False Negative Bilder:\n")
out_file.write(str(fn_list))
print('Results for the Faster R-CNN as Classifier:')
print("Precision " + str(p))
print("Recall " + str(r))
print("F1-Measure " + str(f1))
print(tp, fn)
print(fp, tn)
print("[INFO] Evaluation finished.")
\ No newline at end of file
......@@ -4,35 +4,37 @@ from pathlib import Path, PosixPath
class AnnotationLocationLoader:
_annotation_file = None
_img_path = None
_mouth_annotations_folder = None
_annotated_images = set()
_available_annotations = set()
_available_images = None
_data = {}
def __init__(self, annotation_file='input/caries_dataset_annotation.json', images_base_folder=Path('input/test_data/'), mouth_annotations_folder=Path('input/mouth_annotations/')):
def __init__(self, annotation_file='input/caries_dataset_annotation.json', images_base_folder=Path('input/test_data/'), mouth_annotations_folder=None):
self._annotation_file = annotation_file
self._img_path = None
self._annotated_images = set()
self._available_annotations = set()
self._available_images = None
self._data = {}
if not type(images_base_folder) == PosixPath:
images_base_folder = Path(images_base_folder)
self._img_path = images_base_folder
# get the names of the images witch are available as files
self._available_images = self._get_names_from_available_images()
self._load_annotations()
if not mouth_annotations_folder == None:
if not type(mouth_annotations_folder) == PosixPath:
mouth_annotations_folder = Path(mouth_annotations_folder)
self._mouth_annotations_folder = mouth_annotations_folder
# get the names of the images witch are available as files
self._available_images = self._get_names_from_available_images()
self._load_annotations()
self._load_mouth_annotation_additon()
self._annotated_images = list(self._annotated_images)
self._available_annotations = list(self._available_annotations)
def _get_names_from_available_images(self):
"""
Finds the names for all pictures available in the image base folder.
:return:
"""
names_from_available_images = []
for path in [path for path in self._img_path.iterdir() if path.is_dir()]:
names_from_available_images.extend([filename.name for filename in path.iterdir() if filename.is_file() and not filename.name.startswith('.')])
......@@ -40,6 +42,9 @@ class AnnotationLocationLoader:
def _load_annotations(self):
"""
Loads all annotations for the available images from the json file to an internal structure.
"""
with open(self._annotation_file) as file:
json_data = json.load(file)
......@@ -71,6 +76,10 @@ class AnnotationLocationLoader:
def _load_mouth_annotation_additon(self):
"""
Loads the additional annotations for the mouth for all available images if there is a corresponding folder given
This represents the local constrains for the advanced training.
"""
annotation_files = [file for file in self._mouth_annotations_folder.iterdir() if file.is_file() and not file.name.startswith('.')]
counter_number_of_annotated_but_missing_files = 0
for annotation_file in annotation_files:
......@@ -110,6 +119,12 @@ class AnnotationLocationLoader:
"""
return self._annotated_images
def get_all_available_images(self):
"""
:return: list of the names of all images witch have at least one annotation
"""
return self._available_images
def is_annotated(self, image_name):
"""
Should check weather for the given filename an annotation exists
......
......@@ -2,7 +2,6 @@ from tensorflow.keras import Model
from tensorflow import GradientTape, cast, reduce_mean, reduce_sum, multiply, newaxis, reshape, transpose, squeeze
from tensorflow.image import resize
from tensorflow.math import multiply, reduce_min, reduce_max, divide, add, l2_normalize
from tensorflow.linalg import matmul
import tensorflow as tf
from tensorflow.keras.losses import categorical_crossentropy
from tensorflow.keras.metrics import CategoricalAccuracy, Mean
......
from helpers.AnnotationLocationLoader import AnnotationLocationLoader
from helpers.PredictionLocationLoader import PredictionLocationLoader
from pathlib import Path
from optparse import OptionParser
import shutil
def convertGroundtruths(out_path):
# create folder for gt
......@@ -8,22 +10,24 @@ def convertGroundtruths(out_path):
if not out_path.exists():
out_path.mkdir(parents=True)
annotLoader = AnnotationLocationLoader(annotation_file='../input/caries_dataset_annotation.json', images_base_folder='../input/test_data', mouth_annotations_folder='../input/mouth_annotations')
for image in annotLoader.get_all_annotated_images():
annotLoader = AnnotationLocationLoader(annotation_file='../input/caries_dataset_annotation.json', images_base_folder='../input/evaluation_data')
for image in annotLoader.get_all_available_images():
#create file
file = image[:-3] + 'txt'
with open(out_path / file, "w") as groundtruth_file:
for annotation in annotLoader.get_annotations(image):
# format: <class_name> <left> <top> <right> <bottom>
if not str(annotation[0]) == 'caries':
continue
groundtruth_file.write(str(annotation[0]) + " " + str(annotation[1][0][0]) + " " + str(annotation[1][0][1]) + " " + str(annotation[1][1][0]) + " " + str(annotation[1][1][1]) + "\n")
def convertDetections(out_path):
def convertDetections(out_path, predictions_file):
# create folder for detections
out_path = out_path / 'detections'
if not out_path.exists():
out_path.mkdir(parents=True)
predLoader = PredictionLocationLoader(prediction_file='../out/evaluation/predictions.txt', images_base_folder='../input/test_data')
predLoader = PredictionLocationLoader(prediction_file=predictions_file, images_base_folder='../input/evaluation_data')
for image in predLoader.get_all_annotated_images():
file = image[:-3] + 'txt'
with open(out_path / file, "w") as detection_file:
......@@ -33,14 +37,30 @@ def convertDetections(out_path):
str(annotation[0]) + " " + str(annotation[1][2]) + " " + str(annotation[1][0][0]) + " " + str(annotation[1][0][1]) + " " + str(
annotation[1][1][0]) + " " + str(annotation[1][1][1]) + "\n")
if __name__ == '__main__':
'''
'''
Please run the Pascal VOC Metrics implemented by Rafael Padilla on the output of this Script.
https://github.com/rafaelpadilla/Object-Detection-Metrics
'''
output_path = Path('../out/convertedForObjectDetectionMetrics')
convertGroundtruths(output_path)
convertDetections(output_path)
print('[INFO] Finished conversion for ObjectDetectionMetrics.')
'''
parser = OptionParser()
parser.add_option("-i", "--predictions_file", dest="predictions_file", help="Path to predictions file as input.", default="../out/evaluation_ws_base/predictions.txt")
parser.add_option("-o", "--path_output", dest="output_path", help="Path to output folder.", default='../out/convertedForObjectDetectionMetrics_ws_base')
(options, args) = parser.parse_args()
if not Path(options.predictions_file).is_file():
parser.error('Error: Could not find the prediction file. Check the path passed to -i or --prediction_file')
output_path = Path(options.output_path)
shutil.rmtree(str(output_path))
predictions_file = options.predictions_file
convertGroundtruths(output_path)
convertDetections(output_path, predictions_file)
print('[INFO] Finished conversion for ObjectDetectionMetrics.')
print('[INFO] Saved to folder {}'.format(str(output_path)))
from tensorflow.keras import Model
from tensorflow import GradientTape, cast, reduce_mean, reduce_sum, multiply, newaxis, reshape, transpose, squeeze
from tensorflow.image import resize
from tensorflow.math import multiply, reduce_min, reduce_max, divide, add, l2_normalize
from tensorflow.linalg import matmul
import tensorflow as tf
from tensorflow.keras.losses import categorical_crossentropy
from tensorflow.keras.metrics import CategoricalAccuracy, Mean
......@@ -16,6 +12,7 @@ metric_tracker = CategoricalAccuracy()
loss_tracker = Mean(name='loss')
class NoCAMModel(Model):
class_index_dict = {'caries': 0, 'no_caries': 1} # hardcoded classes
def train_step(self, data):
# Unpack the data. Its structure depends on your model and
......@@ -23,7 +20,6 @@ class NoCAMModel(Model):
x, y = data
img = x['img']
mouth_filter = x['mouth']
with GradientTape() as tape:
y_pred, conv_out = self(img, training=True) # Forward pass
......
import json
from pathlib import Path
import ast
from pathlib import Path, PosixPath
class PredictionLocationLoader:
_prediction_file = None
_img_path = None
_annotated_images = set()
_available_annotations = set()
_available_images = None
_data = {}
def __init__(self, prediction_file='out/predictions.txt', images_base_folder='input/test_data/'):
self._prediction_file = prediction_file
self._img_path = None
self._annotated_images = set()
self._available_annotations = set()
self._available_images = None
self._data = {}
if not type(images_base_folder) == PosixPath:
images_base_folder = Path(images_base_folder)
self._img_path = images_base_folder
# get the names of the images witch are available as files
......@@ -22,6 +21,9 @@ class PredictionLocationLoader:
self._load_predictions()
def _get_names_from_available_images(self):
"""
Finds the names for all pictures available in the image base folder.
"""
names_from_available_images = []
for path in [path for path in Path(self._img_path).iterdir() if path.is_dir()]:
names_from_available_images.extend([filename.name for filename in path.iterdir() if filename.is_file() and not filename.name.startswith('.')])
......@@ -29,6 +31,9 @@ class PredictionLocationLoader:
def _load_predictions(self):
"""
Loads the predictions from a network based on the prediction file.
"""
with open(self._prediction_file) as file:
for line in file.readlines():
filename, predictions = line.split(';')
......
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Utility functions for creating TFRecord data sets."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import tensorflow as tf
def int64_feature(value):
return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))
def int64_list_feature(value):
return tf.train.Feature(int64_list=tf.train.Int64List(value=value))
def bytes_feature(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
def bytes_list_feature(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=value))
def float_list_feature(value):
return tf.train.Feature(float_list=tf.train.FloatList(value=value))
def read_examples_list(path):
"""Read list of training or validation examples.
The file is assumed to contain a single example per line where the first
token in the line is an identifier that allows us to find the image and
annotation xml for that example.
For example, the line:
xyz 3
would allow us to find files xyz.jpg and xyz.xml (the 3 would be ignored).
Args:
path: absolute path to examples list file.
Returns:
list of example identifiers (strings).
"""
with tf.gfile.GFile(path) as fid:
lines = fid.readlines()
return [line.strip().split(' ')[0] for line in lines]
def recursive_parse_xml_to_dict(xml):
"""Recursively parses XML contents to python dict.
We assume that `object` tags are the only ones that can appear
multiple times at the same level of a tree.
Args:
xml: xml tree obtained by parsing XML file contents using lxml.etree
Returns:
Python dictionary holding XML contents.
"""
if not xml:
return {xml.tag: xml.text}
result = {}
for child in xml:
child_result = recursive_parse_xml_to_dict(child)
if child.tag != 'object':
result[child.tag] = child_result[child.tag]
else:
if child.tag not in result:
result[child.tag] = []
result[child.tag].append(child_result[child.tag])
return {xml.tag: result}
from PIL import Image
import glob
import helpers.dataset_util as dataset_util
import hashlib
import io
from helpers.AnnotationLocationLoader import AnnotationLocationLoader
from pathlib import Path
import tensorflow as tf
DEBUG = False
def create_tf_example(filename, annotations, base_folder, labelmap):
search_string = str(base_folder) + '/**/' + filename
#there should only exist one file that matches
full_image_path = glob.glob(search_string, recursive=True)[0]
#pos examples without localisation information should be skipped
if len(annotations) == 0 and Path(full_image_path).parent.name == 'caries':
return None
image = Image.open(full_image_path)
width, height = image.size
with tf.io.gfile.GFile(full_image_path, 'rb') as fid:
encoded_jpg = fid.read()
encoded_jpg_io = io.BytesIO(encoded_jpg)
image = Image.open(encoded_jpg_io)
if image.format != 'JPEG':
raise ValueError('Image format not JPEG')
image_format = image.format
key = hashlib.sha256(encoded_jpg).hexdigest()
#image_format = b'jpg'
xmins = []