general cleanup

dataset_stats.py

+from helpers.AnnotationLocationLoader import AnnotationLocationLoader
+
+'''
+This script is able to load the annotations for each part of the dataset separately and report a few stats about them.
+'''
+
+def get_stats(annotation_loader):
+    bilder = annotation_loader.get_all_annotated_images()
+    bilder_count = len(annotation_loader.get_all_available_images())
+    bilder_mit_annotation = 0
+    summe_der_annotationen = 0
+    for image in bilder:
+        annot_per_image = len(annotation_loader.get_annotations(image, ['Caries']))
+        summe_der_annotationen += annot_per_image
+        if annot_per_image > 0:
+            bilder_mit_annotation += 1
+    print("Anzahl Bilder: {}".format(bilder_count))
+    print("Bilder mit Caries Annotationen: {}".format(bilder_mit_annotation))
+    print("Summe der Annotationen: {}".format(summe_der_annotationen))
+    print()
+
+
+if __name__ == '__main__':
+
+    print('Test')
+    annotLoader_test = AnnotationLocationLoader(annotation_file='input/caries_dataset_annotation.json',
+                                                 images_base_folder='input/test_data',
+                                                 mouth_annotations_folder='input/mouth_annotations')
+    get_stats(annotLoader_test)
+
+    print('Training')
+    annotLoader_train = AnnotationLocationLoader(annotation_file='input/caries_dataset_annotation.json',
+                                                 images_base_folder='input/training_data',
+                                                 mouth_annotations_folder='input/mouth_annotations')
+    get_stats(annotLoader_train)
+
+    print('Evaluation')
+    annotLoader_eval = AnnotationLocationLoader(annotation_file='input/caries_dataset_annotation.json',
+                                                 images_base_folder='input/evaluation_data',
+                                                 mouth_annotations_folder='input/mouth_annotations')
+    get_stats(annotLoader_eval)
+

frcnn_evaluate.py

+import tensorflow as tf
+from pathlib import Path
+import numpy as np
+
+from PIL import Image
+import cv2
+from helpers.AnnotationLocationLoader import AnnotationLocationLoader
+from helpers.PredictionLocationLoader import PredictionLocationLoader
+from optparse import OptionParser
+
+
+def run_inference_for_single_image(model, image):
+    '''
+    Inference function for single image based on function from TF Object Detection API
+    https://github.com/tensorflow/models/blob/master/research/object_detection/colab_tutorials/object_detection_tutorial.ipynb
+    '''
+    image = np.asarray(image)
+    # The input needs to be a tensor, convert it using `tf.convert_to_tensor`.
+    input_tensor = tf.convert_to_tensor(image)
+    # The model expects a batch of images, so add an axis with `tf.newaxis`.
+    input_tensor = input_tensor[tf.newaxis, ...]
+
+    # Run inference
+    model_fn = model.signatures['serving_default']
+    output_dict = model_fn(input_tensor)
+
+    # All outputs are batches tensors.
+    # Convert to numpy arrays, and take index [0] to remove the batch dimension.
+    # We're only interested in the first num_detections.
+    num_detections = int(output_dict.pop('num_detections'))
+    output_dict = {key: value[0, :num_detections].numpy()
+                   for key, value in output_dict.items()}
+    output_dict['num_detections'] = num_detections
+
+    # detection_classes should be ints.
+    output_dict['detection_classes'] = output_dict['detection_classes'].astype(np.int64)
+
+    return output_dict
+
+
+def save_inference(model, image_path, output_path, annotation_loader):
+    """
+    Run the inference for an image, predict localizations, create visualisations and save everything to file
+    :param model: Faster R-CNN model
+    :param image_path: path the image for which to run the inference
+    :param output_path: path to base output folder
+    :param annotation_loader: helper to get the corresponding ground truth
+    :return: None - save results to disk
+    """
+    # get image as numpy array
+    filename = image_path.name
+    image_np = np.array(Image.open(image_path))
+    # run inference
+    output_dict = run_inference_for_single_image(model, image_np)
+
+    selected_indices = tf.image.non_max_suppression(output_dict['detection_boxes'], output_dict['detection_scores'], 20, 0.3)
+
+    img = cv2.imread(str(image_path), cv2.IMREAD_IGNORE_ORIENTATION  | cv2.IMREAD_COLOR)
+    height, width, _ = img.shape
+
+    # draw the annotated labels to the original picture
+    annotations = annotation_loader.get_annotations(filename, ['caries'])
+    for annotation in annotations:
+        cv2.rectangle(img, annotation[1][0], annotation[1][1], (0, 255, 0), 3)
+
+    prepared_boxes = []
+    for i, detection_score in enumerate(output_dict['detection_scores']):
+        if i not in selected_indices:
+            continue
+        if detection_score < 0.5:
+            continue
+        # ymin, xmin, ymax, xmax as relative to the image.
+        bbox = output_dict['detection_boxes'][i]
+        ymin = int(round(bbox[0] * height))
+        xmin = int(round(bbox[1] * width))
+        ymax = int(round(bbox[2] * height))
+        xmax = int(round(bbox[3] * width))
+
+        cv2.rectangle(img, (xmin, ymin), (xmax, ymax), (255, 255, 255), 3)
+        class_prediction = output_dict['detection_classes'][i]
+        class_prediction = index_class_map[class_prediction]
+        class_score = output_dict['detection_scores'][i]
+        label = "{} {:.2f}%".format(class_prediction, (class_score*100))
+        cv2.putText(img, label, (xmin, (ymin - 5)), cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2)
+
+        box = (class_prediction, [(xmin, ymin), (xmax, ymax), class_score])
+        prepared_boxes.append(box)
+
+    with open(str(output_path.parent/"predictions.txt"), "a") as predictions_file:
+        predictions_file.write('{}; {}\n'.format(filename, str(prepared_boxes)))
+
+    cv2.imwrite(str(output_path/filename), img)
+
+
+parser = OptionParser()
+
+parser.add_option("-i", "--path_evaluation", dest="evaluation_path", help="Path to evaluation input.", default="./input/evaluation_data")
+parser.add_option("--model_folder", dest="model_folder", help="Path to the model to evaluate.", default="./out/trained_models/frcnn_model/exported_model")
+parser.add_option("-o", "--path_output", dest="output_path", help="Path to base folder for output data.", default='./out')
+
+(options, args) = parser.parse_args()
+
+evaluation_folder = Path(options.evaluation_path)
+frcnn_checkpoint_folder = Path(options.model_folder)
+frcnn_output_folder = Path(options.output_path) / 'evaluation_frcnn'
+if not frcnn_output_folder.exists():
+    frcnn_output_folder.mkdir()
+
+class_index_map = {'caries': 1, 'no_caries': 2}
+index_class_map = {}
+for element in class_index_map:
+    index_class_map[class_index_map[element]] = element
+
+annotation_loader = AnnotationLocationLoader(images_base_folder=evaluation_folder)
+
+# clean up predicted boxes from last run
+with open(str(frcnn_output_folder/"predictions.txt"), "w") as predictions_file:
+    predictions_file.write("")
+
+model_dir = frcnn_checkpoint_folder / "saved_model"
+model = tf.saved_model.load(str(model_dir))
+
+# run inference for all images in the evaluation folder
+for path in [path for path in evaluation_folder.iterdir() if path.is_dir()]:
+    # define an output folder for each class
+    out_path = Path(frcnn_output_folder) / path.name
+    if not out_path.exists():
+        out_path.mkdir()
+
+    # get the filenames of all test-images in the current folder
+    filenames = [item for item in path.glob('*') if item.name != '.DS_Store']
+
+    for img in filenames:
+        save_inference(model, img, out_path, annotation_loader)
+        print("[INFO] Finished {}/{}".format(img.parent.name, img.name))
+
+# reload currently saved predictions for evaluation the classification
+prediction_loader = PredictionLocationLoader(prediction_file=str(frcnn_output_folder/'predictions.txt'), images_base_folder=evaluation_folder)
+
+tp = 0
+fp = 0
+fn = 0
+tn = 0
+fp_list = []
+fn_list = []
+# evaluate the classification capabilities of the Faster R-CNN model
+for path in [path for path in evaluation_folder.iterdir() if path.is_dir()]:
+    for image in [item for item in path.glob('*') if item.name != '.DS_Store']:
+        predictions = prediction_loader.get_annotations(str(image.name))
+        if len(predictions) > 0: #prediction is positive
+            if path.name == 'caries':
+                tp += 1
+            else:
+                fp += 1
+                fp_list.append(str(image.name))
+        else:   # prediction is negative
+            if path.name == 'caries':
+                fn += 1
+            else:
+                tn += 1
+                fn_list.append(str(image.name))
+
+p = float(tp) / (tp + fp)
+r = float(tp) / (tp + fn)
+f1 = (2 * p * r) / (p + r)
+
+# save and print the classification results
+with open(str(frcnn_output_folder/'classification_results.txt'), 'w') as out_file:
+    out_file.write("Klassifikationsergebnisse für " + str(frcnn_checkpoint_folder)+"\n")
+    out_file.write("\n")
+    out_file.write("tp " + str(tp)+"\n")
+    out_file.write("fp " + str(fp)+"\n")
+    out_file.write("fn " + str(fn)+"\n")
+    out_file.write("tn " + str(tn)+"\n")
+    out_file.write("Precision " + str(p)+"\n")
+    out_file.write("Recall " + str(r)+"\n")
+    out_file.write("F1-Measure " + str(f1)+"\n")
+    out_file.write("\n")
+    out_file.write("False Positive Bilder:\n")
+    out_file.write(str(fp_list))
+    out_file.write("\n")
+    out_file.write("False Negative Bilder:\n")
+    out_file.write(str(fn_list))
+
+print('Results for the Faster R-CNN as Classifier:')
+print("Precision " + str(p))
+print("Recall " + str(r))
+print("F1-Measure " + str(f1))
+print(tp, fn)
+print(fp, tn)
+
+print("[INFO] Evaluation finished.")
\ No newline at end of file

helpers/AnnotationLocationLoader.py

@@ -2,37 +2,39 @@ import json
 from pathlib import Path, PosixPath
 
 
-class AnnotationLocationLoader:
+class AnnotationLocationLoader:    
 
-    _annotation_file = None
-    _img_path = None
-    _mouth_annotations_folder = None
-    _annotated_images = set()
-    _available_annotations = set()
-    _available_images = None
-
-    _data = {}
-
-    def __init__(self, annotation_file='input/caries_dataset_annotation.json', images_base_folder=Path('input/test_data/'), mouth_annotations_folder=Path('input/mouth_annotations/')):
+    def __init__(self, annotation_file='input/caries_dataset_annotation.json', images_base_folder=Path('input/test_data/'), mouth_annotations_folder=None):
         self._annotation_file = annotation_file
+        self._img_path = None
+        self._annotated_images = set()
+        self._available_annotations = set()
+        self._available_images = None
+        self._data = {} 
 
         if not type(images_base_folder) == PosixPath:
             images_base_folder = Path(images_base_folder)
         self._img_path = images_base_folder
 
-        if not type(mouth_annotations_folder) == PosixPath:
-            mouth_annotations_folder = Path(mouth_annotations_folder)
-        self._mouth_annotations_folder = mouth_annotations_folder
-
         # get the names of the images witch are available as files
         self._available_images = self._get_names_from_available_images()
         self._load_annotations()
-        self._load_mouth_annotation_additon()
+
+        if not mouth_annotations_folder == None:
+            if not type(mouth_annotations_folder) == PosixPath:
+                mouth_annotations_folder = Path(mouth_annotations_folder)
+            self._mouth_annotations_folder = mouth_annotations_folder
+
+            self._load_mouth_annotation_additon()
 
         self._annotated_images = list(self._annotated_images)
         self._available_annotations = list(self._available_annotations)
 
     def _get_names_from_available_images(self):
+        """
+        Finds the names for all pictures available in the image base folder.
+        :return:
+        """
         names_from_available_images = []
         for path in [path for path in self._img_path.iterdir() if path.is_dir()]:
             names_from_available_images.extend([filename.name for filename in path.iterdir() if filename.is_file() and not filename.name.startswith('.')])
@@ -40,6 +42,9 @@ class AnnotationLocationLoader:
 
 
     def _load_annotations(self):
+        """
+        Loads all annotations for the available images from the json file to an internal structure.
+        """
         with open(self._annotation_file) as file:
             json_data = json.load(file)
 
@@ -71,6 +76,10 @@ class AnnotationLocationLoader:
 
 
     def _load_mouth_annotation_additon(self):
+        """
+        Loads the additional annotations for the mouth for all available images if there is a corresponding folder given
+        This represents the local constrains for the advanced training.
+        """
         annotation_files = [file for file in self._mouth_annotations_folder.iterdir() if file.is_file() and not file.name.startswith('.')]
         counter_number_of_annotated_but_missing_files = 0
         for annotation_file in annotation_files:
@@ -110,6 +119,12 @@ class AnnotationLocationLoader:
         """
         return self._annotated_images
 
+    def get_all_available_images(self):
+        """
+        :return: list of the names of all images witch have at least one annotation
+        """
+        return self._available_images
+
     def is_annotated(self, image_name):
         """
         Should check weather for the given filename an annotation exists

helpers/CAMModel.py

@@ -2,7 +2,6 @@ from tensorflow.keras import Model
 from tensorflow import GradientTape, cast, reduce_mean, reduce_sum, multiply, newaxis, reshape, transpose, squeeze
 from tensorflow.image import resize
 from tensorflow.math import multiply, reduce_min, reduce_max, divide, add, l2_normalize
-from tensorflow.linalg import matmul
 import tensorflow as tf
 from tensorflow.keras.losses import categorical_crossentropy
 from tensorflow.keras.metrics import CategoricalAccuracy, Mean

helpers/ConvertForObjectDetectionMetrics.py

 from helpers.AnnotationLocationLoader import AnnotationLocationLoader
 from helpers.PredictionLocationLoader import PredictionLocationLoader
 from pathlib import Path
+from optparse import OptionParser
+import shutil
 
 def convertGroundtruths(out_path):
     # create folder for gt
@@ -8,22 +10,24 @@ def convertGroundtruths(out_path):
     if not out_path.exists():
         out_path.mkdir(parents=True)
 
-    annotLoader = AnnotationLocationLoader(annotation_file='../input/caries_dataset_annotation.json', images_base_folder='../input/test_data', mouth_annotations_folder='../input/mouth_annotations')
-    for image in annotLoader.get_all_annotated_images():
+    annotLoader = AnnotationLocationLoader(annotation_file='../input/caries_dataset_annotation.json', images_base_folder='../input/evaluation_data')
+    for image in annotLoader.get_all_available_images():
         #create file
         file = image[:-3] + 'txt'
         with open(out_path  / file, "w") as groundtruth_file:
             for annotation in annotLoader.get_annotations(image):
                 # format: <class_name> <left> <top> <right> <bottom>
+                if not str(annotation[0]) == 'caries':
+                    continue
                 groundtruth_file.write(str(annotation[0]) + " " + str(annotation[1][0][0]) + " " + str(annotation[1][0][1]) + " " + str(annotation[1][1][0]) + " " + str(annotation[1][1][1]) + "\n")
 
-def convertDetections(out_path):
+def convertDetections(out_path, predictions_file):
     # create folder for detections
     out_path = out_path / 'detections'
     if not out_path.exists():
         out_path.mkdir(parents=True)
 
-    predLoader = PredictionLocationLoader(prediction_file='../out/evaluation/predictions.txt', images_base_folder='../input/test_data')
+    predLoader = PredictionLocationLoader(prediction_file=predictions_file, images_base_folder='../input/evaluation_data')
     for image in predLoader.get_all_annotated_images():
         file = image[:-3] + 'txt'
         with open(out_path / file, "w") as detection_file:
@@ -33,14 +37,30 @@ def convertDetections(out_path):
                     str(annotation[0]) + " " + str(annotation[1][2]) + " " + str(annotation[1][0][0]) + " " + str(annotation[1][0][1]) + " " + str(
                         annotation[1][1][0]) + " " + str(annotation[1][1][1]) + "\n")
 
-if __name__ == '__main__':
-    '''
-        Please run the Pascal VOC Metrics implemented by Rafael Padilla on the output of this Script.
-        https://github.com/rafaelpadilla/Object-Detection-Metrics
-    '''
-    output_path = Path('../out/convertedForObjectDetectionMetrics')
-    convertGroundtruths(output_path)
-    convertDetections(output_path)
-    print('[INFO] Finished conversion for ObjectDetectionMetrics.')
+
+'''
+    Please run the Pascal VOC Metrics implemented by Rafael Padilla on the output of this Script.
+    https://github.com/rafaelpadilla/Object-Detection-Metrics
+'''
+
+parser = OptionParser()
+
+parser.add_option("-i", "--predictions_file", dest="predictions_file", help="Path to predictions file as input.", default="../out/evaluation_ws_base/predictions.txt")
+parser.add_option("-o", "--path_output", dest="output_path", help="Path to output folder.", default='../out/convertedForObjectDetectionMetrics_ws_base')
+
+(options, args) = parser.parse_args()
+
+if not Path(options.predictions_file).is_file():
+    parser.error('Error: Could not find the prediction file. Check the path passed to -i or --prediction_file')
+
+output_path = Path(options.output_path)
+shutil.rmtree(str(output_path))
+predictions_file = options.predictions_file
+
+convertGroundtruths(output_path)
+convertDetections(output_path, predictions_file)
+
+print('[INFO] Finished conversion for ObjectDetectionMetrics.')
+print('[INFO] Saved to folder {}'.format(str(output_path)))
 
 

helpers/NoCAMModel.py

 from tensorflow.keras import Model
 from tensorflow import GradientTape, cast, reduce_mean, reduce_sum, multiply, newaxis, reshape, transpose, squeeze
-from tensorflow.image import resize
-from tensorflow.math import multiply, reduce_min, reduce_max, divide, add, l2_normalize
-from tensorflow.linalg import matmul
-import tensorflow as tf
 from tensorflow.keras.losses import categorical_crossentropy
 from tensorflow.keras.metrics import CategoricalAccuracy, Mean
 
@@ -16,6 +12,7 @@ metric_tracker = CategoricalAccuracy()
 loss_tracker = Mean(name='loss')
 
 class NoCAMModel(Model):
+    class_index_dict = {'caries': 0, 'no_caries': 1}  # hardcoded classes
 
     def train_step(self, data):
         # Unpack the data. Its structure depends on your model and
@@ -23,7 +20,6 @@ class NoCAMModel(Model):
         x, y = data
 
         img = x['img']
-        mouth_filter = x['mouth']
 
         with GradientTape() as tape:
             y_pred, conv_out = self(img, training=True)  # Forward pass

helpers/PredictionLocationLoader.py

-import json
-from pathlib import Path
 import ast
+from pathlib import Path, PosixPath
 
 
 class PredictionLocationLoader:
 
-    _prediction_file = None
-    _img_path = None
-    _annotated_images = set()
-    _available_annotations = set()
-    _available_images = None
-
-    _data = {}
-
     def __init__(self, prediction_file='out/predictions.txt', images_base_folder='input/test_data/'):
         self._prediction_file = prediction_file
+        self._img_path = None
+        self._annotated_images = set()
+        self._available_annotations = set()
+        self._available_images = None
+        self._data = {}
+
+        if not type(images_base_folder) == PosixPath:
+            images_base_folder = Path(images_base_folder)
         self._img_path = images_base_folder
 
         # get the names of the images witch are available as files
@@ -22,6 +21,9 @@ class PredictionLocationLoader:
         self._load_predictions()
 
     def _get_names_from_available_images(self):
+        """
+        Finds the names for all pictures available in the image base folder.
+        """
         names_from_available_images = []
         for path in [path for path in Path(self._img_path).iterdir() if path.is_dir()]:
             names_from_available_images.extend([filename.name for filename in path.iterdir() if filename.is_file() and not filename.name.startswith('.')])
@@ -29,6 +31,9 @@ class PredictionLocationLoader:
 
 
     def _load_predictions(self):
+        """
+        Loads the predictions from a network based on the prediction file.
+        """
         with open(self._prediction_file) as file:
             for line in file.readlines():
                 filename,  predictions = line.split(';')

helpers/dataset_util.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Utility functions for creating TFRecord data sets."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import tensorflow as tf
+
+
+def int64_feature(value):
+  return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))