diff --git a/config.py b/config.py new file mode 100644 index 0000000..af939f0 --- /dev/null +++ b/config.py @@ -0,0 +1,37 @@ +import os + +ENV = "dev" # [dev/test/prod] + +PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) +print("PROJECT_DIR:\t", PROJECT_DIR) + +DATA_FOLDER = os.path.join(PROJECT_DIR, 'data') +MODELS_FOLDER = os.path.join(PROJECT_DIR, 'models') +LOGS_FOLDER = os.path.join(PROJECT_DIR, 'logs') + +if ENV == "dev": + print("DATA_FOLDER:\t", DATA_FOLDER) + print("MODELS_FOLDER:\t", MODELS_FOLDER) + print("LOGS_FOLDER:\t", LOGS_FOLDER) + +MODEL_WEIGHTS_PATH = os.path.join(MODELS_FOLDER, "bmi_model_weights.h5") +GENDER_MODEL_PATH = os.path.join(MODELS_FOLDER, "simple_CNN.81-0.96.hdf5") +AGE_TRAINED_MODEL_PATH = os.path.join(MODELS_FOLDER, "age_only_resnet50_weights.061-3.300-4.410.hdf5") +CNN_FACE_DETECTOR_MODEL_PATH = os.path.join(MODELS_FOLDER, "mmod_human_face_detector.dat") +BEAUTY_MODEL_WEIGHTS_PATH = os.path.join(MODELS_FOLDER, "beauty_model-ldl-resnet.h5") + +RESNET50_DEFAULT_IMG_WIDTH = 224 +MARGIN = .1 +TRAIN_BATCH_SIZE = 16 +VALIDATION_SIZE = 100 + +ORIGINAL_IMGS_DIR = 'images' +ORIGINAL_IMGS_INFO_FILE = 'data.csv' +#AGE_TRAINED_WEIGHTS_FILE = 'age_only_resnet50_weights.061-3.300-4.410.hdf5' +CROPPED_IMGS_DIR = 'normalized_images' +CROPPED_IMGS_INFO_FILE = 'normalized_data.csv' +TOP_LAYER_LOG_DIR = 'logs/top_layer' +ALL_LAYERS_LOG_DIR = 'logs/all_layers' +#The minimize percentage size of the targeted face to be considered for metadata extraction +MIN_FACE_SIZE_PERCENTAGE = 0.05 +MIN_FACE_SIZE_PIXELS = (75 * 75) diff --git a/imageProcessing.py b/imageProcessing.py new file mode 100644 index 0000000..0b0fa07 --- /dev/null +++ b/imageProcessing.py @@ -0,0 +1,77 @@ +import cv2 +import dlib +import numpy as np +import tensorflow as tf + +import config +from predictors.predict_gender import detect_genders +from predictors.predict_bmi import predict_bmi +from predictors.predict_age import predict_age +#from predictors.predict_beauty import predict_beauty + +detector = dlib.get_frontal_face_detector() + + +def extract_metadata(image_file_path): + img = cv2.imread(image_file_path) + # img = cv2.resize(img, (640, 480)) + input_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + img_h, img_w, _ = np.shape(input_img) + + detected, scores, idx = detector.run(input_img, 1) + faces = np.empty((len(detected), config.RESNET50_DEFAULT_IMG_WIDTH, + config.RESNET50_DEFAULT_IMG_WIDTH, 3)) + filteredFaces = [] + + for i, d in enumerate(detected): + x1, y1, x2, y2, w, h = d.left(), d.top(), d.right() + 1, \ + d.bottom() + 1, d.width(), d.height() + faceSize = (d.right() - d.left()) * (d.bottom() - d.top()) + faceSizePercentage = (faceSize / (img_w * img_h)) * 100 + xw1 = max(int(x1 - config.MARGIN * w), 0) + yw1 = max(int(y1 - config.MARGIN * h), 0) + xw2 = min(int(x2 + config.MARGIN * w), img_w - 1) + yw2 = min(int(y2 + config.MARGIN * h), img_h - 1) + # cv2.rectangle(img, (x1, y1), (x2, y2), (255, 0, 0), 2) + cv2.imwrite(''.join(image_file_path.split('.')[:-1]) + '-cropped-' + str(i) + '.jpg', + img[yw1:yw2, xw1:xw2]) + faces[i, :, :, :] = cv2.resize(img[yw1:yw2 + 1, xw1:xw2 + 1, :], ( + config.RESNET50_DEFAULT_IMG_WIDTH, + config.RESNET50_DEFAULT_IMG_WIDTH)) / 255.00 + + face_boundary = (d.top(), d.right(), d.bottom(), d.left()) + gender = detect_genders(face_boundary, input_img) + + # FILTERING + female = gender["woman"] >= 0.5 + # faceSizePercentage > config.MIN_FACE_SIZE_PERCENTAGE: + if female and faceSize > config.MIN_FACE_SIZE_PIXELS and idx[i] == 0: + filteredFaces.append((d, faceSize, faceSizePercentage, gender["woman"])) + results = [] + for i, d in enumerate(filteredFaces): + bmiResult = predict_bmi([d[0]], img) + if len(bmiResult) > 0: + bmiResult = float(str(bmiResult[0][0])) + else: + bmiResult = None + ageResult = predict_age([d[0]], img) + if len(ageResult) > 0: + ageResult = float(str(ageResult[0])) + else: + ageResult = None + + # beautyResult = predict_beauty([d[0]], img) + results.append({ + "face": { + "left": d[0].left(), + "top": d[0].top(), + "right": d[0].right(), + "bottom": d[0].bottom(), + "size": d[1], + "sizePercentage": d[2] + }, + "gender_woman": d[3], + "bmi": bmiResult, + "age": ageResult + }) + return results diff --git a/main.py b/main.py new file mode 100644 index 0000000..dc4d098 --- /dev/null +++ b/main.py @@ -0,0 +1,67 @@ +import os + +from datetime import datetime +from flask import Flask, jsonify, request +import config +from werkzeug.utils import secure_filename +from imageProcessing import extract_metadata +ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg'} + +app = Flask("P! Vision Metadata Engine") +app.config["UPLOAD_FOLDER"] = config.DATA_FOLDER +router = '/metadata/' + config.ENV + + +def allowed_file(file_name): + return '.' in file_name and \ + file_name.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS + + +@app.route(router + '/hello', methods=['GET']) +def hello(): + return "Hello from P! Metadata Engine" + + +@app.route(router + '/calculate-image-metadata', + methods=['GET', 'POST']) +def calculate_image_metadata(): + try: + if request.method == 'POST': + print() + print("request.form:\t", request.form) + print("request.files:\t", request.files) + request_id = request.form.get("request_id", 100) + print("request_id:\t", request_id) + if "image_file" not in request.files: + msg = "No file part: 'image_file' in POST request." + print(msg) + return msg + file = request.files["image_file"] + if file.filename == '': + msg = "File name is empty." + print(msg) + return msg + if file and allowed_file(file.filename): + print("Uploaded file:\t", file.filename) + image_file_name = str(datetime.now()).replace(':', '') + \ + f'_{request_id}_' + secure_filename(file.filename) + image_file_path = os.path.join( + app.config["UPLOAD_FOLDER"], image_file_name) + file.save(image_file_path) + print(f"Image file is saved at {image_file_path}") + + modelResponse =extract_metadata(image_file_path) + print("Response :\t", modelResponse) + return jsonify({"request_id": request_id, "metadata": modelResponse}) + else: + msg = "POST the image file. Don't GET it." + print(msg) + return msg + except Exception as e: + msg = str(type(e).__name__) + ': ' + str(e) + print(msg) + return msg + + +if __name__ == '__main__': + app.run(debug=False, host="0.0.0.0", port=4455) diff --git a/models/age_only_resnet50_weights.061-3.300-4.410.hdf5 b/models/age_only_resnet50_weights.061-3.300-4.410.hdf5 new file mode 100644 index 0000000..d598495 Binary files /dev/null and b/models/age_only_resnet50_weights.061-3.300-4.410.hdf5 differ diff --git a/models/beauty_model-ldl-resnet.h5 b/models/beauty_model-ldl-resnet.h5 new file mode 100644 index 0000000..1d5509e Binary files /dev/null and b/models/beauty_model-ldl-resnet.h5 differ diff --git a/models/bmi_model_weights.h5 b/models/bmi_model_weights.h5 new file mode 100644 index 0000000..ae33924 Binary files /dev/null and b/models/bmi_model_weights.h5 differ diff --git a/models/mmod_human_face_detector.dat b/models/mmod_human_face_detector.dat new file mode 100644 index 0000000..f112a0a Binary files /dev/null and b/models/mmod_human_face_detector.dat differ diff --git a/models/simple_CNN.81-0.96.hdf5 b/models/simple_CNN.81-0.96.hdf5 new file mode 100644 index 0000000..50241df Binary files /dev/null and b/models/simple_CNN.81-0.96.hdf5 differ diff --git a/predictors/predict_age.py b/predictors/predict_age.py new file mode 100644 index 0000000..0322035 --- /dev/null +++ b/predictors/predict_age.py @@ -0,0 +1,70 @@ +import cv2 +import dlib +import numpy as np +import tensorflow as tf +from tensorflow.python.keras.models import Model +from tensorflow.python.keras.applications import ResNet50 +from tensorflow.python.keras.layers import Dense + +import config + + +def get_age_model(): + age_model = ResNet50( + include_top=False, + weights='imagenet', + input_shape=(config.RESNET50_DEFAULT_IMG_WIDTH, config.RESNET50_DEFAULT_IMG_WIDTH, 3), + pooling='avg' + ) + + prediction = Dense(units=101, + kernel_initializer='he_normal', + use_bias=False, + activation='softmax', + name='pred_age')(age_model.output) + + age_model = Model(inputs=age_model.input, outputs=prediction) + return age_model + + +def get_model(): + base_model = get_age_model() + base_model.load_weights(config.AGE_TRAINED_MODEL_PATH) + print('Loaded weights from age classifier') + + return base_model + +def get_trained_model(): + _model = get_model() + return _model + +model = get_trained_model() +#detector = dlib.get_frontal_face_detector() + +graph = tf.get_default_graph() + +def predict_age(detected_faces, img): + input_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + img_h, img_w, _ = np.shape(input_img) + + faces = np.empty((len(detected_faces), config.RESNET50_DEFAULT_IMG_WIDTH, + config.RESNET50_DEFAULT_IMG_WIDTH, 3)) + + + for i, d in enumerate(detected_faces): + x1, y1, x2, y2, w, h = d.left(), d.top(), d.right() + 1, \ + d.bottom() + 1, d.width(), d.height() + xw1 = max(int(x1 - config.MARGIN * w), 0) + yw1 = max(int(y1 - config.MARGIN * h), 0) + xw2 = min(int(x2 + config.MARGIN * w), img_w - 1) + yw2 = min(int(y2 + config.MARGIN * h), img_h - 1) + + faces[i, :, :, :] = cv2.resize(img[yw1:yw2 + 1, xw1:xw2 + 1, :], ( + config.RESNET50_DEFAULT_IMG_WIDTH, + config.RESNET50_DEFAULT_IMG_WIDTH)) / 255.00 + + with graph.as_default(): + results = model.predict(faces) + ages = np.arange(0, 101).reshape(101, 1) + predicted_ages = results.dot(ages).flatten() + return predicted_ages diff --git a/predictors/predict_beauty.py b/predictors/predict_beauty.py new file mode 100644 index 0000000..d7b21db --- /dev/null +++ b/predictors/predict_beauty.py @@ -0,0 +1,128 @@ +from keras.layers import Conv2D, Input, MaxPool2D,Flatten, Dense, Permute, GlobalAveragePooling2D +from keras.models import Model +from keras.optimizers import Adam +import numpy as np +import pickle +import keras +import cv2 +import sys +import dlib +import os.path +from keras.models import Sequential +from keras.applications.resnet50 import ResNet50 +#from keras.applications.resnet50 import Dense +from keras.layers.core import Dense +from keras.optimizers import Adam +import pickle +import numpy as np +import cv2 +import os +from keras.layers import Dropout +import config + +#APP_ROOT = os.path.dirname(os.path.abspath(__file__)) + +#parent_path = os.path.dirname(APP_ROOT) +#parent_path = os.path.dirname(parent_path) +#model_path = parent_path+"/common/mmod_human_face_detector.dat" +#cnn_face_detector = dlib.cnn_face_detection_model_v1(model_path) +cnn_face_detector = dlib.cnn_face_detection_model_v1(config.CNN_FACE_DETECTOR_MODEL_PATH) +resnet = ResNet50(include_top=False, pooling='avg') +beautyModel = Sequential() +beautyModel.add(resnet) +beautyModel.add(Dense(5, activation='softmax')) +beautyModel.layers[0].trainable = False + +beautyModel.load_weights(config.BEAUTY_MODEL_WEIGHTS_PATH) + +def score_mapping(modelScore): + + if modelScore <= 1.9: + mappingScore = ((4 - 2.5) / (1.9 - 1.0)) * (modelScore-1.0) + 2.5 + elif modelScore <= 2.8: + mappingScore = ((5.5 - 4) / (2.8 - 1.9)) * (modelScore-1.9) + 4 + elif modelScore <= 3.4: + mappingScore = ((6.5 - 5.5) / (3.4 - 2.8)) * (modelScore-2.8) + 5.5 + elif modelScore <= 4: + mappingScore = ((8 - 6.5) / (4 - 3.4)) * (modelScore-3.4) + 6.5 + elif modelScore < 5: + mappingScore = ((9 - 8) / (5 - 4)) * (modelScore-4) + 8 + + return mappingScore + +def predict_beauty(img): + #im0 = cv2.imread(imgPath) + im0 = img + + if im0.shape[0] > 1280: + new_shape = (1280, im0.shape[1] * 1280 / im0.shape[0]) + elif im0.shape[1] > 1280: + new_shape = (im0.shape[0] * 1280 / im0.shape[1], 1280) + elif im0.shape[0] < 640 or im0.shape[1] < 640: + new_shape = (im0.shape[0] * 2, im0.shape[1] * 2) + else: + new_shape = im0.shape[0:2] + + im = cv2.resize(im0, (int(new_shape[1]), int(new_shape[0]))) + dets = cnn_face_detector(im, 0) + + for i, d in enumerate(dets): + face = [d.rect.left(), d.rect.top(), d.rect.right(), d.rect.bottom()] + croped_im = im[face[1]:face[3], face[0]:face[2], :] + resized_im = cv2.resize(croped_im, (224, 224)) + normed_im = np.array([(resized_im - 127.5) / 127.5]) + + pred = beautyModel.predict(normed_im) + ldList = pred[0] + out = 1 * ldList[0] + 2 * ldList[1] + 3 * ldList[2] + 4 * ldList[3] + 5 * ldList[4] + + out = score_mapping(out) + return out + + # print(img + " 打分:" + str('%.2f' % (out))) + # cv2.rectangle(im, (face[0], face[1]), (face[2], face[3]), (0, 255, 0), 3) + # cv2.putText(im, str('%.2f' % (out)), (face[0], face[3]), cv2.FONT_HERSHEY_SIMPLEX, + # 1, (0, 0, 255), 2) + # + # ret = path + "/output-" + img + # cv2.imwrite(ret, im) + # return ret + +def predict_beauty_from_faces(detected_faces, img): + outList = [] + im0 = img + + if im0.shape[0] > 1280: + new_shape = (1280, im0.shape[1] * 1280 / im0.shape[0]) + elif im0.shape[1] > 1280: + new_shape = (im0.shape[0] * 1280 / im0.shape[1], 1280) + elif im0.shape[0] < 640 or im0.shape[1] < 640: + new_shape = (im0.shape[0] * 2, im0.shape[1] * 2) + else: + new_shape = im0.shape[0:2] + + im = cv2.resize(im0, (int(new_shape[1]), int(new_shape[0]))) + + for i, d in enumerate(detected_faces): + face = [d.left(), d.top(), d.right(), d.bottom()] + croped_im = im[face[1]:face[3], face[0]:face[2], :] + resized_im = cv2.resize(croped_im, (224, 224)) + normed_im = np.array([(resized_im - 127.5) / 127.5]) + + pred = beautyModel.predict(normed_im) + ldList = pred[0] + out = 1 * ldList[0] + 2 * ldList[1] + 3 * ldList[2] + 4 * ldList[3] + 5 * ldList[4] + + out = score_mapping(out) + outList.append(out) + return outList + +# beauty_predict(parent_path+"/samples/image",'fengjie.jpg') +# beauty_predict(parent_path+"/samples/image",'nenghua.jpg') +# beauty_predict(parent_path+"/samples/image",'shunli.jpg') +# beauty_predict(parent_path+"/samples/image",'test1.jpg') +# beauty_predict(parent_path+"/samples/image",'test2.jpg') +# beauty_predict(parent_path+"/samples/image",'test3.jpg') +# beauty_predict(parent_path+"/samples/image",'fty1845.jpg') +#beauty_predict(parent_path+"/samples/image",'fty1959.jpg') +# beauty_predict(parent_path+"/samples/image",'jiyou.png') \ No newline at end of file diff --git a/predictors/predict_bmi.py b/predictors/predict_bmi.py new file mode 100644 index 0000000..c3b874e --- /dev/null +++ b/predictors/predict_bmi.py @@ -0,0 +1,78 @@ +import cv2 +import dlib +import numpy as np +import tensorflow as tf +from tensorflow.python.keras.models import Model +from tensorflow.python.keras.applications import ResNet50 +from tensorflow.python.keras.layers import Dense + +import config + + +def get_age_model(): + age_model = ResNet50( + include_top=False, + weights='imagenet', + input_shape=(config.RESNET50_DEFAULT_IMG_WIDTH, config.RESNET50_DEFAULT_IMG_WIDTH, 3), + pooling='avg' + ) + + prediction = Dense(units=101, + kernel_initializer='he_normal', + use_bias=False, + activation='softmax', + name='pred_age')(age_model.output) + + age_model = Model(inputs=age_model.input, outputs=prediction) + return age_model + + +def get_model(ignore_age_weights=False): + base_model = get_age_model() + if not ignore_age_weights: + base_model.load_weights(config.AGE_TRAINED_WEIGHTS_FILE) + print('Loaded weights from age classifier') + last_hidden_layer = base_model.get_layer(index=-2) + + base_model = Model( + inputs=base_model.input, + outputs=last_hidden_layer.output) + prediction = Dense(1, kernel_initializer='normal')(base_model.output) + + model = Model(inputs=base_model.input, outputs=prediction) + return model + +def get_trained_model(): + weights_file = config.MODEL_WEIGHTS_PATH + _model = get_model(ignore_age_weights=True) + _model.load_weights(weights_file) + return _model + +model = get_trained_model() +#detector = dlib.get_frontal_face_detector() + +graph = tf.get_default_graph() + +def predict_bmi(detected_faces, img): + input_img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) + img_h, img_w, _ = np.shape(input_img) + + faces = np.empty((len(detected_faces), config.RESNET50_DEFAULT_IMG_WIDTH, + config.RESNET50_DEFAULT_IMG_WIDTH, 3)) + + + for i, d in enumerate(detected_faces): + x1, y1, x2, y2, w, h = d.left(), d.top(), d.right() + 1, \ + d.bottom() + 1, d.width(), d.height() + xw1 = max(int(x1 - config.MARGIN * w), 0) + yw1 = max(int(y1 - config.MARGIN * h), 0) + xw2 = min(int(x2 + config.MARGIN * w), img_w - 1) + yw2 = min(int(y2 + config.MARGIN * h), img_h - 1) + + faces[i, :, :, :] = cv2.resize(img[yw1:yw2 + 1, xw1:xw2 + 1, :], ( + config.RESNET50_DEFAULT_IMG_WIDTH, + config.RESNET50_DEFAULT_IMG_WIDTH)) / 255.00 + + with graph.as_default(): + predictions = model.predict(faces) + return predictions diff --git a/predictors/predict_gender.py b/predictors/predict_gender.py new file mode 100644 index 0000000..f524678 --- /dev/null +++ b/predictors/predict_gender.py @@ -0,0 +1,72 @@ +import os + +import cv2 +import numpy as np +from keras.models import load_model +from PIL import Image +import config + +gender_classifier = load_model(config.GENDER_MODEL_PATH, compile=False) + +# https://github.com/keras-team/keras/issues/6462 +gender_classifier._make_predict_function() + +gender_target_size = gender_classifier.input_shape[1:3] +gender_offsets = (10, 10) + + +def detect_genders(faceBoundary, image): + result = "" + face = get_boundary_box(faceBoundary, np.shape(image)) + x1, x2, y1, y2 = apply_offsets(face, gender_offsets) + rgb_face = image[y1:y2, x1:x2] + + try: + rgb_face = cv2.resize(rgb_face, gender_target_size) + except Exception as e: + print('Error while resizing the image', e) + return result + + rgb_face = pre_process_input(rgb_face) + rgb_face = np.expand_dims(rgb_face, 0) + gender_prediction = gender_classifier.predict(rgb_face) + result = { + 'woman': float(gender_prediction[0][0]), + 'man': float(gender_prediction[0][1]) + } + + return result + + +def apply_offsets(face_coordinates, offsets): + x, y, width, height = face_coordinates + x_off, y_off = offsets + return max(x - x_off, 0), x + width + x_off, max(y - y_off, 0), y + height + y_off + + +def pre_process_input(x, v2=False): + x = x.astype('float32') + x = x / 255.0 + if v2: + x = x - 0.5 + x = x * 2.0 + return x + + +def load_image(imageFilename): + try: + img = Image.open(imageFilename) + img = img.convert('RGB') + return np.array(img) + except Exception as e: + print("Not a valid image found on {0}: {1}".format(imageFilename, e)) + return None + + +def get_boundary_box(boundary, image_shape): + X = max(boundary[3], 0) + Y = max(boundary[0], 0) + width = min(abs(X - boundary[1]), image_shape[0]) + height = min(abs(Y - boundary[2]), image_shape[1]) + return [X, Y, width, height] + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..bbad432 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,38 @@ +absl-py==0.7.0 +astor==0.7.1 +Augmentor==0.2.3 +backports.functools-lru-cache==1.5 +backports.weakref==1.0.post1 +cycler==0.10.0 +dlib==19.16.0 +enum34==1.1.6 +funcsigs==1.0.2 +future==0.17.1 +gast==0.2.2 +grpcio==1.18.0 +h5py==2.9.0 +Keras==2.2.4 +Keras-Applications==1.0.7 +Keras-Preprocessing==1.0.9 +kiwisolver==1.0.1 +Markdown==3.0.1 +matplotlib==2.2.3 +mock==2.0.0 +numpy==1.16.1 +opencv-python==4.0.0.21 +pandas==0.24.1 +pbr==5.1.2 +Pillow==5.4.1 +protobuf==3.6.1 +pyparsing==2.3.1 +python-dateutil==2.8.0 +pytz==2018.9 +PyYAML==3.13 +scipy==1.2.0 +six==1.12.0 +subprocess32==3.5.3 +tensorboard==1.12.2 +tensorflow==1.12.0 +termcolor==1.1.0 +tqdm==4.30.0 +Werkzeug==0.14.1 diff --git a/test.html b/test.html new file mode 100644 index 0000000..ba4302b --- /dev/null +++ b/test.html @@ -0,0 +1,21 @@ + + +
+ +