From 027e8571e6758fcbd9d75fffbea5c15b85efd77a Mon Sep 17 00:00:00 2001 From: Rames The Generic <53163624+RamesTheGeneric@users.noreply.github.com> Date: Fri, 28 Apr 2023 00:13:56 -0400 Subject: [PATCH] Added Combined Smiles Combined smiles in data generator and a script to read blendshape data from images and write them to a csv. --- datageneration/DatasetGeneratorV4.py | 29 ++--- datageneration/DatasetLoader.py | 57 ++++++++ .../dataset_tools/csv_from_images.py | 123 ++++++++++++++++++ 3 files changed, 190 insertions(+), 19 deletions(-) create mode 100644 datageneration/DatasetLoader.py create mode 100644 datageneration/dataset_tools/csv_from_images.py diff --git a/datageneration/DatasetGeneratorV4.py b/datageneration/DatasetGeneratorV4.py index dc6c849..5216eab 100644 --- a/datageneration/DatasetGeneratorV4.py +++ b/datageneration/DatasetGeneratorV4.py @@ -65,7 +65,8 @@ class Defines(): exclusives = [ # mark these combinations for special treatment after generating values(combined values, etc...) ['mouthClose', 'jawOpen'], ['mouthShrugUpper', 'mouthShrugLower'], - ['tongueOut', 'jawOpen'] + ['tongueOut', 'jawOpen'], + ['mouthSmileLeft', 'mouthSmileRight'] ] shape_bl = dict( @@ -158,6 +159,13 @@ class ShapeSetter(): if defs.exclusives[2][0] in self.selected_shapes_index and self.selected_shapes[self.selected_shapes_index.index(defs.exclusives[2][1])] <= 0.35: self.selected_shapes[self.selected_shapes_index.index(defs.exclusives[2][1])] = clamp(random.uniform(0.35, 1.05), 0, 1) + if defs.exclusives[3][1] in self.selected_shapes_index and defs.exclusives[3][0] in self.selected_shapes_index: # randomly make mouthSmileLeft and mouthSmileRight equal to the greater value + if random.randint(0, 1) == 1: # 50% chance to make values equal + if self.selected_shapes[self.selected_shapes_index.index(defs.exclusives[3][1])] > self.selected_shapes[self.selected_shapes_index.index(defs.exclusives[3][0])]: + self.selected_shapes[self.selected_shapes_index.index(defs.exclusives[3][0])] = self.selected_shapes[self.selected_shapes_index.index(defs.exclusives[3][1])] + if self.selected_shapes[self.selected_shapes_index.index(defs.exclusives[3][0])] > self.selected_shapes[self.selected_shapes_index.index(defs.exclusives[3][1])]: + self.selected_shapes[self.selected_shapes_index.index(defs.exclusives[3][1])] = self.selected_shapes[self.selected_shapes_index.index(defs.exclusives[3][0])] + def generate_example(self, DefsClass, range, classExample): defs = DefsClass self.reset(defs) @@ -202,7 +210,6 @@ class EncodeDecode(): output_hex = int(output_hex, 0) return(output_hex) - FRAME_START = bpy.context.scene.frame_start FRAME_END = bpy.context.scene.frame_end @@ -217,10 +224,8 @@ range_list = [ [-0.2, 0.5] ] - mesh = 'BabbleCA_M' ob = bpy.data.objects[mesh] - for frame in range(FRAME_START, FRAME_END+1): # Iterate over each frame in the range model_shapes = [] print(f"Generated {frame} of {FRAME_END} shapes") @@ -250,6 +255,7 @@ for frame in range(FRAME_START, FRAME_END+1): # Iterate over each frame in the ob = bpy.data.objects['Quest2'] ob.hide_render = True ob.keyframe_insert(data_path="hide_render", frame=frame) + if hmd_type == 0: ob = bpy.data.objects['Pimax'] ob.hide_render = True @@ -282,28 +288,13 @@ for frame in range(FRAME_START, FRAME_END+1): # Iterate over each frame in the ob.hide_render = False ob.keyframe_insert(data_path="hide_render", frame=frame) - - ob = bpy.data.objects[mesh] encdec.encode(shapecount, 1) bpy.data.scenes["Scene"].node_tree.nodes["Group.002"].inputs[1].keyframe_insert(data_path="default_value", frame=frame) for i in range(shapecount): encdec.encode(encdec.float_to_uint24(ob.data.shape_keys.key_blocks[defs.shape_defs[defs.shape_index[i]]].value), i + 2) - - for i in range(shapecount): bpy.data.scenes["Scene"].node_tree.nodes["Group.002"].inputs[i + 2].keyframe_insert(data_path="default_value", frame=frame) - - # Todo: Figure out why keyframed value isn't fucking changing - - - - ''' - for i in range(len(defs.shape_index)): - encdec.encode(encdec.float_to_uint24(ob.data.shape_keys.key_blocks[defs.shape_defs[defs.shape_index[i]]].value), i + 1) - bpy.data.scenes["Scene"].node_tree.nodes["Group.002"].inputs[i + 2].keyframe_insert(data_path="default_value", frame=frame) - ''' - tick += 1 diff --git a/datageneration/DatasetLoader.py b/datageneration/DatasetLoader.py new file mode 100644 index 0000000..f74dc85 --- /dev/null +++ b/datageneration/DatasetLoader.py @@ -0,0 +1,57 @@ +from PIL import Image, ImageFilter +import os +class EncodeDecode(): + def float_to_uint24(self, f): + f = round(f, 8) + f_clamped = max(0, min(1, f)) + uint24 = int(f_clamped * ((1 << 24) - 1)) + return uint24 + + def uint24_to_float(self, uint24): + f = uint24 / ((1 << 24) - 1) + return f + + def encode(self, f, index): # uint24 to blender pixel (1.0, 1.0, 1.0) RGB + hex_str = format(f, '06x') + hex_str = list(hex_str.strip(" ")) + #bpy.data.scenes["Scene"].node_tree.nodes["Group.002"].inputs[index].default_value = ((int('0x' + hex_str[0] + hex_str[1], 0) / 255),(int('0x' + hex_str[2] + hex_str[3], 0) / 255),(int('0x' + hex_str[4] + hex_str[5], 0) / 255),1) + return ((int('0x' + hex_str[0] + hex_str[1], 0) / 255),(int('0x' + hex_str[2] + hex_str[3], 0) / 255),(int('0x' + hex_str[4] + hex_str[5], 0) / 255),1) + + def decode(self, pixel): # Pixel to uint24 (255, 255, 255) RGB + output_hex = ('0x' + hex(pixel[0]).replace('0x', '').rjust(2, "0") + hex(pixel[1]).replace('0x', '').rjust(2, "0") + hex(pixel[2]).replace('0x', '').rjust(2, "0")) + output_hex = int(output_hex, 0) + return(output_hex) + +def get_shapes(ed, img): + px = img.load() + output = ed.decode(px[0, 256]) # Shape count + shapes = [] + for i in range(output): + uint = ed.decode(px[(i + 1), 256]) + shapes.append(ed.uint24_to_float(uint)) + return shapes + +shape_index = ["cheekPuff", "jawOpen", "jawForward", "jawLeft", "jawRight", "noseSneerLeft", "noseSneerRight", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight", +"mouthRollUpper", "mouthRollLower", "mouthShrugUpper", "mouthShrugLower", "mouthClose", "mouthSmileLeft", +"mouthSmileRight", "mouthFrownLeft", "mouthFrownRight", "mouthDimpleLeft", "mouthDimpleRight", "mouthUpperUpLeft", +"mouthUpperUpRight", "mouthLowerDownLeft", "mouthLowerDownRight", "mouthPressLeft", "mouthPressRight", "mouthStretchLeft", +"mouthStretchRight", "tongueOut"] + +path = r'Z:/BabbleDataGeneration/BabbleDataGeneration/render/' +dir_list = os.listdir(path) +ed = EncodeDecode() +shapes = [] +''' +for i in range(len(dir_list)): + img = Image.open(path + dir_list[i]) + shapes.append(get_shapes(ed, img)) +''' +box = (0, 0, 256, 256) +img = Image.open(path + dir_list[1]).crop(box) +img = img.filter(ImageFilter.GaussianBlur(1.5)) +#box = (0, 0, 256, 256) +#img = img.crop(box) +img.show() + + +print(shapes) diff --git a/datageneration/dataset_tools/csv_from_images.py b/datageneration/dataset_tools/csv_from_images.py new file mode 100644 index 0000000..10f7775 --- /dev/null +++ b/datageneration/dataset_tools/csv_from_images.py @@ -0,0 +1,123 @@ +import os +from PIL import Image + +class EncodeDecode(): + def float_to_uint24(self, f): + f = round(f, 8) + f_clamped = max(0, min(1, f)) + uint24 = int(f_clamped * ((1 << 24) - 1)) + return uint24 + + def uint24_to_float(self, uint24): + f = uint24 / ((1 << 24) - 1) + return f + + def encode(self, f, index): # uint24 to blender pixel (1.0, 1.0, 1.0) + hex_str = format(f, '06x') + hex_str = list(hex_str.strip(" ")) + #print(hex_str) + #bpy.data.scenes["Scene"].node_tree.nodes["Group.002"].inputs[index].default_value = ((int('0x' + hex_str[0] + hex_str[1], 0) / 255),(int('0x' + hex_str[2] + hex_str[3], 0) / 255),(int('0x' + hex_str[4] + hex_str[5], 0) / 255),1) + return ((int('0x' + hex_str[0] + hex_str[1], 0) / 255),(int('0x' + hex_str[2] + hex_str[3], 0) / 255),(int('0x' + hex_str[4] + hex_str[5], 0) / 255),1) + + def decode(self, pixel): # Pixel to uint24 (255, 255, 255) + output_hex = ('0x' + hex(pixel[0]).replace('0x', '').rjust(2, "0") + hex(pixel[1]).replace('0x', '').rjust(2, "0") + hex(pixel[2]).replace('0x', '').rjust(2, "0")) + output_hex = int(output_hex, 0) + return(output_hex) + +# Path to subdirectory containing text files +path_to_files = "X:/Downloads/render/render/" + +# Output file name +output_file = "babble_dataset.csv" + +# Header row +header = str(["cheekPuff", "jawOpen", "jawForward", "jawLeft", "jawRight", "noseSneerLeft", "noseSneerRight", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight", +"mouthRollUpper", "mouthRollLower", "mouthShrugUpper", "mouthShrugLower", "mouthClose", "mouthSmileLeft", +"mouthSmileRight", "mouthFrownLeft", "mouthFrownRight", "mouthDimpleLeft", "mouthDimpleRight", "mouthUpperUpLeft", +"mouthUpperUpRight", "mouthLowerDownLeft", "mouthLowerDownRight", "mouthPressLeft", "mouthPressRight", "mouthStretchLeft", +"mouthStretchRight", "tongueOut", "filename"]) +header = header.replace("'", "") +header = header.replace("[", "") +header = header.replace("]", "") +header = header.replace(' ', "") + + +# Open output file for writing and write header row + +shape_index = ["cheekPuff", "jawOpen", "jawForward", "jawLeft", "jawRight", "noseSneerLeft", "noseSneerRight", "mouthFunnel", "mouthPucker", "mouthLeft", "mouthRight", +"mouthRollUpper", "mouthRollLower", "mouthShrugUpper", "mouthShrugLower", "mouthClose", "mouthSmileLeft", +"mouthSmileRight", "mouthFrownLeft", "mouthFrownRight", "mouthDimpleLeft", "mouthDimpleRight", "mouthUpperUpLeft", +"mouthUpperUpRight", "mouthLowerDownLeft", "mouthLowerDownRight", "mouthPressLeft", "mouthPressRight", "mouthStretchLeft", +"mouthStretchRight", "tongueOut"] + +def get_shapes(ed, img): + px = img.load() + output = ed.decode(px[0, 256]) # Shape count + shapes = [] + for i in range(output): + uint = ed.decode(px[(i + 1), 256]) # Skip the bottom left pixel because it's a uint24 of the shape count + shapes.append(ed.uint24_to_float(uint)) + return shapes + +ed = EncodeDecode() +shape_defs = dict( # 31 shapes + cheekPuff = [], + jawOpen = [], + jawForward = [], # added + jawLeft = [], # added + jawRight = [], # added + noseSneerLeft = [], # added + noseSneerRight = [], # added + mouthFunnel = [], # added + mouthPucker = [], # fixed in vrcft master + mouthLeft = [], # added + mouthRight = [], # added + mouthRollUpper = [], # fixed in vrcft master + mouthRollLower = [], # fixed in vrcft master + mouthShrugUpper = [], # added + mouthShrugLower = [], # added + mouthClose = [], # MUST BE EQUAL TO jawOpen + mouthSmileLeft = [], # added + mouthSmileRight = [], # added + mouthFrownLeft = [], # added + mouthFrownRight = [], # added + mouthDimpleLeft = [], # added + mouthDimpleRight = [], # added + mouthUpperUpLeft = [], # added + mouthUpperUpRight = [], # added + mouthLowerDownLeft = [], # added + mouthLowerDownRight = [], # added + mouthPressLeft = [], # added + mouthPressRight = [], # added + mouthStretchLeft = [], # added + mouthStretchRight = [], # added + tongueOut = [], + filename = [] +) + +count = 0 +with open(output_file, "w", buffering=10000) as outfile: + outfile.write(header) + # Loop over all files in subdirectory + for filename in os.listdir(path_to_files): + # Only process files with .csv extension + if filename.endswith(".png"): + # Read data from input file and write to output file + data = [] + img = Image.open(path_to_files + filename) + data = get_shapes(ed, img) + data.append(path_to_files + filename) + data = str(data) + data = data.replace("'", "") + data = data.replace("[", "") + data = data.replace("]", "") + data = data.replace(' ', "") + contents = data + + + if count == 0: + outfile.write(f"\n{contents}\n") + else: + outfile.write(f"{contents}\n") + print(f"Combined {count} files!") + count += 1