AutoBone: Adjust bone offsets for directly and implement new contribution based error calculation (#204)

This commit is contained in:
Butterscotch! authored and GitHub committed 2022-06-28 22:00:57 +03:00
1 parent d1a02c52a9
commit b1318567e3
13 files changed
+1131 -542

No files matched your search

File diff suppressed because it is too large. Load diff
@@ -8,9 +8,14 @@ import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.lang3.tuple.Pair;
import dev.slimevr.VRServer;
import dev.slimevr.autobone.AutoBone.AutoBoneResults;
import dev.slimevr.autobone.errors.AutoBoneException;
import dev.slimevr.poserecorder.PoseFrameTracker;
import dev.slimevr.poserecorder.PoseFrames;
import dev.slimevr.poserecorder.PoseRecorder;
import dev.slimevr.vr.processor.skeleton.SkeletonConfig;
import dev.slimevr.vr.processor.skeleton.SkeletonConfigValue;
import dev.slimevr.vr.trackers.TrackerPosition;
import io.eiren.util.StringUtils;
import io.eiren.util.collections.FastList;
import io.eiren.util.logging.LogManager;
@@ -82,7 +87,7 @@ public class AutoBoneHandler {
return autoBone.getLengthsString();
}
private float processFrames(PoseFrames frames) {
private AutoBoneResults processFrames(PoseFrames frames) throws AutoBoneException {
return autoBone
.processFrames(frames, autoBone.calcInitError, autoBone.targetHeight, (epoch) -> {
listeners.forEach(listener -> {
@@ -298,39 +303,59 @@ public class AutoBoneHandler {
announceProcessStatus(AutoBoneProcessType.PROCESS, "Processing recording(s)...");
LogManager.info("[AutoBone] Processing frames...");
FastList<Float> heightPercentError = new FastList<Float>(frameRecordings.size());
SkeletonConfig skeletonConfigBuffer = new SkeletonConfig(false);
for (Pair<String, PoseFrames> recording : frameRecordings) {
LogManager
.info("[AutoBone] Processing frames from \"" + recording.getKey() + "\"...");
heightPercentError.add(processFrames(recording.getValue()));
List<PoseFrameTracker> trackers = recording.getValue().getTrackers();
StringBuilder trackerInfo = new StringBuilder();
for (PoseFrameTracker tracker : trackers) {
if (tracker == null)
continue;
TrackerPosition position = tracker
.getBodyPosition();
if (position == null)
continue;
if (trackerInfo.length() > 0) {
trackerInfo.append(", ");
}
trackerInfo.append(position.designation);
}
LogManager
.info(
"[AutoBone] ("
+ trackers.size()
+ " trackers) ["
+ trackerInfo.toString()
+ "]"
);
AutoBoneResults autoBoneResults = processFrames(recording.getValue());
heightPercentError.add(autoBoneResults.getHeightDifference());
LogManager.info("[AutoBone] Done processing!");
// #region Stats/Values
Float neckLength = autoBone.getConfig(SkeletonConfigValue.NECK);
Float chestDistance = autoBone.getConfig(SkeletonConfigValue.CHEST);
Float torsoLength = autoBone.getConfig(SkeletonConfigValue.TORSO);
Float hipWidth = autoBone.getConfig(SkeletonConfigValue.HIPS_WIDTH);
Float legsLength = autoBone.getConfig(SkeletonConfigValue.LEGS_LENGTH);
Float kneeHeight = autoBone.getConfig(SkeletonConfigValue.KNEE_HEIGHT);
skeletonConfigBuffer.setConfigs(autoBoneResults.configValues, null);
float neckTorso = neckLength != null && torsoLength != null
? neckLength / torsoLength
: 0f;
float chestTorso = chestDistance != null && torsoLength != null
? chestDistance / torsoLength
: 0f;
float torsoWaist = hipWidth != null && torsoLength != null
? hipWidth / torsoLength
: 0f;
float legTorso = legsLength != null && torsoLength != null
? legsLength / torsoLength
: 0f;
float legBody = legsLength != null && torsoLength != null && neckLength != null
? legsLength / (torsoLength + neckLength)
: 0f;
float kneeLeg = kneeHeight != null && legsLength != null
? kneeHeight / legsLength
: 0f;
float neckLength = skeletonConfigBuffer.getConfig(SkeletonConfigValue.NECK);
float chestDistance = skeletonConfigBuffer.getConfig(SkeletonConfigValue.CHEST);
float torsoLength = skeletonConfigBuffer.getConfig(SkeletonConfigValue.TORSO);
float hipWidth = skeletonConfigBuffer.getConfig(SkeletonConfigValue.HIPS_WIDTH);
float legsLength = skeletonConfigBuffer.getConfig(SkeletonConfigValue.LEGS_LENGTH);
float kneeHeight = skeletonConfigBuffer.getConfig(SkeletonConfigValue.KNEE_HEIGHT);
float neckTorso = neckLength / torsoLength;
float chestTorso = chestDistance / torsoLength;
float torsoWaist = hipWidth / torsoLength;
float legTorso = legsLength / torsoLength;
float legBody = legsLength / (torsoLength + neckLength);
float kneeLeg = kneeHeight / legsLength;
LogManager
.info(
@@ -377,7 +402,7 @@ public class AutoBoneHandler {
// #endregion
listeners.forEach(listener -> {
listener.onAutoBoneEnd(autoBone.configs);
listener.onAutoBoneEnd(autoBone.legacyConfigs);
});
announceProcessStatus(AutoBoneProcessType.PROCESS, "Done processing!", true, true);
@@ -395,7 +420,7 @@ public class AutoBoneHandler {
}
public void applyValues() {
autoBone.applyConfig();
autoBone.applyAndSaveConfig();
announceProcessStatus(AutoBoneProcessType.APPLY, "Adjusted values applied!", true, true);
// TODO Update GUI values after applying? Is that needed here?
}
@@ -0,0 +1,108 @@
package dev.slimevr.autobone;
import java.util.Map;
import dev.slimevr.poserecorder.PoseFrameSkeleton;
import dev.slimevr.poserecorder.PoseFrames;
import dev.slimevr.vr.processor.skeleton.BoneType;
public class AutoBoneTrainingStep {
private int cursor1 = 0;
private int cursor2 = 0;
private float currentHeight;
private final float targetHeight;
private final PoseFrameSkeleton skeleton1;
private final PoseFrameSkeleton skeleton2;
private final PoseFrames trainingFrames;
private final Map<BoneType, Float> intermediateOffsets;
public AutoBoneTrainingStep(
int cursor1,
int cursor2,
float targetHeight,
PoseFrameSkeleton skeleton1,
PoseFrameSkeleton skeleton2,
PoseFrames trainingFrames,
Map<BoneType, Float> intermediateOffsets
) {
this.cursor1 = cursor1;
this.cursor2 = cursor2;
this.targetHeight = targetHeight;
this.skeleton1 = skeleton1;
this.skeleton2 = skeleton2;
this.trainingFrames = trainingFrames;
this.intermediateOffsets = intermediateOffsets;
}
public AutoBoneTrainingStep(
float targetHeight,
PoseFrameSkeleton skeleton1,
PoseFrameSkeleton skeleton2,
PoseFrames trainingFrames,
Map<BoneType, Float> intermediateOffsets
) {
this.targetHeight = targetHeight;
this.skeleton1 = skeleton1;
this.skeleton2 = skeleton2;
this.trainingFrames = trainingFrames;
this.intermediateOffsets = intermediateOffsets;
}
public int getCursor1() {
return cursor1;
}
public void setCursor1(int cursor1) {
this.cursor1 = cursor1;
}
public int getCursor2() {
return cursor2;
}
public void setCursor2(int cursor2) {
this.cursor2 = cursor2;
}
public void setCursors(int cursor1, int cursor2) {
this.cursor1 = cursor1;
this.cursor2 = cursor2;
}
public float getCurrentHeight() {
return currentHeight;
}
public void setCurrentHeight(float currentHeight) {
this.currentHeight = currentHeight;
}
public float getTargetHeight() {
return targetHeight;
}
public PoseFrameSkeleton getSkeleton1() {
return skeleton1;
}
public PoseFrameSkeleton getSkeleton2() {
return skeleton2;
}
public PoseFrames getTrainingFrames() {
return trainingFrames;
}
public Map<BoneType, Float> getIntermediateOffsets() {
return intermediateOffsets;
}
public float getHeightOffset() {
return getTargetHeight() - getCurrentHeight();
}
}
@@ -0,0 +1,28 @@
package dev.slimevr.autobone.errors;
public class AutoBoneException extends Exception {
public AutoBoneException() {
}
public AutoBoneException(String message) {
super(message);
}
public AutoBoneException(Throwable cause) {
super(cause);
}
public AutoBoneException(String message, Throwable cause) {
super(message, cause);
}
public AutoBoneException(
String message,
Throwable cause,
boolean enableSuppression,
boolean writableStackTrace
) {
super(message, cause, enableSuppression, writableStackTrace);
}
}
@@ -0,0 +1,53 @@
package dev.slimevr.autobone.errors;
import com.jme3.math.FastMath;
import dev.slimevr.autobone.AutoBoneTrainingStep;
import dev.slimevr.vr.processor.skeleton.SkeletonConfig;
import dev.slimevr.vr.processor.skeleton.SkeletonConfigValue;
// The distance from average human proportions
public class BodyProportionError implements IAutoBoneError {
// TODO hip tracker stuff... Hip tracker should be around 3 to 5
// centimeters. Human average is probably 1.1235 (SD 0.07)
public float legBodyRatio = 1.1235f;
// SD of 0.07, capture 68% within range
public float legBodyRatioRange = 0.07f;
// kneeLegRatio seems to be around 0.54 to 0.6 after asking a few people in
// the SlimeVR discord.
public float kneeLegRatio = 0.55f;
// kneeLegRatio seems to be around 0.55 to 0.64 after asking a few people in
// the SlimeVR discord. TODO : Chest should be a bit shorter (0.54?) if user
// has an additional hip tracker.
public float chestTorsoRatio = 0.57f;
@Override
public float getStepError(AutoBoneTrainingStep trainingStep) throws AutoBoneException {
return getBodyProportionError(trainingStep.getSkeleton1().skeletonConfig);
}
public float getBodyProportionError(SkeletonConfig config) {
float neckLength = config.getConfig(SkeletonConfigValue.NECK);
float chestLength = config.getConfig(SkeletonConfigValue.CHEST);
float torsoLength = config.getConfig(SkeletonConfigValue.TORSO);
float legsLength = config.getConfig(SkeletonConfigValue.LEGS_LENGTH);
float kneeHeight = config.getConfig(SkeletonConfigValue.KNEE_HEIGHT);
float chestTorso = FastMath.abs((chestLength / torsoLength) - chestTorsoRatio);
float legBody = FastMath.abs((legsLength / (torsoLength + neckLength)) - legBodyRatio);
float kneeLeg = FastMath.abs((kneeHeight / legsLength) - kneeLegRatio);
if (legBody <= legBodyRatioRange) {
legBody = 0f;
} else {
legBody -= legBodyRatioRange;
}
return (chestTorso + legBody + kneeLeg) / 3f;
}
}
@@ -0,0 +1,54 @@
package dev.slimevr.autobone.errors;
import com.jme3.math.FastMath;
import dev.slimevr.autobone.AutoBoneTrainingStep;
import dev.slimevr.vr.processor.skeleton.HumanSkeleton;
import dev.slimevr.vr.trackers.ComputedTracker;
import dev.slimevr.vr.trackers.TrackerRole;
// The offset between the height both feet at one instant and over time
public class FootHeightOffsetError implements IAutoBoneError {
@Override
public float getStepError(AutoBoneTrainingStep trainingStep) throws AutoBoneException {
return getSlideError(trainingStep.getSkeleton1(), trainingStep.getSkeleton2());
}
public static float getSlideError(HumanSkeleton skeleton1, HumanSkeleton skeleton2) {
ComputedTracker leftTracker1 = skeleton1.getComputedTracker(TrackerRole.LEFT_FOOT);
ComputedTracker rightTracker1 = skeleton1.getComputedTracker(TrackerRole.RIGHT_FOOT);
ComputedTracker leftTracker2 = skeleton2.getComputedTracker(TrackerRole.LEFT_FOOT);
ComputedTracker rightTracker2 = skeleton2.getComputedTracker(TrackerRole.RIGHT_FOOT);
return getFootHeightError(leftTracker1, rightTracker1, leftTracker2, rightTracker2);
}
public static float getFootHeightError(
ComputedTracker leftTracker1,
ComputedTracker rightTracker1,
ComputedTracker leftTracker2,
ComputedTracker rightTracker2
) {
float leftFoot1 = leftTracker1.position.y;
float rightFoot1 = rightTracker1.position.y;
float leftFoot2 = leftTracker2.position.y;
float rightFoot2 = rightTracker2.position.y;
// Compute all combinations of heights
float dist1 = FastMath.abs(leftFoot1 - rightFoot1);
float dist2 = FastMath.abs(leftFoot1 - leftFoot2);
float dist3 = FastMath.abs(leftFoot1 - rightFoot2);
float dist4 = FastMath.abs(rightFoot1 - leftFoot2);
float dist5 = FastMath.abs(rightFoot1 - rightFoot2);
float dist6 = FastMath.abs(leftFoot2 - rightFoot2);
// Divide by 12 (6 values * 2 to halve) to halve and average, it's
// halved because you want to approach a midpoint, not the other point
return (dist1 + dist2 + dist3 + dist4 + dist5 + dist6) / 12f;
}
}
@@ -0,0 +1,21 @@
package dev.slimevr.autobone.errors;
import com.jme3.math.FastMath;
import dev.slimevr.autobone.AutoBoneTrainingStep;
// The difference from the current height to the target height
public class HeightError implements IAutoBoneError {
@Override
public float getStepError(AutoBoneTrainingStep trainingStep) throws AutoBoneException {
return getHeightError(
trainingStep.getCurrentHeight(),
trainingStep.getTargetHeight()
);
}
public float getHeightError(float currentHeight, float targetHeight) {
return FastMath.abs(targetHeight - currentHeight);
}
}
@@ -0,0 +1,8 @@
package dev.slimevr.autobone.errors;
import dev.slimevr.autobone.AutoBoneTrainingStep;
public interface IAutoBoneError {
public float getStepError(AutoBoneTrainingStep trainingStep) throws AutoBoneException;
}
@@ -0,0 +1,61 @@
package dev.slimevr.autobone.errors;
import com.jme3.math.FastMath;
import com.jme3.math.Vector3f;
import dev.slimevr.autobone.AutoBoneTrainingStep;
import dev.slimevr.vr.processor.skeleton.HumanSkeleton;
import dev.slimevr.vr.trackers.ComputedTracker;
import dev.slimevr.vr.trackers.TrackerRole;
// The change in distance between both of the ankles over time
public class OffsetSlideError implements IAutoBoneError {
@Override
public float getStepError(AutoBoneTrainingStep trainingStep) throws AutoBoneException {
return getSlideError(trainingStep.getSkeleton1(), trainingStep.getSkeleton2());
}
public static float getSlideError(HumanSkeleton skeleton1, HumanSkeleton skeleton2) {
ComputedTracker leftTracker1 = skeleton1.getComputedTracker(TrackerRole.LEFT_FOOT);
ComputedTracker rightTracker1 = skeleton1.getComputedTracker(TrackerRole.RIGHT_FOOT);
ComputedTracker leftTracker2 = skeleton2.getComputedTracker(TrackerRole.LEFT_FOOT);
ComputedTracker rightTracker2 = skeleton2.getComputedTracker(TrackerRole.RIGHT_FOOT);
return getSlideError(leftTracker1, rightTracker1, leftTracker2, rightTracker2);
}
public static float getSlideError(
ComputedTracker leftTracker1,
ComputedTracker rightTracker1,
ComputedTracker leftTracker2,
ComputedTracker rightTracker2
) {
Vector3f leftFoot1 = leftTracker1.position;
Vector3f rightFoot1 = rightTracker1.position;
Vector3f leftFoot2 = leftTracker2.position;
Vector3f rightFoot2 = rightTracker2.position;
float slideDist1 = leftFoot1.distance(rightFoot1);
float slideDist2 = leftFoot2.distance(rightFoot2);
float slideDist3 = leftFoot1.distance(rightFoot2);
float slideDist4 = leftFoot2.distance(rightFoot1);
// Compute all combinations of distances
float dist1 = FastMath.abs(slideDist1 - slideDist2);
float dist2 = FastMath.abs(slideDist1 - slideDist3);
float dist3 = FastMath.abs(slideDist1 - slideDist4);
float dist4 = FastMath.abs(slideDist2 - slideDist3);
float dist5 = FastMath.abs(slideDist2 - slideDist4);
float dist6 = FastMath.abs(slideDist3 - slideDist4);
// Divide by 12 (6 values * 2 to halve) to halve and average, it's
// halved because you want to approach a midpoint, not the other point
return (dist1 + dist2 + dist3 + dist4 + dist5 + dist6) / 12f;
}
}
@@ -0,0 +1,61 @@
package dev.slimevr.autobone.errors;
import java.util.List;
import com.jme3.math.FastMath;
import dev.slimevr.autobone.AutoBoneTrainingStep;
import dev.slimevr.poserecorder.PoseFrameTracker;
import dev.slimevr.poserecorder.TrackerFrame;
import dev.slimevr.poserecorder.TrackerFrameData;
import dev.slimevr.vr.processor.skeleton.HumanSkeleton;
import dev.slimevr.vr.trackers.ComputedTracker;
// The distance of any points to the corresponding absolute position
public class PositionError implements IAutoBoneError {
@Override
public float getStepError(AutoBoneTrainingStep trainingStep) throws AutoBoneException {
List<PoseFrameTracker> trackers = trainingStep.getTrainingFrames().getTrackers();
return (getPositionError(
trackers,
trainingStep.getCursor1(),
trainingStep.getSkeleton1()
)
+ getPositionError(
trackers,
trainingStep.getCursor2(),
trainingStep.getSkeleton2()
))
/ 2f;
}
public static float getPositionError(
List<PoseFrameTracker> trackers,
int cursor,
HumanSkeleton skeleton
) {
float offset = 0f;
int offsetCount = 0;
for (PoseFrameTracker tracker : trackers) {
TrackerFrame trackerFrame = tracker.safeGetFrame(cursor);
if (
trackerFrame == null
|| !trackerFrame.hasData(TrackerFrameData.POSITION)
|| trackerFrame.designation.trackerRole.isEmpty()
) {
continue;
}
ComputedTracker computedTracker = skeleton
.getComputedTracker(trackerFrame.designation.trackerRole.get());
if (computedTracker != null) {
offset += FastMath.abs(computedTracker.position.distance(trackerFrame.position));
offsetCount++;
}
}
return offsetCount > 0 ? offset / offsetCount : 0f;
}
}
@@ -0,0 +1,79 @@
package dev.slimevr.autobone.errors;
import java.util.List;
import com.jme3.math.FastMath;
import dev.slimevr.autobone.AutoBoneTrainingStep;
import dev.slimevr.poserecorder.PoseFrameTracker;
import dev.slimevr.poserecorder.TrackerFrame;
import dev.slimevr.poserecorder.TrackerFrameData;
import dev.slimevr.vr.processor.skeleton.HumanSkeleton;
import dev.slimevr.vr.trackers.ComputedTracker;
// The difference between offset of absolute position and the corresponding point over time
public class PositionOffsetError implements IAutoBoneError {
@Override
public float getStepError(AutoBoneTrainingStep trainingStep) throws AutoBoneException {
List<PoseFrameTracker> trackers = trainingStep.getTrainingFrames().getTrackers();
return getPositionOffsetError(
trackers,
trainingStep.getCursor1(),
trainingStep.getCursor2(),
trainingStep.getSkeleton1(),
trainingStep.getSkeleton2()
);
}
public float getPositionOffsetError(
List<PoseFrameTracker> trackers,
int cursor1,
int cursor2,
HumanSkeleton skeleton1,
HumanSkeleton skeleton2
) {
float offset = 0f;
int offsetCount = 0;
for (PoseFrameTracker tracker : trackers) {
TrackerFrame trackerFrame1 = tracker.safeGetFrame(cursor1);
if (
trackerFrame1 == null
|| !trackerFrame1.hasData(TrackerFrameData.POSITION)
|| trackerFrame1.designation.trackerRole.isEmpty()
) {
continue;
}
TrackerFrame trackerFrame2 = tracker.safeGetFrame(cursor2);
if (
trackerFrame2 == null
|| !trackerFrame2.hasData(TrackerFrameData.POSITION)
|| trackerFrame2.designation.trackerRole.isEmpty()
) {
continue;
}
ComputedTracker computedTracker1 = skeleton1
.getComputedTracker(trackerFrame1.designation.trackerRole.get());
if (computedTracker1 == null) {
continue;
}
ComputedTracker computedTracker2 = skeleton2
.getComputedTracker(trackerFrame2.designation.trackerRole.get());
if (computedTracker2 == null) {
continue;
}
float dist1 = FastMath.abs(computedTracker1.position.distance(trackerFrame1.position));
float dist2 = FastMath.abs(computedTracker2.position.distance(trackerFrame2.position));
offset += FastMath.abs(dist2 - dist1);
offsetCount++;
}
return offsetCount > 0 ? offset / offsetCount : 0f;
}
}
@@ -0,0 +1,38 @@
package dev.slimevr.autobone.errors;
import dev.slimevr.autobone.AutoBoneTrainingStep;
import dev.slimevr.vr.processor.skeleton.HumanSkeleton;
import dev.slimevr.vr.trackers.ComputedTracker;
import dev.slimevr.vr.trackers.TrackerRole;
// The change in position of the ankle over time
public class SlideError implements IAutoBoneError {
@Override
public float getStepError(AutoBoneTrainingStep trainingStep) throws AutoBoneException {
return getSlideError(trainingStep.getSkeleton1(), trainingStep.getSkeleton2());
}
public static float getSlideError(HumanSkeleton skeleton1, HumanSkeleton skeleton2) {
// Calculate and average between both feet
return (getSlideError(skeleton1, skeleton2, TrackerRole.LEFT_FOOT)
+ getSlideError(skeleton1, skeleton2, TrackerRole.RIGHT_FOOT)) / 2f;
}
public static float getSlideError(
HumanSkeleton skeleton1,
HumanSkeleton skeleton2,
TrackerRole trackerRole
) {
// Calculate and average between both feet
return getSlideError(
skeleton1.getComputedTracker(trackerRole),
skeleton2.getComputedTracker(trackerRole)
);
}
public static float getSlideError(ComputedTracker tracker1, ComputedTracker tracker2) {
// Return the midpoint distance
return tracker1.position.distance(tracker2.position) / 2f;
}
}
@@ -1347,6 +1347,61 @@ public class HumanSkeleton extends Skeleton implements SkeletonConfigCallback {
}
}
public TransformNode getNode(BoneType nodeOffset) {
if (nodeOffset == null) {
return null;
}
switch (nodeOffset) {
case HEAD:
return headNode;
case NECK:
return neckNode;
case CHEST:
return chestNode;
case CHEST_TRACKER:
return trackerChestNode;
case WAIST:
return waistNode;
case HIP:
return hipNode;
case HIP_TRACKER:
return trackerWaistNode;
case LEFT_HIP:
return leftHipNode;
case RIGHT_HIP:
return rightHipNode;
case LEFT_UPPER_LEG:
return leftKneeNode;
case RIGHT_UPPER_LEG:
return rightKneeNode;
case RIGHT_KNEE_TRACKER:
return trackerRightKneeNode;
case LEFT_KNEE_TRACKER:
return trackerLeftKneeNode;
case LEFT_LOWER_LEG:
return leftAnkleNode;
case RIGHT_LOWER_LEG:
return rightAnkleNode;
case LEFT_FOOT:
return leftFootNode;
case RIGHT_FOOT:
return rightFootNode;
case LEFT_FOOT_TRACKER:
return trackerLeftFootNode;
case RIGHT_FOOT_TRACKER:
return trackerRightFootNode;
}
return null;
}
public void updatePoseAffectedByConfig(SkeletonConfigValue config) {
switch (config) {
case HEAD: