From f69fbe8e5b2001fb14136d1ae24daae892b10689 Mon Sep 17 00:00:00 2001 From: Luca Warmenhoven Date: Wed, 5 Jun 2024 11:29:32 +0200 Subject: [PATCH] Updated InputProcessor's convertRecordedDataToString to JSON output, updated GesturePath -> AnglePath --- .../com/example/fitbot/exercise/Exercise.java | 11 +- .../fitbot/exercise/ExerciseManager.java | 6 +- .../fitbot/ui/activities/FitnessActivity.java | 12 +- .../fitbot/ui/activities/MainActivity.java | 1 + .../example/fitbot/util/path/AnglePath.java | 63 +++++++++ .../example/fitbot/util/path/GesturePath.java | 126 ------------------ .../util/processing/InputProcessor.java | 97 ++++++++------ .../com/example/fitbot/PathSegmentTest.java | 37 ----- 8 files changed, 132 insertions(+), 221 deletions(-) create mode 100644 code/src/Fitbot/app/src/main/java/com/example/fitbot/util/path/AnglePath.java delete mode 100644 code/src/Fitbot/app/src/main/java/com/example/fitbot/util/path/GesturePath.java delete mode 100644 code/src/Fitbot/app/src/test/java/com/example/fitbot/PathSegmentTest.java diff --git a/code/src/Fitbot/app/src/main/java/com/example/fitbot/exercise/Exercise.java b/code/src/Fitbot/app/src/main/java/com/example/fitbot/exercise/Exercise.java index 18153d7..7444f2b 100644 --- a/code/src/Fitbot/app/src/main/java/com/example/fitbot/exercise/Exercise.java +++ b/code/src/Fitbot/app/src/main/java/com/example/fitbot/exercise/Exercise.java @@ -1,12 +1,12 @@ package com.example.fitbot.exercise; -import com.example.fitbot.util.path.GesturePath; +import com.example.fitbot.util.path.AnglePath; public class Exercise { public final EMuscleGroup muscleGroup; - public final GesturePath leftPath; - public final GesturePath rightPath; + public final AnglePath leftPath; + public final AnglePath rightPath; public final String name; public final String shortDescription; public final String description; @@ -26,7 +26,10 @@ public class Exercise { * @param imageUrl The URL of the image. * @param videoUrl The URL of the video. */ - public Exercise(EMuscleGroup muscleGroup, String name,String shortDescription, String description, String imageUrl, String videoUrl, GesturePath leftPath, GesturePath rightPath, float exerciseTimeInSeconds) { + public Exercise(EMuscleGroup muscleGroup, String name, String shortDescription, + String description, String imageUrl, String videoUrl, + AnglePath leftPath, AnglePath rightPath, float exerciseTimeInSeconds) { + this.name = name; this.muscleGroup = muscleGroup; this.shortDescription = shortDescription; diff --git a/code/src/Fitbot/app/src/main/java/com/example/fitbot/exercise/ExerciseManager.java b/code/src/Fitbot/app/src/main/java/com/example/fitbot/exercise/ExerciseManager.java index 0de72d0..8a9750e 100644 --- a/code/src/Fitbot/app/src/main/java/com/example/fitbot/exercise/ExerciseManager.java +++ b/code/src/Fitbot/app/src/main/java/com/example/fitbot/exercise/ExerciseManager.java @@ -1,6 +1,6 @@ package com.example.fitbot.exercise; -import com.example.fitbot.util.path.GesturePath; +import com.example.fitbot.util.path.AnglePath; import com.google.gson.JsonObject; import com.google.gson.JsonParser; @@ -118,8 +118,8 @@ public class ExerciseManager { content.get(PROPERTY_DESC).getAsString(), content.get(PROPERTY_IMAGE_URL).getAsString(), content.get(PROPERTY_VIDEO_URL).getAsString(), - GesturePath.fromString(leftRightData[0]), - GesturePath.fromString(leftRightData[1]), + AnglePath.fromString(leftRightData[0]), + AnglePath.fromString(leftRightData[1]), DEFAULT_SEGMENT_SPEED ); } catch (Exception e) { diff --git a/code/src/Fitbot/app/src/main/java/com/example/fitbot/ui/activities/FitnessActivity.java b/code/src/Fitbot/app/src/main/java/com/example/fitbot/ui/activities/FitnessActivity.java index 7282278..c6a89ab 100644 --- a/code/src/Fitbot/app/src/main/java/com/example/fitbot/ui/activities/FitnessActivity.java +++ b/code/src/Fitbot/app/src/main/java/com/example/fitbot/ui/activities/FitnessActivity.java @@ -3,13 +3,10 @@ package com.example.fitbot.ui.activities; import android.app.Dialog; import android.content.Context; import android.graphics.drawable.ColorDrawable; -import android.media.MediaPlayer; -import android.net.Uri; import android.os.Bundle; import android.util.Log; import android.widget.TextView; import android.view.View; -import android.view.WindowManager; import android.widget.Button; import android.widget.VideoView; @@ -102,7 +99,7 @@ public class FitnessActivity extends RobotActivity implements RobotLifecycleCall exerciseStatusElement.post(() -> { this.fetchExerciseAsync((exercise) -> { // Acquire paths from the exercise and provide them to the motion processor - Vector3f[][] vectors = new Vector3f[][]{exercise.leftPath.getVectors(), exercise.rightPath.getVectors()}; + Vector3f[][] vectors = new Vector3f[][]{exercise.leftPath.getAngleVectors(), exercise.rightPath.getAngleVectors()}; motionProcessor = new InputProcessor(vectors, exercise.exerciseTimeInSeconds, SENSOR_SAMPLE_RATE); @@ -147,12 +144,7 @@ public class FitnessActivity extends RobotActivity implements RobotLifecycleCall // Set a listener to repeat the video while (EXERCISE_REP > 1) { - videoView.setOnCompletionListener(new MediaPlayer.OnCompletionListener() { - @Override - public void onCompletion(MediaPlayer mp) { - videoView.start(); // start the video again - } - }); + videoView.setOnCompletionListener(mp -> videoView.start()); // start the video again); EXERCISE_REP--; } }); diff --git a/code/src/Fitbot/app/src/main/java/com/example/fitbot/ui/activities/MainActivity.java b/code/src/Fitbot/app/src/main/java/com/example/fitbot/ui/activities/MainActivity.java index 3aaca38..934a9df 100644 --- a/code/src/Fitbot/app/src/main/java/com/example/fitbot/ui/activities/MainActivity.java +++ b/code/src/Fitbot/app/src/main/java/com/example/fitbot/ui/activities/MainActivity.java @@ -16,6 +16,7 @@ import android.view.WindowManager; import android.widget.Button; import com.example.fitbot.R; +import com.example.fitbot.pepper.Pepper; import com.example.fitbot.util.NavigationManager; import java.io.OutputStream; diff --git a/code/src/Fitbot/app/src/main/java/com/example/fitbot/util/path/AnglePath.java b/code/src/Fitbot/app/src/main/java/com/example/fitbot/util/path/AnglePath.java new file mode 100644 index 0000000..7df8ef9 --- /dev/null +++ b/code/src/Fitbot/app/src/main/java/com/example/fitbot/util/path/AnglePath.java @@ -0,0 +1,63 @@ +package com.example.fitbot.util.path; + +import org.joml.Vector3f; + +public class AnglePath { + + // The angles + private final Vector3f[] angles; + + /** + * Create a new gesture path with a given set of vectors and curvature. + * + * @param angles The vectors that make up the path. + */ + public AnglePath(Vector3f[] angles) { + this.angles = angles; + } + + /** + * Method for retrieving the vectors of the GesturePath. + */ + public Vector3f[] getAngleVectors() { + return this.angles; + } + + /** + * Function for converting a string to a GesturePath object. + * The input string bytes will be directly converted into 3d vectors. + * Every scalar is composed of 32 bits (4 characters), meaning 96 bits per vector. + *

+ * Note: ASCII to Vector conversion is done in Big Endian format (most significant byte first). + * + * @param input The string to convert + * @return The GesturePath object + */ + + public static AnglePath fromString(String input) { + if (input == null) + throw new IllegalArgumentException("Input string cannot be null"); + byte[] bytes = input.getBytes(); + + // Check if the input string contains a valid amount of bytes (12 bytes per vector) + if (input.length() % 12 != 0) { + throw new IllegalArgumentException("Invalid input string length (" + input.length() + " bytes provided - must be a multiple of 12)"); + } + Vector3f[] vectors = new Vector3f[input.length() / 12]; + + float[] xyz = new float[3]; + for (int i = 0; i < bytes.length; i += 12) { + for (int j = 0; j < 3; j++) { + + xyz[j] = Float.intBitsToFloat( + (bytes[i + j * 4] & 0xFF) << 24 | + (bytes[i + j * 4 + 1] & 0xFF) << 16 | + (bytes[i + j * 4 + 2] & 0xFF) << 8 | + (bytes[i + j * 4 + 3] & 0xFF) + ); + } + vectors[i / 12] = new Vector3f(xyz[0], xyz[1], xyz[2]); + } + return new AnglePath(vectors); + } +} diff --git a/code/src/Fitbot/app/src/main/java/com/example/fitbot/util/path/GesturePath.java b/code/src/Fitbot/app/src/main/java/com/example/fitbot/util/path/GesturePath.java deleted file mode 100644 index f3b8cb3..0000000 --- a/code/src/Fitbot/app/src/main/java/com/example/fitbot/util/path/GesturePath.java +++ /dev/null @@ -1,126 +0,0 @@ -package com.example.fitbot.util.path; - -import org.joml.Vector3f; - -public class GesturePath { - - // The vectors that make up the path. - private final PathSegment[] segments; - - /** - * Create a new gesture path with a given set of vectors and curvature. - * - * @param vectors The vectors that make up the path. - */ - public GesturePath(Vector3f[] vectors) { - if (vectors.length < 2) { - this.segments = new PathSegment[1]; - this.segments[0] = new PathSegment( - new Vector3f(0, 0, 0), - new Vector3f(0, 0, 0) - ); - return; - } - - this.segments = new PathSegment[vectors.length - 1]; - for (int i = 0; i < vectors.length - 1; i++) - segments[i] = new PathSegment(vectors[i], vectors[i + 1]); - } - - /** - * Constructor for a GesturePath with provided PathSegments. - * - * @param segments The PathSegments to initialize the path with. - */ - public GesturePath(PathSegment... segments) { - this.segments = segments; - } - - /** - * Getter method for retrieving the path segments of this GesturePath. - * - * @return The path segments. - */ - public PathSegment[] getSegments() { - return segments; - } - - /** - * Method for retrieving the vectors of the GesturePath. - */ - public Vector3f[] getVectors() { - Vector3f[] vectors = new Vector3f[segments.length + 1]; - vectors[0] = segments[0].getStart(); - for (int i = 0; i < segments.length; i++) - vectors[i + 1] = segments[i].getEnd(); - - return vectors; - } - - /** - * Method for retrieving the closest path segment to a reference point. - * - * @param reference The reference point to find the closest path segment to. - * @return The closest path segment to the reference point. - */ - public PathSegment closest(Vector3f reference) { - // If there's only one segment, return that one. - if (segments.length == 1) - return segments[0]; - - PathSegment closest = segments[0]; - for (int i = 1; i < segments.length; i++) - closest = PathSegment.closer(closest, segments[i], reference); - - return closest; - } - - /** - * Get the error between an arbitrary path segment and the reference point. - * - * @param referencePoint The reference point to calculate the error of. - * @return The error offset between the path and the reference point. - */ - public double getError(Vector3f referencePoint) { - return closest(referencePoint).difference(referencePoint); // Get the closest segment and calculate the error. - } - - - /** - * Function for converting a string to a GesturePath object. - * The input string bytes will be directly converted into 3d vectors. - * Every scalar is composed of 32 bits (4 characters), meaning 96 bits per vector. - *

- * Note: ASCII to Vector conversion is done in Big Endian format (most significant byte first). - * - * @param input The string to convert - * @return The GesturePath object - */ - - public static GesturePath fromString(String input) { - if (input == null) - throw new IllegalArgumentException("Input string cannot be null"); - byte[] bytes = input.getBytes(); - - // Check if the input string contains a valid amount of bytes (12 bytes per vector) - if (input.length() % 12 != 0) { - throw new IllegalArgumentException("Invalid input string length (" + input.length() + " bytes provided - must be a multiple of 12)"); - } - Vector3f[] vectors = new Vector3f[input.length() / 12]; - - float[] xyz = new float[3]; - for (int i = 0; i < bytes.length; i += 12) { - for (int j = 0; j < 3; j++) { - - xyz[j] = Float.intBitsToFloat( - (bytes[i + j * 4] & 0xFF) << 24 | - (bytes[i + j * 4 + 1] & 0xFF) << 16 | - (bytes[i + j * 4 + 2] & 0xFF) << 8 | - (bytes[i + j * 4 + 3] & 0xFF) - ); - } - vectors[i / 12] = new Vector3f(xyz[0], xyz[1], xyz[2]); - } - return new GesturePath(vectors); - } -} diff --git a/code/src/Fitbot/app/src/main/java/com/example/fitbot/util/processing/InputProcessor.java b/code/src/Fitbot/app/src/main/java/com/example/fitbot/util/processing/InputProcessor.java index e95008a..8e358b7 100644 --- a/code/src/Fitbot/app/src/main/java/com/example/fitbot/util/processing/InputProcessor.java +++ b/code/src/Fitbot/app/src/main/java/com/example/fitbot/util/processing/InputProcessor.java @@ -2,12 +2,15 @@ package com.example.fitbot.util.processing; import android.util.Log; +import com.aldebaran.qi.sdk.object.geometry.Vector3; import com.example.fitbot.exercise.Exercise; import com.example.fitbot.exercise.ExerciseManager; import com.example.fitbot.util.server.WebServer; +import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; +import com.google.gson.JsonPrimitive; import org.jetbrains.annotations.NotNull; import org.joml.Vector3f; @@ -17,12 +20,15 @@ import java.util.List; public class InputProcessor { - private Vector3f[][] selfRotationVectorPaths; // Relative path of the motion data + private List[] selfRotationVectorPaths; // Relative path of the motion data + //private Vector3f[][] selfRotationVectorPaths; // Relative path of the motion data private Vector3f[][] targetRotationVectorPaths; // Target path of the motion data private final float sampleRate; // The sample rate of the motion sensor private float exerciseDurationInSeconds; + private float exerciseScore = 0.0F; + /** * This field is used to determine if the motion data is being recorded. * If this is the case, instead of functioning normally, the element @@ -45,7 +51,7 @@ public class InputProcessor { private IInputHandler motionDataConsumer; private static final String[] REQUIRED_SENSOR_JSON_PROPERTIES = - {"rotationX", "rotationY", "rotationZ", "type", "deviceId"}; + {"rotationX", "rotationY", "rotationZ", "deviceId"}; // The web server that listens for incoming motion data. private WebServer server; @@ -60,7 +66,9 @@ public class InputProcessor { * @param inputSampleRate The sample rate of the motion sensor. */ public InputProcessor(Vector3f[][] paths, float exerciseTime, float inputSampleRate) { - selfRotationVectorPaths = new Vector3f[paths.length][(int) (exerciseTime * inputSampleRate)]; + this.selfRotationVectorPaths = new ArrayList[2]; + this.selfRotationVectorPaths[0] = new ArrayList<>(); + this.selfRotationVectorPaths[1] = new ArrayList<>(); targetRotationVectorPaths = paths; this.sampleRate = inputSampleRate; @@ -78,10 +86,11 @@ public class InputProcessor { if ( this.recordingMovement ) throw new IllegalStateException("Cannot change exercise while recording movement."); - this.selfRotationVectorPaths = new Vector3f[2][(int) (exercise.exerciseTimeInSeconds * this.sampleRate)]; - this.targetRotationVectorPaths = new Vector3f[2][exercise.rightPath.getVectors().length]; - this.targetRotationVectorPaths[0] = exercise.leftPath.getVectors(); - this.targetRotationVectorPaths[1] = exercise.rightPath.getVectors(); + this.selfRotationVectorPaths[0] = new ArrayList<>(); + this.selfRotationVectorPaths[1] = new ArrayList<>(); + this.targetRotationVectorPaths = new Vector3f[2][exercise.rightPath.getAngleVectors().length]; + this.targetRotationVectorPaths[0] = exercise.leftPath.getAngleVectors(); + this.targetRotationVectorPaths[1] = exercise.rightPath.getAngleVectors(); this.exerciseDurationInSeconds = exercise.exerciseTimeInSeconds; this.secondsPassed = 0.0D; this.lastTime = System.currentTimeMillis(); @@ -101,7 +110,8 @@ public class InputProcessor { if (recording) { this.secondsPassed = 0.0D; this.lastTime = System.currentTimeMillis(); - this.selfRotationVectorPaths = new Vector3f[2][(int) (duration * this.sampleRate)]; + this.selfRotationVectorPaths[0] = new ArrayList<>(); + this.selfRotationVectorPaths[1] = new ArrayList<>(); } } @@ -161,7 +171,8 @@ public class InputProcessor { try { Log.i("MotionProcessor", "Time passed: " + this.secondsPassed + "s"); - Log.i("MotionProcessor", "Recording: " + this.recordingMovement + ", " + this.secondsPassed + " / " + this.recordingDurationInSeconds); + if ( this.recordingMovement) + Log.i("MotionProcessor", this.secondsPassed + " / " + this.recordingDurationInSeconds); Log.i("MotionProcessor", "Received packet data: " + data); JsonElement json = JsonParser.parseString(data); @@ -178,9 +189,11 @@ public class InputProcessor { } // Parse the data - Vector3f rotation = new Vector3f(object.get("rotationX").getAsFloat(), object.get("rotationY").getAsFloat(), object.get("rotationZ").getAsFloat()); + Vector3f rotation = new Vector3f( + object.get("rotationX").getAsFloat(), + object.get("rotationY").getAsFloat(), + object.get("rotationZ").getAsFloat()); int deviceId = object.get("deviceId").getAsInt(); - String type = object.get("type").getAsString(); // Parse the retrieved data parseRotationVector(rotation, deviceId); @@ -203,7 +216,7 @@ public class InputProcessor { // Supposed index of the current rotation vector in the `rotationVectorPaths` array - int selfIndex = (int) (secondsPassed * sampleRate); + this.selfRotationVectorPaths[deviceId].add(rotation); if ( this.recordingMovement && this.secondsPassed >= this.recordingDurationInSeconds) { // Do something with the recorded data. @@ -217,11 +230,8 @@ public class InputProcessor { Log.i("MotionProcessor", "Converted data: "); Log.i("MotionProcessor", converted); } - motionDataConsumer.accept(rotation, deviceId); - if (selfIndex >= selfRotationVectorPaths[deviceId].length || selfIndex < 0) - return; - selfRotationVectorPaths[deviceId][selfIndex] = rotation; + motionDataConsumer.accept(rotation, deviceId); } } @@ -234,38 +244,36 @@ public class InputProcessor { */ private String convertRecordedDataToString() { - // First, remove empty entries - StringBuilder pathBuilder = new StringBuilder(); - int[] intBits = new int[3]; char[] vectorChars = new char[12]; // 4 bytes per scalar, 12 chars per vector + JsonArray jsonArray = new JsonArray(); + /* + * Convert to JSON array in the following format: + * [ + * { "deviceId": number, "data": [ [x, y, z], [x, y, z], ... ] }, + * ] + */ // Iterate over all devices. In the current instance, it's 2. for ( int deviceId = 0; deviceId < selfRotationVectorPaths.length; deviceId++) { - for (Vector3f dataPoint : selfRotationVectorPaths[deviceId]) { - if (dataPoint != null) { - // Convert float to int bits for conversion to char - intBits[0] = Float.floatToIntBits(dataPoint.x); - intBits[1] = Float.floatToIntBits(dataPoint.y); - intBits[2] = Float.floatToIntBits(dataPoint.z); + JsonObject jsonDeviceObject = new JsonObject(); + jsonDeviceObject.addProperty("deviceId", deviceId); - // Convert int bits to char, in Big Endian order. - // This is important for converting back to float later. - for (int i = 0; i < 3; i++) { - vectorChars[i * 4] = (char) (intBits[i] >> 24); - vectorChars[i * 4 + 1] = (char) (intBits[i] >> 16); - vectorChars[i * 4 + 2] = (char) (intBits[i] >> 8); - vectorChars[i * 4 + 3] = (char) intBits[i]; - } + // Data array + JsonArray jsonDeviceDataArray = new JsonArray(); - pathBuilder.append(vectorChars); - } + for ( Vector3f vector : selfRotationVectorPaths[deviceId]) { + JsonArray jsonScalarArray = new JsonArray(); + jsonScalarArray.add(vector.x); + jsonScalarArray.add(vector.y); + jsonScalarArray.add(vector.z); + jsonDeviceDataArray.add(jsonScalarArray); } - // Add a separator between devices - if ( deviceId < selfRotationVectorPaths.length - 1) - pathBuilder.append(ExerciseManager.PATH_DELIMITER); + jsonDeviceObject.add("data", jsonDeviceDataArray); + + jsonArray.add(jsonDeviceObject); } - return pathBuilder.toString(); + return jsonArray.toString(); } /** @@ -288,6 +296,13 @@ public class InputProcessor { this.motionDataConsumer = consumer; } + /** + * Function for getting the combined (average) error value of both sensors. + public double getCombinedError() + { + + }*/ + /** * Function for getting the error offsets of the user's path compared to the * target path at a given point in time. @@ -300,7 +315,7 @@ public class InputProcessor { */ public double getError(int sensorId, float time) { - // Ensure the sensor ID is within the bounds of the array + /*// Ensure the sensor ID is within the bounds of the array if (sensorId < 0 || sensorId >= selfRotationVectorPaths.length) return 0.0d; @@ -316,7 +331,7 @@ public class InputProcessor { this.targetRotationVectorPaths[sensorId][targetIndex] != null ) { return this.selfRotationVectorPaths[sensorId][selfIndex].distance(this.targetRotationVectorPaths[sensorId][targetIndex]); - } + }*/ return 0.0d; } diff --git a/code/src/Fitbot/app/src/test/java/com/example/fitbot/PathSegmentTest.java b/code/src/Fitbot/app/src/test/java/com/example/fitbot/PathSegmentTest.java deleted file mode 100644 index 0ab0f1d..0000000 --- a/code/src/Fitbot/app/src/test/java/com/example/fitbot/PathSegmentTest.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.example.fitbot; - -import static org.junit.Assert.assertEquals; - -import com.example.fitbot.util.path.GesturePath; -import com.example.fitbot.util.path.PathSegment; - -import org.joml.Vector3f; -import org.junit.Test; - -public class PathSegmentTest { - - @Test - public void testPathSegment() { - // Test the PathSegment class - Vector3f[] vectors = new Vector3f[2]; - vectors[0] = new Vector3f(0, 0, 0); - vectors[1] = new Vector3f(1, 1, 1); - GesturePath path = new GesturePath(vectors); - PathSegment[] segments = path.getSegments(); - assertEquals(1, segments.length); - assertEquals(new Vector3f(0, 0, 0), segments[0].getStart()); - assertEquals(new Vector3f(1, 1, 1), segments[0].getEnd()); - assertEquals(new Vector3f(0.5f, 0.5f, 0.5f), segments[0].interpolate(0.5)); - } - - - @Test - public void test_pathSegmentInterpolation() { - Vector3f start = new Vector3f(0, 0, 0); - Vector3f end = new Vector3f(1, 1, 1); - PathSegment segment = new PathSegment(start, end); - assertEquals(new Vector3f(0.5f, 0.5f, 0.5f), segment.interpolate(0.5)); - } - - -}