From 826ae8a9c7417a248084086c048a9505e99bd194 Mon Sep 17 00:00:00 2001 From: Luca Warmenhoven Date: Thu, 30 May 2024 17:41:06 +0200 Subject: [PATCH] Added motion recording functionality in InputProcessor (setRecording) and recorded data conversion to string for in the database (convertRecordedDataToString) --- .../com/example/fitbot/pepper/Pepper.java | 4 +- .../fitbot/ui/activities/FitnessActivity.java | 4 +- .../ui/components/ExerciseStatusElement.java | 2 +- .../util/processing/InputProcessor.java | 135 +++++++++++++++--- 4 files changed, 120 insertions(+), 25 deletions(-) diff --git a/code/src/Fitbot/app/src/main/java/com/example/fitbot/pepper/Pepper.java b/code/src/Fitbot/app/src/main/java/com/example/fitbot/pepper/Pepper.java index cb3cac8..7964ee2 100644 --- a/code/src/Fitbot/app/src/main/java/com/example/fitbot/pepper/Pepper.java +++ b/code/src/Fitbot/app/src/main/java/com/example/fitbot/pepper/Pepper.java @@ -43,7 +43,7 @@ public class Pepper { * * @param phrase The phrase to speak */ - public static void speak(String phrase) { + public static void say(String phrase) { addToEventQueue(new PepperSpeechEvent(phrase, DEFAULT_LOCALE)); } @@ -89,6 +89,7 @@ public class Pepper { case ACTION_SPEAK: if (!(event instanceof PepperSpeechEvent) || isSpeaking.get()) break; + PepperSpeechEvent speechEvent = (PepperSpeechEvent) event; isSpeaking.set(true); speechEvent @@ -105,6 +106,7 @@ public class Pepper { case ACTION_ANIMATE: if (!(event instanceof PepperAnimationEvent) || isAnimating.get()) break; + PepperAnimationEvent animationEvent = (PepperAnimationEvent) event; animationEvent .getAnimation(latestContext) diff --git a/code/src/Fitbot/app/src/main/java/com/example/fitbot/ui/activities/FitnessActivity.java b/code/src/Fitbot/app/src/main/java/com/example/fitbot/ui/activities/FitnessActivity.java index d25eb75..d9f9b3e 100644 --- a/code/src/Fitbot/app/src/main/java/com/example/fitbot/ui/activities/FitnessActivity.java +++ b/code/src/Fitbot/app/src/main/java/com/example/fitbot/ui/activities/FitnessActivity.java @@ -83,8 +83,8 @@ public class FitnessActivity extends RobotActivity implements RobotLifecycleCall motionProcessor.setInputHandler(personalMotionPreviewElement); }, (n) -> { int randomMessageIndex = (int) Math.floor(Math.random() * EXERCISE_NOT_FOUND_MESSAGES.length); - Pepper.speak(EXERCISE_NOT_FOUND_MESSAGES[randomMessageIndex]); - Pepper.speak(EXERCISE_NOT_FOUND_SEEK_HELP_MESSAGE); + Pepper.say(EXERCISE_NOT_FOUND_MESSAGES[randomMessageIndex]); + Pepper.say(EXERCISE_NOT_FOUND_SEEK_HELP_MESSAGE); NavigationManager.navigateToActivity(this, MainActivity.class); }); }); diff --git a/code/src/Fitbot/app/src/main/java/com/example/fitbot/ui/components/ExerciseStatusElement.java b/code/src/Fitbot/app/src/main/java/com/example/fitbot/ui/components/ExerciseStatusElement.java index 6363370..bdea480 100644 --- a/code/src/Fitbot/app/src/main/java/com/example/fitbot/ui/components/ExerciseStatusElement.java +++ b/code/src/Fitbot/app/src/main/java/com/example/fitbot/ui/components/ExerciseStatusElement.java @@ -103,7 +103,7 @@ public class ExerciseStatusElement extends View implements IInputHandler { projectVertex(new Vector3f(0, 0, 5.0f), getWidth(), getHeight()) }; - Pepper.speak(STARTING_PHRASES[(int) Math.floor(Math.random() * STARTING_PHRASES.length)]); + Pepper.say(STARTING_PHRASES[(int) Math.floor(Math.random() * STARTING_PHRASES.length)]); // Handler that is called every time the motion processor receives new data. this.motionProcessor.setInputHandler((rotationVector, deviceId) -> { diff --git a/code/src/Fitbot/app/src/main/java/com/example/fitbot/util/processing/InputProcessor.java b/code/src/Fitbot/app/src/main/java/com/example/fitbot/util/processing/InputProcessor.java index ea0775e..7e99392 100644 --- a/code/src/Fitbot/app/src/main/java/com/example/fitbot/util/processing/InputProcessor.java +++ b/code/src/Fitbot/app/src/main/java/com/example/fitbot/util/processing/InputProcessor.java @@ -11,21 +11,42 @@ import com.google.gson.JsonParser; import org.jetbrains.annotations.NotNull; import org.joml.Vector3f; +import java.util.ArrayList; +import java.util.List; + public class InputProcessor { private Vector3f[][] selfRotationVectorPaths; // Relative path of the motion data private Vector3f[][] targetRotationVectorPaths; // Target path of the motion data private final float sampleRate; // The sample rate of the motion sensor - private float exerciseDuration; + private float exerciseDurationInSeconds; + + /** + * This field is used to determine if the motion data is being recorded. + * If this is the case, instead of functioning normally, the element + * will record the movement that the user makes, store it in the + * `selfRotationVectorPaths` field and send it to the server. + */ + private boolean recordingMovement = false; + + /** + * Represents the duration of the recording in seconds. + * This field only has effect when the `recordingMovement` field is set to true. + */ + private float recordingDurationInSeconds = 0.0f; // How many seconds have passed since the start of the exercise. // The exercise starts whenever the `startListening` function is called. private double secondsPassed = 0.0D; private long lastTime; - private IInputHandler motionDataConsumer = (rotationVector, deviceId) -> { - }; + private IInputHandler motionDataConsumer; + + private static final String[] REQUIRED_SENSOR_JSON_PROPERTIES = + {"rotationX", "rotationY", "rotationZ", "type", "deviceId"}; + + // The web server that listens for incoming motion data. private WebServer server; @@ -42,7 +63,7 @@ public class InputProcessor { targetRotationVectorPaths = paths; this.sampleRate = inputSampleRate; - this.exerciseDuration = exerciseTime; + this.exerciseDurationInSeconds = exerciseTime; } /** @@ -53,20 +74,45 @@ public class InputProcessor { * @param exercise The exercise to use the paths for. */ public void useExercise(Exercise exercise) { + if ( this.recordingMovement ) + throw new IllegalStateException("Cannot change exercise while recording movement."); + this.selfRotationVectorPaths = new Vector3f[2][(int) (exercise.exerciseTimeInSeconds * this.sampleRate)]; this.targetRotationVectorPaths = new Vector3f[2][exercise.rightPath.getVectors().length]; - this.exerciseDuration = exercise.exerciseTimeInSeconds; + this.exerciseDurationInSeconds = exercise.exerciseTimeInSeconds; this.secondsPassed = 0.0D; this.lastTime = System.currentTimeMillis(); } /** - * Function for checking if the exercise has finished. + * Function for setting whether the motion data + * should be recorded or not. * - * @return True if the exercise has finished, false otherwise. + * @param recording Whether the motion data should be recorded. + * @param duration For how long the motion data should be recorded. + * This only has an effect if `recording` is true. + */ + public void setRecording(boolean recording, float duration) { + this.recordingMovement = recording; + this.recordingDurationInSeconds = duration; + if (recording) { + this.secondsPassed = 0.0D; + this.lastTime = System.currentTimeMillis(); + + } + } + + /** + * Function for checking if the exercise or recording has finished. + * This function will return true if the execution of the exercise has finished or + * if the recording has finished, depending on the state of the `recordingMovement` field. + * + * @return Whether the exercise or recording has finished. */ public boolean hasFinished() { - return this.secondsPassed >= this.exerciseDuration; + return this.recordingMovement ? + (this.secondsPassed >= this.recordingDurationInSeconds) : + (this.secondsPassed >= this.exerciseDurationInSeconds); } /** @@ -120,14 +166,8 @@ public class InputProcessor { JsonObject object = json.getAsJsonObject(); - String[] required = { - "rotationX", "rotationY", "rotationZ", - "type", - "deviceId" - }; - // Ensure all properties are present in the received JSON object - for (String s : required) { + for (String s : REQUIRED_SENSOR_JSON_PROPERTIES) { if (!object.has(s)) return; } @@ -164,9 +204,58 @@ public class InputProcessor { return; selfRotationVectorPaths[deviceId][selfIndex] = rotation; + + if ( this.recordingMovement && this.hasFinished()) { + // Do something with the recorded data. + this.recordingMovement = false; + // Convert recorded data from `selfRotationVectorPaths` to string, and + // publish to database, or do something else with it. + + String converted = convertRecordedDataToString(); + + // Do something with it + Log.i("MotionProcessor", "Converted data: "); + Log.i("MotionProcessor", converted); + } } } + private String convertRecordedDataToString() + { + // First, remove empty entries + StringBuilder pathBuilder = new StringBuilder(); + + int[] intBits = new int[3]; + char[] vectorChars = new char[12]; // 4 bytes per scalar, 12 chars per vector + + // Iterate over all devices. In the current instance, it's 2. + for ( int deviceId = 0; deviceId < selfRotationVectorPaths.length; deviceId++) { + for (Vector3f dataPoint : selfRotationVectorPaths[deviceId]) { + if (dataPoint != null) { + // Convert float to int bits for conversion to char + intBits[0] = Float.floatToIntBits(dataPoint.x); + intBits[1] = Float.floatToIntBits(dataPoint.y); + intBits[2] = Float.floatToIntBits(dataPoint.z); + + // Convert int bits to char, in Big Endian order. + // This is important for converting back to float later. + for (int i = 0; i < 3; i++) { + vectorChars[i * 4] = (char) (intBits[i] >> 24); + vectorChars[i * 4 + 1] = (char) (intBits[i] >> 16); + vectorChars[i * 4 + 2] = (char) (intBits[i] >> 8); + vectorChars[i * 4 + 3] = (char) intBits[i]; + } + + pathBuilder.append(vectorChars); + } + } + // Add a separator between devices + if ( deviceId < selfRotationVectorPaths.length - 1) + pathBuilder.append(";"); + } + return pathBuilder.toString(); + } + /** * Method for getting the current progress of the exercise. * The return value will range between 0.0 and 1.0. @@ -174,7 +263,7 @@ public class InputProcessor { * @return The current progress of the exercise. */ public double getCurrentProgress() { - return secondsPassed / exerciseDuration; + return secondsPassed / exerciseDurationInSeconds; } /** @@ -204,12 +293,16 @@ public class InputProcessor { return 0.0d; // Index of the current rotation vector - int targetIndex = (int) ((this.exerciseDuration / this.targetRotationVectorPaths[sensorId].length) * time); + int targetIndex = (int) ((this.exerciseDurationInSeconds / this.targetRotationVectorPaths[sensorId].length) * time); int selfIndex = (int) (this.selfRotationVectorPaths[sensorId].length / this.sampleRate * time); // Ensure the indexes are within the bounds of the array - if (targetIndex >= 0 && targetIndex <= this.targetRotationVectorPaths[sensorId].length - 1 && - selfIndex >= 0 && selfIndex <= this.selfRotationVectorPaths[sensorId].length - 1) { + if ( + targetIndex >= 0 && targetIndex <= this.targetRotationVectorPaths[sensorId].length - 1 && + selfIndex >= 0 && selfIndex <= this.selfRotationVectorPaths[sensorId].length - 1 && + this.selfRotationVectorPaths[sensorId][selfIndex] != null && + this.targetRotationVectorPaths[sensorId][targetIndex] != null + ) { return this.selfRotationVectorPaths[sensorId][selfIndex].distance(this.targetRotationVectorPaths[sensorId][targetIndex]); } return 0.0d; @@ -224,10 +317,10 @@ public class InputProcessor { */ public double getAverageError(int sensorId) { double error = 0; - for (int i = 0; i < this.exerciseDuration; i++) { + for (int i = 0; i < this.exerciseDurationInSeconds; i++) { error += getError(sensorId, i); } - return error / this.exerciseDuration; + return error / this.exerciseDurationInSeconds; } public float secondsPassed() {