Resistance Arena

Resistance Arena is a VR game where the user attacks by performing upper body exercises with VR controllers instead of pressing buttons. A machine learning algorithm determines if you have performed a given exercise, and gives a score based on how correct the form was.

Technologies

Unity

My Role

Sole Developer

This project required a very strong understanding of 3D math for creating realistic physics-based gameplay with a VR headset - quaternion math, physics, and matrix transformations between spaces were all utilized to make the game experience. This was the most hands-on gameplay development experience I have had due to the nature of the machine learning, I needed to train the AI by performing the upper body exercises correctly hundreds of times then developing and attaching abilities to the gestures to see how it felt in the game. There were many iterations and I quickly learned the importance of game feel and play testing to really make a game experience that players will be engaged with. It also posed the additional gameplay and design challenge of making a traditionally difficult or sometimes unenjoyable activity (exercising) fun for the player.

Media

Screenshots

The VR Menu navigation

The arm and ray-cast pointer used to interact with the menu

The "Fireball" attack cast by performing a bicep curl with one hand

The "Fireball" attack colliding with a target practice crate

The "Rock Throw" attack cast by performing an overhead press with both hands

The "Rock Throw" attack hitting the ground and producing an AOE attack

Gameplay Video

Code

The code that acts as the interface between user input and the machine learning Gesture Recognition code. This begins and ends gestures, parses the gesture recognition result, then creates the appropriate in-game attack with their respective physics components. This is the conduit through which all abilities must pass, so this code needed to be the most stable, refined, and have the most carefully tweaked values. The thresholds for gesture recognition and correctness came from many hours of play testing and having people of different sizes playing the game to come to the final values that allowed for the most accessibility while still keeping the gesture recognition robust enough to require reasonable form when performing exercises.

									
float trigger_left = Input.GetAxis("LeftControllerTrigger");
float trigger_right = Input.GetAxis("RightControllerTrigger");

GameObject hmd = GameObject.Find("VR Camera");
Vector3 hmd_p = hmd.transform.position;
Quaternion hmd_q = hmd.transform.rotation;

// If the user presses either controller's trigger, we start a new gesture.
if (trigger_pressed_left == false && trigger_left > 0.9)
{
	// Controller trigger pressed.
	trigger_pressed_left = true;
	gc.startStroke(Side_Left, hmd_p, hmd_q);
	gesture_started = true;
}
if (trigger_pressed_right == false && trigger_right > 0.9)
{
	// Controller trigger pressed.
	trigger_pressed_right = true;
	gc.startStroke(Side_Right, hmd_p, hmd_q);
	gesture_started = true;
}
if (gesture_started == false)
{
	// nothing to do.
	return;
}

// If we arrive here, the user is currently dragging with one of the controllers.
if (trigger_pressed_left == true)
{
	if (trigger_left < 0.85)
	{
		// User let go of a trigger and held controller still
		gc.endStroke(Side_Left);
		trigger_pressed_left = false;
	}
	else
	{
		// User still dragging or still moving after trigger pressed
		GameObject left_hand = GameObject.Find("Left Hand");
		gc.contdStrokeQ(Side_Left, left_hand.transform.position, left_hand.transform.rotation);
		// Show the stroke by instatiating new objects
		GameObject left_hand_pointer = GameObject.FindGameObjectWithTag("Left Pointer");
	}
}

if (trigger_pressed_right == true)
{
	if (trigger_right < 0.85)
	{
		// User let go of a trigger and held controller still
		gc.endStroke(Side_Right);
		trigger_pressed_right = false;
	}
	else
	{
		// User still dragging or still moving after trigger pressed
		GameObject right_hand = GameObject.Find("Right Hand");
		gc.contdStrokeQ(Side_Right, right_hand.transform.position, right_hand.transform.rotation);

		// Show the stroke by instatiating new objects
		GameObject right_hand_pointer = GameObject.FindGameObjectWithTag("Right Pointer");
	}
}

if (trigger_pressed_left || trigger_pressed_right)
{
	// User still dragging with either hand
	return;
}
// else: if we arrive here, the user let go of both triggers, ending the gesture.
gesture_started = false;

double similarity = -1.0;

int gestureId = gc.identifyGestureCombination(ref similarity);

if (gestureId < 0)
{
	HUDText.text = "Failed to identify gesture";
	return; // something went wrong
}

string gestureName = gc.getGestureCombinationName(gestureId);

if (similarity < /*0.46*/0.3)
{
	HUDText.text = "Gesture not fully recognized\nWere you trying to perform a(n): " + gestureName + "?\nAccuracy Score: " + similarity;
}
else
{
	if (gestureName.Contains("bc"))
	{
		if (gestureName.Contains("left"))
		{
			HUDText.text = "You performed a left-handed bicep curl!";
		}
		else
		{
			HUDText.text = "You performed a right-handed bicep curl!";
		}

		bcStrike.transform.position = new Vector3
			(
				hmd_p.x + (hmd.transform.forward.x * 1.5f),
				hmd_p.y + (hmd.transform.forward.y * 1.5f),
				hmd_p.z + (hmd.transform.forward.z * 1.5f)
			);
		bcStrike.transform.rotation = hmd_q;
		bcStrike.GetComponent().velocity = hmd.transform.forward * 6.5f;
		bcStrike.GetComponent().angularVelocity = new Vector3();
	}

	if (gestureName.Contains("ohp"))
	{
		HUDText.text = "You performed an overhead press!";

		ohpAttack.GetComponent().constraints = RigidbodyConstraints.None;
		ohpAttack.transform.position = new Vector3
			(
				hmd_p.x + (hmd.transform.forward.x * 1.5f),
				hmd_p.y + (hmd.transform.forward.y * 1.5f),
				hmd_p.z + (hmd.transform.forward.z * 1.5f)
			);
		ohpAttack.transform.rotation = hmd_q;
		ohpAttack.GetComponent().velocity = new Vector3
			(
				hmd.transform.forward.x * 4.5f,
				(hmd.transform.forward.y + 5f) * 2f,
				hmd.transform.forward.z * 4.5f
			);
		ohpAttack.GetComponent().angularVelocity = new Vector3(1.5f, 1f, 0f);
	}
}

The code for the bicep curl "Fireball" attack. This attack travels in a straight line where ever the user is facing and spawns at the hand that performed the bicep curl. The fireball immediately explodes on impact. This attack is meant to feel quick, efficient, and easy to understand; this is accomplished with its clear projectile path and fast travel speed.

									
public class BCStrike : MonoBehaviour
{
	public GameObject explosion;
	void OnCollisionEnter(Collision collision)
	{
		// Place explosion Prefab on impact
		var expl = Instantiate(explosion, gameObject.transform.position, new Quaternion());
		Destroy(expl, 1.0f);

		// Reset the strike's position
		gameObject.transform.rotation = new Quaternion();
		gameObject.transform.position = new Vector3(0, -100f, 0);
		gameObject.GetComponent().velocity = new Vector3(0, 0, 0);
	}
}

The code for the overhead press "Rock Throw" attack. This attack hurls a rock into the air in whatever direction the user is facing. This rock falls until it hits the ground and spawns a large area of effect force wave destroying anything in it. This attack is meant to feel strong, bulky, and somewhat unwieldy; this is accomplished with its slow speed, large arc, and the fact that it breaks through any objects on its intended path.

									
public class OHPAttack : MonoBehaviour
{
	public GameObject shockWave;
	public GameObject shockWaveHitBox;

	void OnCollisionEnter(Collision collision)
	{
		// Only handle collisions at ground level AND don't handle collisions with the hitbox
		if (gameObject.transform.position.y < 0.8 && !collision.gameObject.name.Equals("ShockwaveCollision(Clone)"))
		{
			// Place impact effects
			var swPos = new Vector3
				(
					gameObject.transform.position.x,
					0.0f,
					gameObject.transform.position.z
				);

			var sw = Instantiate(shockWave, swPos, new Quaternion());
			Destroy(sw, 1.0f);
			var swHit = Instantiate(shockWaveHitBox, swPos, new Quaternion());
			Destroy(swHit, 1.0f);

			// Freeze rock in place to simulate "landing"
			gameObject.GetComponent().constraints = RigidbodyConstraints.FreezeAll;
		}
	}
}

See the code repository HERE