Creating Unity VR Drawing App
by Jennifer Wang (2022)
Requirements
Installation of Unity (2019)
Follow the Unity XR Setup Guide (use a device-based XR Origin instead)
Step 1. Create an empty object called Mesh and add the components Mesh Filter and Mesh Renderer (select a material for element 0 as the color of the brush stroke). Add the script "Mesh" with the following code as a component to this object.
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
public class BrushStrokeMesh : MonoBehaviour {
[SerializeField]
private float _brushStrokeWidth = 0.05f;
private Mesh _mesh;
private List<Vector3> _vertices;
private List<Vector3> _normals;
private bool _skipLastRibbonPoint;
public bool skipLastRibbonPoint { get { return _skipLastRibbonPoint; } set { if (value == _skipLastRibbonPoint) return; _skipLastRibbonPoint = value; UpdateGeometry(); } }
private void Awake() {
MeshFilter filter = gameObject.GetComponent<MeshFilter>();
_mesh = filter.mesh;
_vertices = new List<Vector3>();
_normals = new List<Vector3>();
// In addition to clearing the ribbon, this adds a ribbon point that we'll move each frame to match the
// brush tip position so that the brush stroke appears to paint continuously.
ClearRibbon();
}
// Insert a new ribbon point just before the final ribbon point
public void InsertRibbonPoint(Vector3 position, Quaternion rotation) {
// Calculate vertices + normal for ribbon point
Vector3 p1;
Vector3 p2;
Vector3 normal;
CalculateVerticesAndNormalForRibbonPoint(position, rotation, _brushStrokeWidth, out p1, out p2, out normal);
// Insert into vertices array
_vertices.Insert(_vertices.Count-4, p1);
_vertices.Insert(_vertices.Count-4, p2);
_vertices.Insert(_vertices.Count-4, p1);
_vertices.Insert(_vertices.Count-4, p2);
// Insert into normals array
_normals.Insert(_normals.Count-4, normal);
_normals.Insert(_normals.Count-4, normal);
_normals.Insert(_normals.Count-4, -normal);
_normals.Insert(_normals.Count-4, -normal);
// Update the mesh
UpdateGeometry();
}
public void UpdateLastRibbonPoint(Vector3 position, Quaternion rotation) {
// Calculate vertices + normal for ribbon point
Vector3 p1;
Vector3 p2;
Vector3 normal;
CalculateVerticesAndNormalForRibbonPoint(position, rotation, _brushStrokeWidth, out p1, out p2, out normal);
int lastIndex = _vertices.Count-4;
// Update vertices
_vertices[lastIndex] = p1;
_vertices[lastIndex+1] = p2;
_vertices[lastIndex+2] = p1;
_vertices[lastIndex+3] = p2;
// Update normals
_normals[lastIndex] = normal;
_normals[lastIndex+1] = normal;
_normals[lastIndex+2] = -normal;
_normals[lastIndex+3] = -normal;
// Update the mesh
UpdateGeometry();
}
public void ClearRibbon() {
// Clear vertices & normals
_vertices.Clear();
_normals.Clear();
// Create last ribbon point
_vertices.Add(Vector3.zero);
_vertices.Add(Vector3.zero);
_vertices.Add(Vector3.zero);
_vertices.Add(Vector3.zero);
_normals.Add(Vector3.zero);
_normals.Add(Vector3.zero);
_normals.Add(Vector3.zero);
_normals.Add(Vector3.zero);
// Update the mesh
UpdateGeometry();
}
private void CalculateVerticesAndNormalForRibbonPoint(Vector3 position, Quaternion rotation, float width, out Vector3 p1, out Vector3 p2, out Vector3 normal) {
p1 = position + rotation * new Vector3(-width/2.0f, 0.0f, 0.0f);
p2 = position + rotation * new Vector3( width/2.0f, 0.0f, 0.0f);
normal = rotation * Vector3.up;
}
private void UpdateGeometry() {
int numberOfVertices = _vertices.Count;
if (skipLastRibbonPoint)
numberOfVertices -= 4;
if (numberOfVertices < 8) {
_mesh.vertices = new Vector3[0];
_mesh.normals = new Vector3[0];
_mesh.triangles = new int[0];
_mesh.RecalculateBounds();
return;
}
// Would probably make sense to just generate the new triangles rather than regenerating all of them all of the time.
int numberOfSegments = numberOfVertices/4 - 1;
int numberOfTriangles = numberOfSegments * 4; // Two on the front side, two on the back.
int[] triangles = new int[numberOfTriangles*3];
for (int i = 0; i < numberOfSegments; i++) {
// Front
int p1 = i*4;
int p2 = i*4+1;
int p3 = i*4+4;
int p4 = i*4+5;
// Back
int p1b = i*4+2;
int p2b = i*4+3;
int p3b = i*4+6;
int p4b = i*4+7;
// Front
triangles[i*12] = p1;
triangles[i*12+1] = p2;
triangles[i*12+2] = p3;
triangles[i*12+3] = p2;
triangles[i*12+4] = p4;
triangles[i*12+5] = p3;
// Back
triangles[i*12+6] = p1b;
triangles[i*12+7] = p3b;
triangles[i*12+8] = p2b;
triangles[i*12+9] = p2b;
triangles[i*12+10] = p3b;
triangles[i*12+11] = p4b;
}
_mesh.vertices = _vertices.ToArray();
_mesh.normals = _normals.ToArray();
_mesh.triangles = triangles;
_mesh.RecalculateBounds();
}
}
Step 2. Create an empty object called BrushStroke and add the script "BrushStroke" with the following code as a component to this object. Then add Mesh as a child to this object.
using System.Collections.Generic;
using UnityEngine;
public class BrushStroke : MonoBehaviour {
[SerializeField]
private BrushStrokeMesh _mesh = null;
// Ribbon State
struct RibbonPoint {
public Vector3 position;
public Quaternion rotation;
}
private List<RibbonPoint> _ribbonPoints = new List<RibbonPoint>();
private Vector3 _brushTipPosition;
private Quaternion _brushTipRotation;
private bool _brushStrokeFinalized;
// Smoothing
private Vector3 _ribbonEndPosition;
private Quaternion _ribbonEndRotation = Quaternion.identity;
// Mesh
private Vector3 _previousRibbonPointPosition;
private Quaternion _previousRibbonPointRotation = Quaternion.identity;
// Unity Events
private void Update() {
// Animate the end of the ribbon towards the brush tip
AnimateLastRibbonPointTowardsBrushTipPosition();
// Add a ribbon segment if the end of the ribbon has moved far enough
AddRibbonPointIfNeeded();
}
// Interface
public void BeginBrushStrokeWithBrushTipPoint(Vector3 position, Quaternion rotation) {
// Update the model
_brushTipPosition = position;
_brushTipRotation = rotation;
// Update last ribbon point to match brush tip position & rotation
_ribbonEndPosition = position;
_ribbonEndRotation = rotation;
_mesh.UpdateLastRibbonPoint(_ribbonEndPosition, _ribbonEndRotation);
}
public void MoveBrushTipToPoint(Vector3 position, Quaternion rotation) {
_brushTipPosition = position;
_brushTipRotation = rotation;
}
public void EndBrushStrokeWithBrushTipPoint(Vector3 position, Quaternion rotation) {
// Add a final ribbon point and mark the stroke as finalized
AddRibbonPoint(position, rotation);
_brushStrokeFinalized = true;
}
// Ribbon drawing
private void AddRibbonPointIfNeeded() {
// If the brush stroke is finalized, stop trying to add points to it.
if (_brushStrokeFinalized)
return;
if (Vector3.Distance(_ribbonEndPosition, _previousRibbonPointPosition) >= 0.01f ||
Quaternion.Angle(_ribbonEndRotation, _previousRibbonPointRotation) >= 10.0f) {
// Add ribbon point model to ribbon points array. This will fire the RibbonPointAdded event to update the mesh.
AddRibbonPoint(_ribbonEndPosition, _ribbonEndRotation);
// Store the ribbon point position & rotation for the next time we do this calculation
_previousRibbonPointPosition = _ribbonEndPosition;
_previousRibbonPointRotation = _ribbonEndRotation;
}
}
private void AddRibbonPoint(Vector3 position, Quaternion rotation) {
// Create the ribbon point
RibbonPoint ribbonPoint = new RibbonPoint();
ribbonPoint.position = position;
ribbonPoint.rotation = rotation;
_ribbonPoints.Add(ribbonPoint);
// Update the mesh
_mesh.InsertRibbonPoint(position, rotation);
}
// Brush tip + smoothing
private void AnimateLastRibbonPointTowardsBrushTipPosition() {
// If the brush stroke is finalized, skip the brush tip mesh, and stop animating the brush tip.
if (_brushStrokeFinalized) {
_mesh.skipLastRibbonPoint = true;
return;
}
Vector3 brushTipPosition = _brushTipPosition;
Quaternion brushTipRotation = _brushTipRotation;
// If the end of the ribbon has reached the brush tip position, we can bail early.
if (Vector3.Distance(_ribbonEndPosition, brushTipPosition) <= 0.0001f &&
Quaternion.Angle(_ribbonEndRotation, brushTipRotation) <= 0.01f) {
return;
}
// Move the end of the ribbon towards the brush tip position
_ribbonEndPosition = Vector3.Lerp(_ribbonEndPosition, brushTipPosition, 25.0f * Time.deltaTime);
_ribbonEndRotation = Quaternion.Slerp(_ribbonEndRotation, brushTipRotation, 25.0f * Time.deltaTime);
// Update the end of the ribbon mesh
_mesh.UpdateLastRibbonPoint(_ribbonEndPosition, _ribbonEndRotation);
}
}
Step 3. Drag the object BrushStroke into assets to set it to a prefab. Delete the object from scene.
Step 4. Create an empty object called Brush and add the following script to its components.
using System.Collections.Generic;
using UnityEngine;
using UnityEngine.XR;
using UnityEngine.UI;
public class Brush : MonoBehaviour
{
// Prefab to instantiate when we draw a new brush stroke
[SerializeField] private GameObject _brushStrokePrefab = null;
// Which hand should this brush instance track?
private enum Hand { LeftHand, RightHand };
[SerializeField] private Hand _hand = Hand.RightHand;
// Used to keep track of the current brush tip position and the actively drawing brush stroke
private Vector3 _handPosition;
private Quaternion _handRotation;
private BrushStroke _activeBrushStroke;
public InputDeviceCharacteristics controllerCharacteristics;
public GameObject test;
private InputDevice targetDevice;
void Start()
{
controllerCharacteristics = InputDeviceCharacteristics.Right | InputDeviceCharacteristics.Controller;
var devices = new List<InputDevice>();
InputDevices.GetDevicesWithCharacteristics(controllerCharacteristics, devices);
if (devices.Count > 0){
targetDevice = devices[0];
}
}
private void Update()
{
// Start by figuring out which hand we're tracking
XRNode node = _hand == Hand.LeftHand ? XRNode.LeftHand : XRNode.RightHand;
string trigger = _hand == Hand.LeftHand ? "Left Trigger" : "Right Trigger";
// Get the position & rotation of the hand
bool handIsTracking = UpdatePose(node, ref _handPosition, ref _handRotation);
// Figure out if the trigger is pressed or not
bool triggerPressed = targetDevice.TryGetFeatureValue(CommonUsages.trigger, out float triggerValue) && triggerValue > 0.1f;
// If we lose tracking, stop drawing
if (!handIsTracking)
triggerPressed = false;
// If the trigger is pressed and we haven't created a new brush stroke to draw, create one!
if (triggerPressed && _activeBrushStroke == null)
{
// Instantiate a copy of the Brush Stroke prefab.
GameObject brushStrokeGameObject = Instantiate(_brushStrokePrefab);
// Grab the BrushStroke component from it
_activeBrushStroke = brushStrokeGameObject.GetComponent<BrushStroke>();
// Tell the BrushStroke to begin drawing at the current brush position
_activeBrushStroke.BeginBrushStrokeWithBrushTipPoint(_handPosition, _handRotation);
}
// If the trigger is pressed, and we have a brush stroke, move the brush stroke to the new brush tip position
if (triggerPressed)
_activeBrushStroke.MoveBrushTipToPoint(_handPosition, _handRotation);
// If the trigger is no longer pressed, and we still have an active brush stroke, mark it as finished and clear it.
if (!triggerPressed && _activeBrushStroke != null)
{
_activeBrushStroke.EndBrushStrokeWithBrushTipPoint(_handPosition, _handRotation);
_activeBrushStroke = null;
}
}
//// Utility
// Given an XRNode, get the current position & rotation. If it's not tracking, don't modify the position & rotation.
private static bool UpdatePose(XRNode node, ref Vector3 position, ref Quaternion rotation)
{
List<XRNodeState> nodeStates = new List<XRNodeState>();
InputTracking.GetNodeStates(nodeStates);
foreach (XRNodeState nodeState in nodeStates)
{
if (nodeState.nodeType == XRNode.RightHand)
{
Vector3 nodePosition;
Quaternion nodeRotation;
bool gotPosition = nodeState.TryGetPosition(out nodePosition);
bool gotRotation = nodeState.TryGetRotation(out nodeRotation);
if (gotPosition)
position = nodePosition;
if (gotRotation)
rotation = nodeRotation;
return gotPosition;
}
}
return false;
}
}
Step 5. Test on your VR! When you press your right trigger, a brush stroke with the color of your choice should now appear. If it doesn't, make sure that you are on Unity version 2019 and that you have a device-based XR Origin