Attach the 2nd (normal view) and 3rd (segmentation view) cameras to the game scene
Create Renderer Target x2 to receive images from the cameras
Create UI/Raw Image to show the images from the cameras
The segmentation view camera: attach the below script
using UnityEngine;
[ExecuteInEditMode]
public class ReplacementShaderTest : MonoBehaviour
{
private Camera _camera;
private Shader _shader;
void OnEnable()
{
_camera = GetComponent<Camera>();
_shader = Shader.Find("ReplaceObjectColorInCameraView");
_camera?.SetReplacementShader(_shader, "Annotation");
}
private void OnDisable()
{
_camera?.ResetReplacementShader();
}
}
We will have the below shader code in Assets folder. It is called by ReplaceObjectColorInCameraView.cs. It will check the Annotation tag set in another script and change the gameobject material color.
At any gameobject, attach the below code to set an annotation tag. A gameobject may contain many materials or has a material attached in its child object.
using System.Collections.Generic;
using System.Linq;
using UnityEngine;
[ExecuteInEditMode]
public class SetShaderTagsAnnotation : MonoBehaviour
{
public enum AnnotationType
{
green, red
}
public AnnotationType annotation_name;
void Start()
{
List<Material> list_material = obj.GetComponent<Renderer>().materials.ToList();
foreach (Material mat in list_material)
{
mat.SetOverrideTag("Annotation", annotation_name.ToString());
}
ApplyAnnotation(gameObject.transform);
}
void ApplyAnnotation(Transform obj)
{
Renderer myRenderer = obj.GetComponent<Renderer>();
if (myRenderer != null)
{
myRenderer.material.SetOverrideTag("Annotation", annotation_name.ToString());
//Debug.Log(obj.name + " is set to " + annotation_name.ToString());
}
else
{
//Debug.Log(obj.name + " has no Renderer component");
TraverseHierarchy(obj.transform);
}
}
void TraverseHierarchy(Transform parent)
{
//Debug.Log("TraverseHierarchy of " + parent.name);
foreach (Transform child in parent)
{
ApplyAnnotation(child);
}
}
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
public class PathReader : MonoBehaviour
{
public GameObject MarkStart, MarkGoal, MarkFP;
public float smooth_distance;
public TextAsset filename;
//private List<string> file_data_line;
private List<Vector3> positions, rotations;
private LineRenderer lineRenderer;
void Awake()
{
lineRenderer = GetComponent<LineRenderer>();
//file_data = System.IO.File.ReadAllLines(filename_path);
//Debug.Log(filename);
}
void Start()
{
List<string> file_data_each_line = TextAssetToListString(filename);
ListStringToListVector(file_data_each_line, out positions, out rotations, smooth_distance);
//MapToGroundPosition(); //cannot change the value of List<Vector3> after assigned.
//Assign positinos to line renderer
lineRenderer.positionCount = positions.Count;
lineRenderer.SetPositions(positions.ToArray());
//Assign Mark object position
MarkStart.transform.position = positions[0];
MarkGoal.transform.position = positions[positions.Count-1];
MarkFP.transform.position = positions[1];
Debug.Log("Positions count = " + positions.Count);
}
private List<string> TextAssetToListString(TextAsset ta)
{
return new List<string>(ta.text.Split('\n'));
}
private void ListStringToListVector(List<string> list_text, out List<Vector3> list_pos, out List<Vector3> list_rot, float smd)
{
list_pos = new List<Vector3>();
list_rot = new List<Vector3>();
// 1st line is label
// 2nd line add to thevector
string[] columns = list_text[1].Split(',');
Vector3 load_position = new Vector3(float.Parse(columns[1]), 0f, float.Parse(columns[3])); //set y to ground
Vector3 load_rotation = new Vector3(float.Parse(columns[4]), float.Parse(columns[5]), float.Parse(columns[6]));
positions.Add(load_position);
rotations.Add(load_rotation);
for (int i = 2; i < list_text.Count; i++) // from 3nd line, check distance before add to the vector
{
//Debug.Log(line);
columns = list_text[i].Split(',');
if (columns.Length != 7) // not enough information to extract (end of line)
return;
//Debug.Log(i+" "+columns.Length);
//Vector3 load_position = new Vector3(float.Parse(columns[1]), float.Parse(columns[2]), float.Parse(columns[3]));
load_position = new Vector3(float.Parse(columns[1]), 0f, float.Parse(columns[3])); //set y to ground
load_rotation = new Vector3(float.Parse(columns[4]), float.Parse(columns[5]), float.Parse(columns[6]));
if(Vector3.Distance(positions[positions.Count-1], load_position) > smd)
{
positions.Add(load_position);
rotations.Add(load_rotation);
}
}
}
}
# Get activations of a few sample layers
activations = model.run_graph([image], [
("input_image_meta", tf.identity(model.keras_model.get_layer("input_image_meta").output)),
("rpn_bbox", model.keras_model.get_layer("rpn_bbox").output),
("fpn_p6", model.keras_model.get_layer("fpn_p6").output),
])
Each airtap do: create object, add rigidbody (gravity) to the object, and finally delete the object.
Somehow, the GazeManager.Instance.HitObject hit the child obj that contain a mesh/box collider. Therefore, we need to check tag name on its parent. Note that, Mesh Collider needs to check on Convex, otherwise when add a rigidbody, it will fall down eternity without colliding with the room floor.
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using HoloToolkit.Unity.InputModule;
public class SpawnDropObject : MonoBehaviour, IInputClickHandler
{
//[SerializeField] private PanelDebug panelDebug;
public GameObject iprefab;
private int ObjCount;
private List<GameObject> ObjList;
void Start()
{
InputManager.Instance.PushFallbackInputHandler(gameObject);
ObjList = new List<GameObject>();
}
public void OnInputClicked(InputClickedEventData eventData)
{
if (!GazeManager.Instance.HitObject) //airtap at nothing = create new obj
{
Debug.Log("!GazeManager.Instance.HitObject");
Vector3 obj_position = Camera.main.transform.position + Camera.main.transform.forward;
CreateNewObject(obj_position);
}
else
{
//panelDebug.ShowMessage(GazeManager.Instance.HitObject.name);
Debug.Log("\n"+GazeManager.Instance.HitObject.name + " " + GazeManager.Instance.HitObject.tag + " " + GazeManager.Instance.HitObject.transform.parent.tag);
// Airtap at floating object. Then, add gravity to the obj = drop it down
if (GazeManager.Instance.HitObject.tag == "Floating")
{
Debug.Log("HitObject.tag == Floating");
GazeManager.Instance.HitObject.AddComponent<Rigidbody>();
GazeManager.Instance.HitObject.tag = "Falldown";
}
else if (GazeManager.Instance.HitObject.transform.parent.tag == "Floating")
{
Debug.Log("HitObject.parent.tag == Floating");
GazeManager.Instance.HitObject.AddComponent<Rigidbody>();
GazeManager.Instance.HitObject.transform.parent.tag = "Falldown";
}
// Airtap at object on floor. Then, remove it.
else if (GazeManager.Instance.HitObject.tag == "Falldown")
{
Debug.Log("HitObject.tag == Falldown");
ObjList.Remove(GazeManager.Instance.HitObject);
Destroy(GazeManager.Instance.HitObject);
}
else if (GazeManager.Instance.HitObject.transform.parent.tag == "Falldown")
{
Debug.Log("HitObject.parent.tag == Falldown");
ObjList.Remove(GazeManager.Instance.HitObject.transform.parent.gameObject);
Destroy(GazeManager.Instance.HitObject.transform.parent.gameObject);
}
// Airtap at something (room mesh). Then, create new obj.
else
{
Debug.Log("HitObject.tag == ??");
Debug.Log("HitObject" + GazeManager.Instance.HitObject.transform.position.ToString());
Debug.Log("HitPosition" + GazeManager.Instance.HitObject.ToString());
//CreateNewObject(GazeManager.Instance.HitObject.transform.position); // this position is not the world coordinate
CreateNewObject(GazeManager.Instance.HitPosition);
}
}
string objlistname = "\nObj in List";
foreach (GameObject obj in ObjList)
{
string text = "\n"+obj.name + " " + obj.tag;
objlistname += text;
}
Debug.Log(objlistname);
}
private void CreateNewObject(Vector3 position)
{
Debug.Log("CreateNewObject at"+ position.ToString());
GameObject newobj = Instantiate(iprefab, position, Quaternion.identity);
newobj.tag = "Floating";
ObjList.Add(newobj);
}
}
I wanted something to show a program state message while I’m wearing Hololens. That is to add UI > Panel and UI > Text inside the Panel. You will get The Canvas gameobject as a parent object that contain Panel nested inside. Set the Canvas render mode to “World Space”
Then we need to attach Tagalong and Billboard scripts (from HoloToolkit 2017.4.3.0) to the Canvas.
Tagalong will move the Canvas to always be inside the Hololens view. The default setting it will move the Canvas to the border of the Hololens view, not exactly at the view center.
Billboard will change the Canvas orientation to face the Hololens Camera. Set Pivot axis to “free” so that the Canvas can rotate along all x,y,z axes.
While recording a video, the tagalong function did not work properly. It let the Canvas to not come inside Hololens view when moved. Normally, the Canvas will be around inner border of the view.
using System;
using HoloToolkit.Unity.InputModule;
public class XboxControllerInputHandle : XboxControllerHandlerBase
{
public event Action On_Y_ButtonPressed = delegate { };
public event Action On_B_ButtonPressed = delegate { };
public event Action On_X_ButtonPressed = delegate { };
public override void OnXboxInputUpdate(XboxControllerEventData eventData)
{
base.OnXboxInputUpdate(eventData);
//XboxA_Down overlap with the default select
if (eventData.XboxY_Down)
{
On_Y_ButtonPressed();
}
if (eventData.XboxB_Down)
{
On_B_ButtonPressed();
}
if (eventData.XboxX_Down)
{
On_X_ButtonPressed();
}
}
}
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
public class DrawLineByPoints : MonoBehaviour
{
private LineRenderer lr;
private List<Vector3> points;
public bool visible { get; private set; }
private void Awake()
{
visible = true;
lr = GetComponent<LineRenderer>();
points = new List<Vector3>();
}
public void Hide()
{
visible = false;
gameObject.SetActive(false);
}
public void Show()
{
visible = true;
gameObject.SetActive(true);
}
public void SetUpLine(List<Vector3> points)
{
lr.positionCount = points.Count;
this.points = points;
}
private void Update()
{
for(int i=0;i<points.Count;i++)
{
lr.SetPosition(i,points[i]);
}
}
}
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using HoloToolkit.Unity.InputModule;
public class PointAdding : MonoBehaviour
{
private List<Vector3> positions;
[SerializeField] private DrawLineByPoints line;
[SerializeField] private XboxControllerInputHandle XBoxInputHandleScript;
void Start()
{
positions = new List<Vector3>();
InputManager.Instance.PushFallbackInputHandler(gameObject);
XBoxInputHandleScript.On_Y_ButtonPressed += Handle_Y_ButtonPressed;
XBoxInputHandleScript.On_B_ButtonPressed += Handle_B_ButtonPressed;
XBoxInputHandleScript.On_X_ButtonPressed += Handle_X_ButtonPressed;
}
private void Handle_Y_ButtonPressed()
{
if(line.visible) line.Hide();
else line.Show();
}
private void Handle_B_ButtonPressed()
{
Vector3 hitPoint = GazeManager.Instance.HitPosition;
positions.Add(hitPoint);
Debug.Log("Click on " + hitPoint.ToString());
line.SetUpLine(positions);
}
private void Handle_X_ButtonPressed()
{
positions.Clear();
line.SetUpLine(positions);
}
}
Press B : add new point for drawing a line Press X : remove all points Press Y : Show/Hide the line Note that the A button is a select action by default.
2. Attach DebugLogBroadcaster.cs from the above blog to a gameobject, change Broadcast Port to 9999. (Any port number is fine. It only need to match the SocketTest program)
3. In Unity, Publishing Setting : enable the networking capabilities ( InternetClient, InternetClientServer and PrivateNetworkClientServer)
I’m gonna write a program to detect the airtap event to generate a new object in a room. To use Hololens interface -> import HoloToolkit 2017.4.3.0
using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using HoloToolkit.Unity.InputModule;
public class SpawnObject : MonoBehaviour, IInputClickHandler
{
public GameObject iprefab;
void Start ()
{
InputManager.Instance.PushFallbackInputHandler(gameObject);
}
public void OnInputClicked(InputClickedEventData eventData)
{
Debug.Log("\nOnInputClicked");
GameObject newobj = Instantiate(iprefab, new Vector3(0, 0, 0), Quaternion.Euler(-90, 0, 0));
newobj.transform.position = Camera.main.transform.position + Camera.main.transform.forward*10;
Debug.Log("Camera at " + Camera.main.transform.position);
Debug.Log("Spawn Object at " + newobj.transform.position);
}
}
Here is the result: We see a cursor which indicate the Hololens lookat position, when we do airtap, the programe will create a new star at the cursor position. Also the Debug.Log() can be observed by the SocketTest program.
