Category: hololens

Each airtap do: create object, add rigidbody (gravity) to the object, and finally delete the object.

Somehow, the GazeManager.Instance.HitObject hit the child obj that contain a mesh/box collider. Therefore, we need to check tag name on its parent.
Note that, Mesh Collider needs to check on Convex, otherwise when add a rigidbody, it will fall down eternity without colliding with the room floor.

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using HoloToolkit.Unity.InputModule;

public class SpawnDropObject : MonoBehaviour, IInputClickHandler
{
    //[SerializeField] private PanelDebug panelDebug;
    public GameObject iprefab;
    private int ObjCount;
    private List<GameObject> ObjList;
    void Start()
    {
        InputManager.Instance.PushFallbackInputHandler(gameObject);
        ObjList = new List<GameObject>();
    }

    public void OnInputClicked(InputClickedEventData eventData)
    {
        if (!GazeManager.Instance.HitObject) //airtap at nothing = create new obj
        {
            Debug.Log("!GazeManager.Instance.HitObject");
            Vector3 obj_position = Camera.main.transform.position + Camera.main.transform.forward;
            CreateNewObject(obj_position);
        }
        else
        {
            //panelDebug.ShowMessage(GazeManager.Instance.HitObject.name);
            Debug.Log("\n"+GazeManager.Instance.HitObject.name + " " + GazeManager.Instance.HitObject.tag + " " + GazeManager.Instance.HitObject.transform.parent.tag);

            // Airtap at floating object. Then, add gravity to the obj = drop it down
            if (GazeManager.Instance.HitObject.tag == "Floating") 
            {
                Debug.Log("HitObject.tag == Floating");
                GazeManager.Instance.HitObject.AddComponent<Rigidbody>();  
                GazeManager.Instance.HitObject.tag = "Falldown";
            }
            else if (GazeManager.Instance.HitObject.transform.parent.tag == "Floating")
            {
                Debug.Log("HitObject.parent.tag == Floating");
                GazeManager.Instance.HitObject.AddComponent<Rigidbody>(); 
                GazeManager.Instance.HitObject.transform.parent.tag = "Falldown";
            }

            // Airtap at object on floor. Then, remove it.
            else if (GazeManager.Instance.HitObject.tag == "Falldown") 
            {
                Debug.Log("HitObject.tag == Falldown");
                ObjList.Remove(GazeManager.Instance.HitObject);
                Destroy(GazeManager.Instance.HitObject);
            }
            else if (GazeManager.Instance.HitObject.transform.parent.tag == "Falldown") 
            {
                Debug.Log("HitObject.parent.tag == Falldown");
                ObjList.Remove(GazeManager.Instance.HitObject.transform.parent.gameObject);
                Destroy(GazeManager.Instance.HitObject.transform.parent.gameObject);
            }

            // Airtap at something (room mesh). Then, create new obj.
            else 
            {
                Debug.Log("HitObject.tag == ??");
                Debug.Log("HitObject" + GazeManager.Instance.HitObject.transform.position.ToString());
                Debug.Log("HitPosition" + GazeManager.Instance.HitObject.ToString());
                //CreateNewObject(GazeManager.Instance.HitObject.transform.position); // this position is not the world coordinate
                CreateNewObject(GazeManager.Instance.HitPosition);
            }
        }

        
        string objlistname = "\nObj in List";
        foreach (GameObject obj in ObjList)
        {
            string text = "\n"+obj.name + " " + obj.tag;
            objlistname += text;         
        }
        Debug.Log(objlistname);
    }
    private void CreateNewObject(Vector3 position)
    {
        Debug.Log("CreateNewObject at"+ position.ToString());
        GameObject newobj = Instantiate(iprefab, position, Quaternion.identity);
        newobj.tag = "Floating";

        ObjList.Add(newobj);
    }
}

This application will record Hololens transformation overtime and save to any text file (my code is csv file) inside the device.

using UnityEngine;
using System.Collections;
using System.Collections.Generic;
using System;
using System.IO;
using System.Linq;

#if WINDOWS_UWP
using Windows.Storage;
using Windows.System;
using System.Threading.Tasks;
using Windows.Storage.Streams;
#endif

// saved folder : User Folders \ LocalAppData \ Appname \ LocalState \
public class WriteCSVFile : MonoBehaviour
{
#if WINDOWS_UWP
    Windows.Storage.ApplicationDataContainer localSettings = Windows.Storage.ApplicationData.Current.LocalSettings;
    Windows.Storage.StorageFolder localFolder = Windows.Storage.ApplicationData.Current.LocalFolder;
#endif

    private string timeStamp;
    private string fileName;
    private string saveInformation;
 
    public void WriteTextToFile(string text)
    {
        timeStamp = System.DateTime.Now.ToString().Replace("/", "_").Replace(":", "-").Replace(" ", "_");
        fileName = "transform-" + timeStamp + ".csv";
        saveInformation = text;
        
#if WINDOWS_UWP
        WriteData();
#endif
    }

#if WINDOWS_UWP
    async void WriteData()
    {
        StorageFile saveFile = await localFolder.CreateFileAsync(fileName, CreationCollisionOption.ReplaceExisting);
        string fileheader = "id,pos_x,pos_y,pos_z,rot_x,rot_y,rot_z" + "\r\n";
        await FileIO.AppendTextAsync(saveFile, fileheader + saveInformation);
    }
#endif

}

In another script file, we record transform data and call WriteTextToFile() to write a text.

[SerializeField] private WriteCSVFile writeCSVFile;
private StringBuilder csv;

private void PreparePositiontosave()
{
    csv.Remove(0, csv.Length);
    for (int i = 0; i < positions.Count; i++)
    {
        var newLine = string.Format("{0},{1},{2},{3},{4},{5},{6}", i,
                        positions[i].x, positions[i].y, positions[i].z,
                        rotations[i].x, rotations[i].y, rotations[i].z);
        csv.AppendLine(newLine);
        writeCSVFile.WriteTextToFile(csv.ToString());
    }
} 

Here is where the file was saved and its content.

I wanted something to show a program state message while I’m wearing Hololens. That is to add UI > Panel and UI > Text inside the Panel. You will get The Canvas gameobject as a parent object that contain Panel nested inside. Set the Canvas render mode to “World Space”

Then we need to attach Tagalong and Billboard scripts (from HoloToolkit 2017.4.3.0) to the Canvas.

Tagalong will move the Canvas to always be inside the Hololens view. The default setting it will move the Canvas to the border of the Hololens view, not exactly at the view center.

Billboard will change the Canvas orientation to face the Hololens Camera. Set Pivot axis to “free” so that the Canvas can rotate along all x,y,z axes.

Import HoloToolkit 2017.4.3.0
Menu : Mixed Reality Toolkit > Configure > Apply Mixed Reality Project Setting > Check “Use Toolkit-specific InputManager axes”

using System;
using HoloToolkit.Unity.InputModule;


public class XboxControllerInputHandle : XboxControllerHandlerBase
{
    public event Action On_Y_ButtonPressed = delegate { };
    public event Action On_B_ButtonPressed = delegate { };
    public event Action On_X_ButtonPressed = delegate { };

    public override void OnXboxInputUpdate(XboxControllerEventData eventData)
    {
        base.OnXboxInputUpdate(eventData);

        //XboxA_Down overlap with the default select
        if (eventData.XboxY_Down) 
        {
            On_Y_ButtonPressed();
        }
        if (eventData.XboxB_Down)
        {
            On_B_ButtonPressed();
        }
        if (eventData.XboxX_Down)
        {
            On_X_ButtonPressed();
        }
    }
}

using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class DrawLineByPoints : MonoBehaviour
{
    private LineRenderer lr;
    private List<Vector3> points;
    public bool visible { get; private set; }

    private void Awake()
    {
        visible = true;
        lr = GetComponent<LineRenderer>();
        points = new List<Vector3>();
    }

    public void Hide()
    {
        visible = false;
        gameObject.SetActive(false);
    }
    public void Show()
    {
        visible = true;
        gameObject.SetActive(true);
    }

    public void SetUpLine(List<Vector3> points)
    {
        lr.positionCount = points.Count;
        this.points = points;
    }

    private void Update()
    {
        for(int i=0;i<points.Count;i++)
        {
            lr.SetPosition(i,points[i]);
        }
    }
}

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using HoloToolkit.Unity.InputModule;

public class PointAdding : MonoBehaviour
{
    private List<Vector3> positions;
    [SerializeField] private DrawLineByPoints line;
    [SerializeField] private XboxControllerInputHandle XBoxInputHandleScript;

    void Start()
    {
        positions = new List<Vector3>();
        InputManager.Instance.PushFallbackInputHandler(gameObject);

        XBoxInputHandleScript.On_Y_ButtonPressed += Handle_Y_ButtonPressed;
        XBoxInputHandleScript.On_B_ButtonPressed += Handle_B_ButtonPressed;
        XBoxInputHandleScript.On_X_ButtonPressed += Handle_X_ButtonPressed;
    }

    private void Handle_Y_ButtonPressed()
    {
        if(line.visible) line.Hide();
        else line.Show();
    }
    private void Handle_B_ButtonPressed()
    {
        Vector3 hitPoint = GazeManager.Instance.HitPosition;
        positions.Add(hitPoint);
        Debug.Log("Click on " + hitPoint.ToString());
        line.SetUpLine(positions);
    }

    private void Handle_X_ButtonPressed()
    {
        positions.Clear();
        line.SetUpLine(positions);
    }
}

Press B : add new point for drawing a line
Press X : remove all points
Press Y : Show/Hide the line
Note that the A button is a select action by default.

Import HoloToolkit 2017.4.3.0
Hierarchy : Add HoloLensCamera, InputManager, DefaultCursor, SpatialMapping
Publishing Setting, Capabilities : Check SpatialPerception

Then we can access environment mesh that the Hololens perceive. (we can select to show or hide mesh in SpatialMapping prefab)

This program will draw a line as we click on room positions.

using System.Collections;
using System.Collections.Generic;
using UnityEngine;

public class DrawLineByPoints : MonoBehaviour
{
    private LineRenderer lr;
    private List<Vector3> points;

    private void Awake()
    {
        lr = GetComponent<LineRenderer>();
        points = new List<Vector3>();
    }

    public void SetUpLine(List<Vector3> points)
    {
        lr.positionCount = points.Count;
        this.points = points;
    }

    private void Update()
    {
        for(int i=0;i<points.Count;i++)
        {
            lr.SetPosition(i,points[i]);
            //Debug.Log(points[i].position);
        }
    }
}

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using HoloToolkit.Unity.InputModule;

public class PointAdding : MonoBehaviour, IInputClickHandler
{
    [SerializeField] private DrawLineByPoints line;
    private List<Vector3> positions;

    void Start()
    {
        positions = new List<Vector3>();
        InputManager.Instance.PushFallbackInputHandler(gameObject);
    }

    public void OnInputClicked(InputClickedEventData eventData)
    {
        Debug.Log("\nOnInputClicked");
        Vector3 hitPoint = GazeManager.Instance.HitPosition;

        Debug.Log("Click on " + hitPoint.ToString());
        positions.Add(hitPoint);

        line.SetUpLine(positions); 
    }
}

I refer from http://talesfromtherift.com/hololens-contest-week-9/

1. Download SocketTest from http://sourceforge.net/projects/sockettest/ ,
   extract file and open the program. Goto Udp tab, IP = 0.0.0.0, Port = 9999, Press “Start Listening”

2. Attach DebugLogBroadcaster.cs from the above blog to a gameobject, change Broadcast Port to 9999. (Any port number is fine. It only need to match the SocketTest program)

3. In Unity, Publishing Setting :  enable the networking capabilities ( InternetClient, InternetClientServer and PrivateNetworkClientServer)

I’m gonna write a program to detect the airtap event to generate a new object in a room. To use Hololens interface -> import HoloToolkit 2017.4.3.0

Hierarchy : Add HoloLensCamera, InputManager, DefaultCursor

The Yellow star prefab is from https://assetstore.unity.com/packages/3d/props/simple-gems-ultimate-animated-customizable-pack-73764

using System.Collections;
using System.Collections.Generic;
using UnityEngine;
using HoloToolkit.Unity.InputModule;

public class SpawnObject : MonoBehaviour, IInputClickHandler
{

    public GameObject iprefab;

    void Start ()
    {
        InputManager.Instance.PushFallbackInputHandler(gameObject);
    }

    public void OnInputClicked(InputClickedEventData eventData)
    {
        Debug.Log("\nOnInputClicked");

        GameObject newobj = Instantiate(iprefab, new Vector3(0, 0, 0), Quaternion.Euler(-90, 0, 0));
        newobj.transform.position = Camera.main.transform.position + Camera.main.transform.forward*10;

        Debug.Log("Camera at " + Camera.main.transform.position);
        Debug.Log("Spawn Object at " + newobj.transform.position);

    }
}

Here is the result: We see a cursor which indicate the Hololens lookat position, when we do airtap, the programe will create a new star at the cursor position. Also the Debug.Log() can be observed by the SocketTest program.

1. USB connection

Type http://127.0.0.1:10080/ on the Firefox web browser

If this error appear.

Type “about:config“, search for “network.security.ports.banned.override“

If it does not exist, create a new one with string as a port number 10080, then we can access http://127.0.0.1:10080/

It will ask username and password of this Hololens. If you don’t know, type anything for 3 times, then it will let you set a new one.

2. Wifi connection

Type the IP shown in the Holographic app on the Firefox web browser.

Note that your pc need to be in the same LAN group IP.
Typing “ipconfig” in command prompt (windows search > “cmd”), to see IP.

A desktop pc with LAN cable connected may not be in the same IP group with wifi connection. Use a usb wifi adapter plug to the PC to access the wifi instead.

I didn’t touch Hololens for almost 2 years…then I forgot how to use it… It took several days to recall my memory. So I need to write this out otherwise I will forget again.

I have a Hololens with the current OS is 10.0.17763.2114 (Check it at Settings > About), that is Windows 10 Version 1809 (RS5)

https://en.wikipedia.org/wiki/Windows_10_version_history

First I need Visual Studio 2017 and Unity 2018.2.1f1, the same version I used around 2018-2019.

While install Visual Studio Community, ensure that Windows 10 SDK 10.0.17763.0, Universal Windows Platform (UWP), .NET, USB device connection? are selected.

While install Unity, select UWP Build Support (Both IL2CPP and .NET), Vuforia AR Support. Note that Unity 2018.2.1f1 is the last version that can build app with .NET

New Unity 3D project:

1. Main Camera > Clear Flags >
– Change from Skybox to Solid Color,
– Background = Black color.
2. You may change Field of View, Clipping Planes Near-Far later.
3. Add some object to the scene.
– Right click in Hierarchy > 3D Object > Cube.
– Right click in Assets > Create > Material > name = red_material, change albedo color to red.
– Drag the red_material to the cube, then the cube color will become red
Here we finish set up a sample scene.

Unity Setting:

1. Windows > Package Manager > remove Text Mesh Pro package, it will cause a failure build in Visual Studio.
2. Edit > Project Settings > Quality > Select Very Low level. (may not necessary)
3. File > Build Settings >
– Platform = Universal Windows Platform, then click Switch Platform
– Player Settings >
— XR Setting > Select Virtual Reality Support

— Publishing Setting > Capabilities > Select: InternetClient, Webcam, Microphone, SpatialPerception (any stuff you will use in your app)

— Other Setting > Scripting Backend = .NET
(note that I use .NET because I used it since the Unity5.6. Building with the default IL2CPP is failed for my project, though it is the new standard now.)

— Icon > Tiles and Logos > Square 150×150 Logo > Scale 200% (300×300 pixel), Add icon image for your app here! (Scale 100% did not work)

Player Settings >
– Follow settings in the below image.
– Note that don’t select the Enable Mixed Reality Remoting, it will build the app as a 2D window.

Build an app:

Click Build, then create a new folder “app”, the source code will generated in this folder. Otherwise, all the files to deploy to Hololens will messed up in your unity project folder.

It is also easy to just delete the “app” folder to rebuild the app from the scratch. Or create another folder “app2” to build another version of your app.

If build completed, window explorer will pop up showing “app” folder.
Note that the projectname.sln file outside the “app” folder is a project file for unity, we use it when doing coding together with scene design in unity.
To deploy an app to Hololens, you need to open “app/projectname.sln”

<- open this one

Note that you need to turn on developer mode on both pc (Setting > Security) and Hololens.

Connect the Hololens to pc with usb cable.
For the first time of deployment, you will be asked for pairing Hololens with the pc. Go to get the PIN from Hololens (Settings > Updates & Security > For developers > Device discovery > Pair)

Deploying App:

1. Release mode x86
2. Select Device (it will deploy through usb cable)
3. Select “Start without debug” ( Debug > Start without debug). Get the icon by clicking at the small triangle on the right > Customize > Command > Add Command .

Here is the app deployed to the Hololens.

HoloToolkitのサンプルSharingTestの中身を見る

Vuforiaを使ってSharingの接続成功率をあげる

お手軽にSharingの接続成功率と精度をあげる

 

 

 

จะทำ ถ่ายวีดีโอ(stream realtimeได้ก็ดี) จาก hololens อีกตัวนึง

 

ที่ไม่ใช้ hololens ตัวที่ userใช้งานอยู่ อัดวีดีโอ first person view

เพราะว่า พอเอากล้องไปอัดวีดีโอแล้ว จะทำ realtime registration ของ vuforia ไม่ได้

พอ user ขยับนิดๆหน่อยๆ hologram มันก็อยู่ไม่ตรงจุดที่จะให้อยู่ละ

 

ถ้าใช้ hololens อีกตัว อัดวีดีโอ third person view ได้ ก็จะดี

แต่ว่าถ้าอัดผ่าน hololens จะอัดวีดีโอได้แค่ 5 นาที (ต้องเขียนโค๊ดเองเพื่อไม่ให้มันจำกัดเวลา…แต่ก็ยังไม่เคยลองอัดเกิน5นาที)

แล้วก็ stream realtime จาก hololens มาที่คอม มัน ช้าๆอยู่นะ

อัดวีดีโอเสร็จไฟล์ก็อยู่ใน hololens ต้องมาลากลงคอมอีก

 

ก็เลยว่าจะใช้ spectator view hololens อัดวีดีโอ/stream hologram

ซึ่ง จาก https://docs.microsoft.com/en-us/windows/mixed-reality/spectator-view

มันมี2วิธี 1.ใช้ iphone/ipad ถ่ายวีดีโอ  2.ใช้กล้องDSLR+hololensอีกตัว ถ่ายวีดีโอ

วิธี Iphone ต้องเป็นรุ่น 6s ขึ้นไป เพราะว่ามันอัด 4K video ได้, ไหนจะต้องใช้เครื่อง mac เขียนโปรแกรม ไปสมัคร apple developer มีแบบฟรีอยู่ ดูยุ่งยาก เลยข้าม

 

วิธีกล้อง DSLR ก็ดูยุ่งยาก พอดีว่า concept ของเค้าคือถ่ายวีดีโอ hidef + hologram ก็เลยต้องใช้กล้อง ที่อัดวีดีโอ 2k, 4k ได้ (จะ gopro ก็ได้)

คราวนี้ พอจะ stream สด ก็ต้องต่อสายวีดีโอเข้าคอม แต่ปกติแล้ว port hdmi ที่คอมเป็น output อย่างเดียว

ก็เลยต้องมี capture card ราคา 25,000yenได้ เพื่อ รับ input hdmi เข้าคอม (แล้วก็ต้องลงโปรแกรมของ capture card อีก)

แล้ววีธีนี้คือ เอากล้องตั้งไว้ เอา hololens วางไว้บนกล้องเพื่อ  calibrate ระบุตำแหน่ง

แต่พอเริ่มอัดวีดีโอแล้ว ห้ามขยับกล้อง เดี่ยว hologram เคลื่อน…

 

คือว่าถ้าใช้กล้องทั่วไปที่มันมีสาย usb ต่อคอมจะสะดวกกว่ารึป่าวนะ..

ก็มีคนลองใช้ กล้อง webcam ธรรมดา เห็นว่าก็ใช้ได้อยู่ แต่ไม่ ok เพราะว่า fov กล้องwebcam มันแคบ เค้าบอกว่าต้องติดเลนส์เพิ่ม…แล้วก็ ลำดับสี rbg มันเรียงสลับกันอะไรสักอย่าง แต่ไม่ได้เป็นปัญหาตอน calibrate

คุ้นๆว่าจะต้องแก้โค๊ดนิดหน่อย ตอนเปิดกล้อง ให้เปิดกล้องก่อนเซ็ต resolutionกล้อง ไม่งั้นมันจะ error

 

แล้วก็มีคนใช้ zed (depth camera) ทำ อันนี้น่าสนใจ เห็นบอกว่า พอ calibrate เสร็จ ก็ถอด hololens ออกได้

ตอนถ่ายวีดีโอจะแพนกล้องนิดๆหน่อยๆก็ไม่มีปัญหา เพราะ zed มันเป็น 6DOF trackตำแหน่งได้

 

ส่วนของตัวเอง ตอนนี้มี kinectv2 อยู่ ว่าจะลองต่อดู…

แล้วก็ต้องเขียน app ให้มัน share hologram ได้…

Using Markers with World Anchors in Mixed Reality Scene Sharing

Using Markers with World Anchors in Mixed Reality Scene Sharing