Browser via WebRTC

index.html

@@ -487,6 +487,7 @@ <h1>Loading...</h1>
       </div>
     </header>
     <canvas id="canvas"></canvas>
+    <!-- <script src="https://webrtc.github.io/adapter/adapter-latest.js"></script> -->
     <script src="https://kit.fontawesome.com/0735724151.js"></script>
     <script src="three.js"></script>
     <script src="OrbitControls.js"></script>
@@ -523,6 +524,7 @@ <h1>Loading...</h1>
 };
 const DEFAULT_URL = 'https://google.com';
 const PARCEL_SIZE = 8;
+const BROWSER_SIZE = 1920;
 const TOKEN_PRICE = 5e15;
 const DEFAULT_SKIN_URL = 'skin.png';
 const RAY_COLOR = 0x44c2ff;
@@ -4370,7 +4372,9 @@ <h3>
   };
 
   mesh.getIntersectionCandidates = () => mesh.visible ?
-    [mesh, keyboardMesh].concat(appIconMeshes.children)
+    [mesh, keyboardMesh]
+      .concat(appIconMeshes.children)
+      .concat(browserMesh || [])
   : [];
   return mesh;
 };
@@ -5782,6 +5786,9 @@ <h3>
             intersectionSpec.onclose(i);
           } else if (type === 'screen') {
             screenMesh.click(intersectionSpec);
+          } else if (type === 'browser') {
+            const {mesh: browserMesh} = intersectionSpec;
+            browserMesh.click();
           } else if (type === 'map') {
             window.document.xrOffset.position[0] += intersectionSpec.x*7 - camera.position.x;
             window.document.xrOffset.position[2] += intersectionSpec.y*7 - camera.position.z;
@@ -6603,14 +6610,13 @@ <h3>
       {'urls': 'stun:stun.l.google.com:19302'},
     ],
   };
-
   const peerConnection = new RTCPeerConnection(peerConnectionConfig);
   peerConnection.ontrack = e => {
     console.log('got track', e);
   };
 
   let skinMesh = null;
-  const sendChannel = peerConnection.createDataChannel('sendChannel');
+  const sendChannel = peerConnection.createDataChannel('pose');
   let pingInterval = 0;
   let updateInterval = 0;
   sendChannel.onopen = () => {
@@ -6736,6 +6742,9 @@ <h3>
 
 const connectionId = Math.random().toString(36).replace(/[^a-z]+/g, '').substr(0, 5);
 const peerConnections = [];
+let browserPeerConnection = null;
+let browserDataChannel = null;
+let browserMesh = null;
 const _rtcConnect = () => {
   const s = new WebSocket(LAMBDA_URLS.presence);
   s.onopen = () => {
@@ -6745,9 +6754,111 @@ <h3>
       method: 'init',
       connectionId,
     }));
+
+    const peerConnectionConfig = {
+      iceServers: [
+        {'urls': 'stun:stun.stunprotocol.org:3478'},
+        {'urls': 'stun:stun.l.google.com:19302'},
+      ],
+    };
+    browserPeerConnection = new RTCPeerConnection(peerConnectionConfig);
+    const dataChannel = browserPeerConnection.createDataChannel('browser', {id: 0, negotiated: true});
+    dataChannel.onopen = () => {
+      console.log('got browser data channel', dataChannel);
+    };
+    browserDataChannel = dataChannel;
+    browserPeerConnection.ontrack = e => {
+      console.log('got browser track', e, e.streams.length);
+
+      const video = document.createElement('video');
+      video.style.width = '100%';
+      video.srcObject = e.streams[0];
+      video.play();
+
+      const s = 2;
+      const r = s*4/(2*Math.PI);
+      const geometry = new THREE.CylinderBufferGeometry(r, r, s, 10, 10, true, Math.PI*3/2+Math.PI/4, Math.PI/2)
+        .applyMatrix(new THREE.Matrix4().makeScale(1, 1, -1))
+        .applyMatrix(new THREE.Matrix4().makeTranslation(0, 0, r));
+
+      const texture = new THREE.VideoTexture(video);
+      texture.minFilter = THREE.LinearFilter;
+      texture.magFilter = THREE.LinearFilter;
+      texture.format = THREE.RGBFormat;
+
+      const material = new THREE.MeshBasicMaterial({
+        map: texture,
+        side: THREE.DoubleSide,
+      });
+
+      const mesh = new THREE.Mesh(geometry, material);
+      mesh.frustumCulled = false;
+      const lastCoords = [NaN, NaN];
+      mesh.intersect = ray => {
+        localRaycaster.ray.copy(ray);
+        const intersection = localRaycaster.intersectObject(mesh)[0];
+        if (intersection) {
+          const {uv, distance} = intersection;
+          uv.y = 1-uv.y;
+          uv.x *= BROWSER_SIZE;
+          uv.y *= BROWSER_SIZE;
+          const {x, y} = uv;
+          if (x !== lastCoords[0] || y !== lastCoords[1]) {
+            mesh.mousemove(x, y);
+
+            lastCoords[0] = x;
+            lastCoords[1] = y;
+          }
+          return {
+            type: 'browser',
+            x,
+            y,
+            distance,
+            mesh,
+            cancel: true,
+          };
+        } else {
+          return null;
+        }
+      };
+      mesh.mousemove = (x, y) => {
+        console.log('mouse move', x, y, dataChannel && dataChannel.readyState);
+        if (dataChannel && dataChannel.readyState === 'open') {
+          dataChannel.send(JSON.stringify({
+            method: 'mousemove',
+            x,
+            y: y + 160*(1 - y/BROWSER_SIZE),
+          }));
+        }
+      };
+      mesh.click = () => {
+        if (dataChannel && dataChannel.readyState === 'open') {
+          dataChannel.send(JSON.stringify({
+            method: 'click',
+          }));
+        }
+      };
+      mesh.position.y = 2;
+      scene.add(mesh);
+      browserMesh = mesh;
+
+      e.track.onended = () => {
+        console.log('track ended!!!!!!!!!!!!!!!!!!!!!!!!!', e.track);
+      };
+
+      /* document.body.querySelector('header').style.display = 'none';
+      document.body.querySelector('#canvas').style.display = 'none';
+      document.body.appendChild(video); */
+    };
+    s.send(JSON.stringify({
+      method: 'requestBrowser',
+      url: 'https://google.com/',
+      connectionId,
+    }));
+    peerConnections.push(browserPeerConnection);
   };
   s.onmessage = e => {
-    // console.log('got message', e.data);
+    console.log('got message', e.data);
 
     const data = JSON.parse(e.data);
     const {method} = data;
@@ -6837,6 +6948,29 @@ <h3>
       } else {
         console.warn('no such peer connection', peerConnectionId, peerConnections.map(peerConnection => peerConnection.connectionId));
       }
+    } else if (method === 'respondBrowser') {
+      const {src: peerConnectionId, offer} = data;
+      browserPeerConnection.connectionId = peerConnectionId;
+      browserPeerConnection.onicecandidate = e => {
+        s.send(JSON.stringify({
+          dst: peerConnectionId,
+          src: connectionId,
+          method: 'iceCandidate',
+          candidate: e.candidate,
+        }));
+      };
+      browserPeerConnection.setRemoteDescription(offer);
+      browserPeerConnection.createAnswer()
+        .then(answer => {
+          browserPeerConnection.setLocalDescription(answer);
+
+          s.send(JSON.stringify({
+            dst: peerConnectionId,
+            src: connectionId,
+            method: 'answer',
+            answer,
+          }));
+        });
     }
   };
   s.onclose = () => {
@@ -6859,11 +6993,5 @@ <h3>
 })();
     </script>
     <!-- <audio src="background.mp3" autoplay loop></audio> -->
-    <script src="bignumber.js"></script>
-    <script src="web3.js"></script>
-    <script src="bip32.js"></script>
-    <script src="bip39.js"></script>
-    <script src="api.js"></script>
-    <script src="ethereumjs-util.js"></script>
   </body>
 </html>