PDA

View Full Version : Transfer Image Data using Web Sockets

goro
09-06-2012, 08:37 AM
I am trying to send Image data captured with vizcapture using Web Sockets. I am not expert with computer networks or network protocols but done this work using Vizard Help, Help from World Viz Forum & HTML5 Tutorial from W3Schools. When I run the code and open HTML 5 supporting web browser, Google Chrome in my case with url http://localhost:8080/vizhtml/websocket for local host it runs fine for some time. Bellow is the code given:


import viz
import vizhtml
import vizact
import vizcapture

viz.setMultiSample(4)
viz.fov(60)
viz.go()

import vizinfo
info = vizinfo.add('This script demonstrates how to add multiple windows.\nThe upper right window is a rear view of the scene\nThe upper left window is a birds eye view')
info.translate([0.95,0.05])
info.alignment(vizinfo.LOWER_RIGHT)

#Add environment
viz.addChild('maze.osgb')

# Create a new window in the upper left corner
UpperLeftWindow = viz.addWindow(pos=(0,1.0),size=(0.2,0.2))
UpperLeftWindow.visible(0,viz.SCREEN)

#Create a new window in the upper right corner
UpperRightWindow = viz.addWindow(pos=(0.8,1.0),size=(0.2,0.2))
UpperRightWindow.visible(0,viz.SCREEN)

# Create a new viewpoint
BirdView = viz.addView()

#Attach the bird's eye view to the upper left window
UpperLeftWindow.setView(BirdView)

#Move the view above the center of the room
BirdView.setPosition([0,25,0])

#Rotate the view so that it looks down
BirdView.setEuler([0,90,0])

#Create another viewpoint
RearView = viz.addView()

#Attach the rear view to the upper right window
UpperRightWindow.setView(RearView)

#Increase the field-of-view for both windows
UpperLeftWindow.fov(60)
UpperRightWindow.fov(60)

#Add an arrow marker to bird's eye view window to represent our current position/orientation
arrow = viz.addTexQuad(parent=viz.ORTHO,scene=UpperLeftWin dow,size=20)
arrow.texture(viz.add('arrow.tif'))

# Web Socket & HTML 5 Code
code = """
<html>
<head>
<title>vizhtml WebSocket Example</title>
<script type="text/javascript" src="vizhtml.js"></script>
</head>
<body>
<canvas id="VizardScreen" width="180" height="140" style="border:1px solid #d3d3d3;">
Your browser does not support the HTML5 canvas tag.</canvas>
<script language="javascript">

var socket = new VizardWebSocket();
var c=document.getElementById("VizardScreen");
var ctx=c.getContext("2d");
var imgData=ctx.createImageData(160,120);

socket.onevent('open', function(e){
document.getElementById('status').innerHTML = 'connected';
document.getElementById('my_button').disabled = false;
})

socket.onevent('close', function(e){
document.getElementById('status').innerHTML = 'waiting for connection';
document.getElementById('my_button').disabled = true;
socket.reconnect();
})

socket.onevent('set_time', function(e){
document.getElementById('script_time').innerHTML = e.data.time;
})

socket.onevent('set_data', function(e){
var j = 0;
for (var i=0;i<imgData.data.length;i+=4)
{
imgData.data[i+0]=e.data.iData[j+2];
imgData.data[i+1]=e.data.iData[j+1];
imgData.data[i+2]=e.data.iData[j];
imgData.data[i+3]=255;
j+=3;
}
ctx.putImageData(imgData,10,10);
})

var click_count = 0;
function onButtonClick()
{
click_count += 1;
socket.send( 'button_click' , VizardData({'count':click_count}) );
}

</script>

<div>Status:</div><div id='status'>waiting for connection</div></br>
<div>Time:</div><div id='script_time'></div></br>
<div><input id='my_button' type=button value="Send Message" onclick="onButtonClick();"></div></br>

</body>
</html>
"""

# http://localhost:8080/vizhtml/websocket
vizhtml.registerCode('websocket',code)

def ClientConnect(e):
print 'Client connected at',e.client.address
vizhtml.onConnect(ClientConnect)

def ClientDisconnect(e):
print 'Client disconnected at',e.client.address
vizhtml.onDisconnect(ClientDisconnect)

def SendTime():
d = viz.Data(time='%.1f'%(viz.tick()))
vizhtml.sendAll('set_time',d)
vizact.ontimer(0.1,SendTime)

def OnButtonClick(e):
print 'BUTTON CLICK',e.data.count,'from',e.client.address
vizhtml.onMessage('button_click',OnButtonClick)

def SendImageData():
width,height,data = vizcapture.screenBuffer(flip = True)
lendata = len(data)
print lendata
iData = []
for j in range (120):
for i in range(160):
iData.append(ord(data[800*3*5*(j-1)+3*5*(i-1)]))
iData.append(ord(data[800*3*5*(j-1)+3*5*(i-1)+1]))
iData.append(ord(data[800*3*5*(j-1)+3*5*(i-1)+2]))
d = viz.Data(iData = (iData))
print len(iData)
vizhtml.sendAll('set_data',d)
vizact.ontimer(0.1,SendImageData)

def UpdateViews():

#Get the current head orientation and position
yaw,pitch,roll = viz.MainView.getEuler()
pos = viz.MainView.getPosition()

#Move the rear view to the current position
RearView.setPosition(pos)

#Rotate the rear view so that it faces behind us
RearView.setEuler([yaw+180,0,0])

#Move arrow to our current location
x,y,z = UpperLeftWindow.worldToScreen(pos,mode=viz.WINDOW_ PIXELS)
arrow.setPosition([x,y,0])
arrow.setEuler([0,0,-yaw])

vizact.ontimer(0,UpdateViews)

# Turn on collision detection so we can't go through walls
viz.collision(viz.ON)


This is the output which I get:
http://i438.photobucket.com/albums/qq105/gaurav_kudalkar/ScreenCapture.jpg

Now the issues are:
1. When I run this code for certain time it starts throwing errors
Traceback (most recent call last):
File "C:\Program Files\WorldViz\Vizard4\python\vizact.py", line 3606, in __ontimer
val = event.call()
File "C:\Program Files\WorldViz\Vizard4\python\vizact.py", line 3379, in _callStatic
return func(*args,**kwargs)
File "C:\Documents and Settings\User\Desktop\screenCapture\screenCapture. py", line 142, in SendImageData
iData.append(ord(data[800*3*5*(j-1)+3*5*(i-1)]))
IndexError: string index out of range

2. Here I am reducing the resolution from 800 x 600 to 160 x 120 but if I don't do that my for loop runs for 800 x 600 x 3 = 1440000 times for single update. That slows down the performance of the system.

Please help me resolving these two issues.
Thanks!
goro
09-06-2012, 10:05 AM
I have solved first issue of my own. That error was because Vizard free version changes its resolution after 20 Seconds. I have made the code dynamic.


import viz
import vizhtml
import vizact
import vizcapture

viz.setMultiSample(4)
viz.fov(60)
viz.go()

import vizinfo
info = vizinfo.add('This script demonstrates how to add multiple windows.\nThe upper right window is a rear view of the scene\nThe upper left window is a birds eye view')
info.translate([0.95,0.05])
info.alignment(vizinfo.LOWER_RIGHT)

#resolution of Web Page Image
w = 160
h = 120

#Add environment
viz.addChild('maze.osgb')

# Create a new window in the upper left corner
UpperLeftWindow = viz.addWindow(pos=(0,1.0),size=(0.2,0.2))
UpperLeftWindow.visible(0,viz.SCREEN)

#Create a new window in the upper right corner
UpperRightWindow = viz.addWindow(pos=(0.8,1.0),size=(0.2,0.2))
UpperRightWindow.visible(0,viz.SCREEN)

# Create a new viewpoint
BirdView = viz.addView()

#Attach the bird's eye view to the upper left window
UpperLeftWindow.setView(BirdView)

#Move the view above the center of the room
BirdView.setPosition([0,25,0])

#Rotate the view so that it looks down
BirdView.setEuler([0,90,0])

#Create another viewpoint
RearView = viz.addView()

#Attach the rear view to the upper right window
UpperRightWindow.setView(RearView)

#Increase the field-of-view for both windows
UpperLeftWindow.fov(60)
UpperRightWindow.fov(60)

#Add an arrow marker to bird's eye view window to represent our current position/orientation
arrow = viz.addTexQuad(parent=viz.ORTHO,scene=UpperLeftWin dow,size=20)
arrow.texture(viz.add('arrow.tif'))

# Web Socket & HTML 5 Code
code = """
<html>
<head>
<title>vizhtml WebSocket Example</title>
<script type="text/javascript" src="vizhtml.js"></script>
</head>
<body>
<canvas id="VizardScreen" width="180" height="140" style="border:1px solid #d3d3d3;">
Your browser does not support the HTML5 canvas tag.</canvas>
<script language="javascript">

var socket = new VizardWebSocket();
var c=document.getElementById("VizardScreen");
var ctx=c.getContext("2d");
var imgData=ctx.createImageData(160,120);

socket.onevent('open', function(e){
document.getElementById('status').innerHTML = 'connected';
document.getElementById('my_button').disabled = false;
})

socket.onevent('close', function(e){
document.getElementById('status').innerHTML = 'waiting for connection';
document.getElementById('my_button').disabled = true;
socket.reconnect();
})

socket.onevent('set_time', function(e){
document.getElementById('script_time').innerHTML = e.data.time;
})

socket.onevent('set_data', function(e){
var j = 0;
for (var i=0;i<imgData.data.length;i+=4)
{
imgData.data[i+0]=e.data.iData[j+2];
imgData.data[i+1]=e.data.iData[j+1];
imgData.data[i+2]=e.data.iData[j];
imgData.data[i+3]=255;
j+=3;
}
ctx.putImageData(imgData,10,10);
})

var click_count = 0;
function onButtonClick()
{
click_count += 1;
socket.send( 'button_click' , VizardData({'count':click_count}) );
}

</script>

<div>Status:</div><div id='status'>waiting for connection</div></br>
<div>Time:</div><div id='script_time'></div></br>
<div><input id='my_button' type=button value="Send Message" onclick="onButtonClick();"></div></br>

</body>
</html>
"""

# http://localhost:8080/vizhtml/websocket
vizhtml.registerCode('websocket',code)

def ClientConnect(e):
print 'Client connected at',e.client.address
vizhtml.onConnect(ClientConnect)

def ClientDisconnect(e):
print 'Client disconnected at',e.client.address
vizhtml.onDisconnect(ClientDisconnect)

def SendTime():
d = viz.Data(time='%.1f'%(viz.tick()))
vizhtml.sendAll('set_time',d)
vizact.ontimer(0.1,SendTime)

def OnButtonClick(e):
print 'BUTTON CLICK',e.data.count,'from',e.client.address
vizhtml.onMessage('button_click',OnButtonClick)

def SendImageData():
global w
global h
width,height,data = vizcapture.screenBuffer(flip = True)
lendata = len(data)
if data:
print width,height,lendata
iData = []
for j in range (h):
for i in range(w):
iData.append(ord(data[800*3*int((height/h)*(j-1))+3*int((width/w)*(i-1))]))
iData.append(ord(data[800*3*int((height/h)*(j-1))+3*int((width/w)*(i-1))+1]))
iData.append(ord(data[800*3*int((height/h)*(j-1))+3*int((width/w)*(i-1))+2]))
d = viz.Data(iData = (iData))

print len(iData)
vizhtml.sendAll('set_data',d)
print len(iData)
vizhtml.sendAll('set_data',d)

vizact.ontimer(0.1,SendImageData)

def UpdateViews():

#Get the current head orientation and position
yaw,pitch,roll = viz.MainView.getEuler()
pos = viz.MainView.getPosition()

#Move the rear view to the current position
RearView.setPosition(pos)

#Rotate the rear view so that it faces behind us
RearView.setEuler([yaw+180,0,0])

#Move arrow to our current location
x,y,z = UpperLeftWindow.worldToScreen(pos,mode=viz.WINDOW_ PIXELS)
arrow.setPosition([x,y,0])
arrow.setEuler([0,0,-yaw])

vizact.ontimer(0,UpdateViews)

# Turn on collision detection so we can't go through walls
viz.collision(viz.ON)
farshizzo
09-06-2012, 11:20 AM
Here is a sample script that will capture the screen to a texture over a specified time interval and send a downsized version of the image to all the connected websocket clients. The image is downsized and compressed to PNG in a separate thread, so it should not slow down the main graphics thread too much, depending on the size of your window. If you need to reduce the network traffic, you could try compressing to JPG instead.
import viz
import vizhtml
import viztask
import base64
import Image
import StringIO
viz.go()

viz.add('maze.osgb')

code = """
<html>
<head>
<title>vizhtml Screen Share Example</title>
<script type="text/javascript" src="vizhtml.js"></script>
</head>
<body>
<script language="javascript">

var socket = new VizardWebSocket();

socket.onevent('open', function(e){
document.getElementById('status').innerHTML = 'connected';
})

socket.onevent('close', function(e){
document.getElementById('status').innerHTML = 'waiting for connection';
socket.reconnect();
})

socket.onevent('set_image', function(e){
document.getElementById('screen').src = 'data:image/png;base64,' + e.data.data;
})

</script>

<div>Status:</div><div id='status'>waiting for connection</div></br>
<div>Screen:</div>
<img id='screen'>
</body>
</html>
"""

# http://localhost:8080/vizhtml/screen_share
vizhtml.registerCode('screen_share',code)

# Size of image to send
IMAGE_SIZE = [160,120]

# Time interval (in seconds) to send screen capture
CAPTURE_INTERVAL = 0.25

def ScreenCaptureTask():

# Blank texture that will hold screen capture
tex = viz.addBlankTexture([1,1])

def SendScreenImage(texture):

# Compute downsample size while preserving original aspect ratio
width, height = texture.getSize()[:2]
ratio = min( IMAGE_SIZE[0] / float(width), IMAGE_SIZE[1] / float(height) )
if ratio < 1.0:
size = [ int(round(width * ratio)), int(round(height * ratio)) ]
else:
size = [ width, height ]

# Convert texture to PIL image
im = Image.frombuffer('RGB',[width,height],texture.getImageData(),'raw','RGB',0,1)

# Resize image and flip
im = im.resize(size).transpose(Image.FLIP_TOP_BOTTOM)

# Save image to base 64 encoded PNG buffer
output = StringIO.StringIO()
im.save(output,'PNG')
buf = base64.b64encode(output.getvalue())
output.close()

# Send buffer to all connected clients
vizhtml.sendAll('set_image',viz.Data(data=buf))

while True:

# Wait for capture interval
yield viztask.waitTime(CAPTURE_INTERVAL)

# Capture screen to texture
viz.window.screenCapture(tex)
yield None

# Send texture to clients asynchronously
yield viztask.waitDirector(SendScreenImage,tex)

viztask.schedule( ScreenCaptureTask() )