Today we are going to build a simple TCP proxy server. The scenario is the following one. We have got one host (the client) that establishes a TCP connection to another one (the remote).
client —> remote
We want to set up a proxy server in the middle, so the client will establish the connection with the proxy and the proxy will forward it to the remote, keeping in mind the remote response also.
With node.js is really simple to perform those kind of network operations.
client —> proxy -> remote
var net = require('net');
var LOCAL_PORT = 6512;
var REMOTE_PORT = 6512;
var REMOTE_ADDR = "192.168.1.25";
var server = net.createServer(function (socket) {
socket.on('data', function (msg) {
console.log(' ** START **');
console.log('<< From client to proxy ', msg.toString());
var serviceSocket = new net.Socket();
serviceSocket.connect(parseInt(REMOTE_PORT), REMOTE_ADDR, function () {
console.log('>> From proxy to remote', msg.toString());
serviceSocket.write(msg);
});
serviceSocket.on("data", function (data) {
console.log('<< From remote to proxy', data.toString());
socket.write(data);
console.log('>> From proxy to client', data.toString());
});
});
});
server.listen(LOCAL_PORT);
console.log("TCP server accepting connection on port: " + LOCAL_PORT);
Why this script? OK. It was a crazy idea. It started with one “Is it possible? Yes, let’s code it” in my mind. Let start. I want to scroll one web page in the TV’s web browser (or PC’s browser) using my smartphone lying on my couch. I’ve got a wireless mouse so I don’t really need it, but scroll the TV browser with the smartphone sounds cool, isn’t it?
The idea is the following one:
One QR code in our web page (added dinamically with JavaScrip with Google’s Chart API ). Write urls with the smartphone is hard and QR has a good hype, so we will add a QR code at the bottom of the web page with the link to the node.js server.
One socket.io server built with a node.js server for the real time communications. This node.js server will serve also a jQuery Mobile application with four buttons (with express and jade):
The server will register the WebSocket and send the real time commands to the browser (with one easy-to-hack security token).
The browser will handle the socket.io actions and controls the scroll of the web page.
The code is probably crowded by bugs and security problems, but it works and it was enough in my experiment 🙂 :
The node.js server:
var io = require('socket.io').listen(8080);
var app = require('express').createServer();
app.set('view engine', 'jade');
app.set('view options', {
layout:false
});
app.get('/panel/:key', function (req, res) {
var key = req.params.key;
console.log(key);
res.render('mobile.jade', {key:key});
});
app.get('/action/:key/:y/:action', function (req, res) {
var key = req.params.key;
var y = req.params.y;
var action = req.params.action;
sockets[key].emit('scrollTo', {y:y, action:action});
res.send('OK');
});
app.listen(8000);
var sockets = {};
io.sockets.on('connection', function (socket) {
socket.on('setKey', function (key) {
sockets[key] = socket;
});
});
The jade template with the jquery mobile application:
!!! 5
html
head
meta(charset="utf-8")
meta(name="viewport", content="width=device-width, initial-scale=1")
title title
link(rel='stylesheet', href='http://code.jquery.com/mobile/1.1.0/jquery.mobile-1.1.0.min.css')
script(src='http://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js')
script(src='http://code.jquery.com/mobile/1.1.0/jquery.mobile-1.1.0.min.js')
script(type='text/javascript')
$(document).bind('pageinit', function() {
$('#toTop').tap(function() {
$.get('/action/#{key}/0/go', function() {});
});
$('#toBotton').tap(function() {
$.get('/action/#{key}/max/go', function() {});
});
$('#toUp').tap(function() {
$.get('/action/#{key}/100/rew', function() {});
});
$('#toDown').tap(function() {
$.get('/action/#{key}/100/ffd', function() {});
});
});
body
#page1(data-role="page")
#header(data-theme="a", data-role="header")
h3 Header
#content(data-role="content")
a(data-role="button", data-transition="fade", data-theme="a", href="#", id="toTop", data-icon="minus", data-iconpos="left") Top
a(data-role="button", data-transition="fade", href="#", id="toUp", data-icon="arrow-u", data-iconpos="left") Up
a(data-role="button", data-transition="fade", href="#", id="toDown", data-icon="arrow-d", data-iconpos="left") Down
a(data-r
Our Html client page
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
<title>jQuery Smooth Scroll - Design Woop</title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
</head>
<body>
<p>
Lorem ipsum ….. put a big lorem ipsum here to make possible the scroll
</p>
<script src="http://localhost:8080/socket.io/socket.io.js"></script>
<script src="client.js"></script>
</body>
</html>
And finally our client magic with the QR code and the functions to handle the node.js actions
var key = "secret";
function getDocHeight() {
var D = document;
return Math.max(
Math.max(D.body.scrollHeight, D.documentElement.scrollHeight),
Math.max(D.body.offsetHeight, D.documentElement.offsetHeight),
Math.max(D.body.clientHeight, D.documentElement.clientHeight)
);
}
var socket = io.connect('http://localhost:8080');
var y = 0;
socket.emit('setKey', key);
socket.on('scrollTo', function (data) {
if (data.y == 'max') {
y = getDocHeight();
} else {
if (data.action == 'ffd') {
y += parseInt(data.y);
} else if (data.action == 'go') {
y = parseInt(data.y);
} else {
y -= parseInt(data.y);
}
}
window.scrollTo(0, y);
});
document.write('<img src="https://chart.googleapis.com/chart?chs=150x150&cht=qr&chl=http://192.168.2.3:8000/panel/' + key + '&choe=UTF-8" alt=""/>');
You can see the code in github too. We also can see the script in action here:
We can also add more features to our application but that’s enought for this experiment. What do you think?
Normally we perform our database connection at server side with PHP and PDO for example. In this post I will show a simple technique to send queries to our database (PostgreSql in this example) asynchronously using node.js and socket.io.
The idea is pretty straightforward. We will send the SQL string and the values with a WebSocket and we will execute a callback in the client when the server (node.js script) fetches the recordset.
Our server:
var pg = require('pg');
var conString = "tcp://user:password@localhost:5432/db";
var client = new pg.Client(conString);
client.connect();
var io = require('socket.io').listen(8888);
io.sockets.on('connection', function (socket) {
socket.on('sql', function (data) {
var query = client.query(data.sql, data.values);
query.on('row', function(row) {
socket.emit('sql', row);
});
});
});
And our client:
<script src="http://localhost:8888/socket.io/socket.io.js"></script>
<script>
var socket = io.connect('http://vmserver:8888');
function sendSql(sql, values, cbk) {
socket.emit('sql', { sql: sql, values : values});
socket.on('sql', function(data){
console.log(data);
});
}
</script>
<p>
<a href="#" onclick="sendSql('select * from users', [], function(data) {console.log(data);})">select * from users</a>
</p>
<p>
<a href="#" onclick="sendSql('select * from users where uid=$1', [4], function(data) {console.log(data);})">select * from users where uid=$1</a>
</p>
Simple, isn’t it?
You must take care if you use this script at production. Our database is exposed to raw SQL sent from the client. It’s a concept example. Maybe it would be better not to send the SQL. Store them into key-value table in the server and send only an ID from the browser.
2011 is about to finish, and I want to speak about my way through the world of TDD. In the beginning of the year appeared a new cool project called 12meses12katas (12 months – 12 katas). The aim of the project was to propose one coding kata per month, and allow to the people to publish their implementation of the kata over github. In the line of this project a crew of crazy coders started another cool project called Katayunos. Katayunos is a small pun with the word Desayuno (Breakfast) and Coding Kata. It’s something similar than a code retreat. The purpose of the katayunos was to meet together in one place one saturday morning, have a breakfast and play with pair programming and TDD with one coding kata. Something too geek to explain to non-geek people, I known, but if you are reading this, It’s probable that your understand this ;). Our first katayuno was in the cafeteria of one Hotel. One cold Saturday morning (a really cold one believe me). The main problem was that we didn’t have any electrical plug, so our pomodoros were marked by the laptop batteries.
After this cold first katayuno we achieve new places with wifi and those kind of luxuries. Those kind of events are really interesting because we meet together different people and different programming languages. I still remember when I played with c# one time (I still have the sulfur smell in my fingers 🙂 ).
Basically the idea is:
Choose a coding kata
Pomodoros of 25 minutes
Pair programming (change the couples with each iteration)
Have fun
And now I’m going to show my 12 implementations of the 12 katas from 12meses12katas’s project:
Jaunary: String-Calculator
A good kata for beginners. That’s a good one because the testing strategy is very clear and we can focus on Testing: Red (test NOK) -> implement code -> Green (test OK).
My solution: php + phpunit
March: FizzBuzz
A famous kata. A Really simple one. Because of that we can focus on the TDD (baby steps, refactoring, …)
My solution: php + phpunit
I also wrote a post with a crazy dynamic classes and the implementation of FizzBuzz kata with those dinamic classes:
April: Bowling
My problem was that I didn’t know the rules of Bowling so I needed to learn how to play Bowling first.
My solution: php + phpunit
May: KataLonja
A strange kata. I like it because it’s not a theoretical one. It’s close to a real problem.
My solution: php + phpunit
June: Simple-Lists
It was my first kata with javascript and node.js.
My solution: node.js + nodeunit (with John Resig’s class implementation)
So, what are you waiting for to create a local group and clone the katayuno in your city? It’s very easy: broadcast the event over twitter, pick a place, pick a kata andenjoy.
Last friday I spoke about node.js and Websockets with the people of The Mêlée. The talk was an introduction to node.js and focused in the new HTML5 feature: the WebSockets.
When I spoke about Websockets I also introduced the great library socket.io. The jQuery of WebSockets.
We use sessions when we want to preserve certain data across subsequent accesses. PHP allows us to use different handlers when we’re using sessions. The default one is filesystem, but we can change it with session.save_handler in the php.ini. session.save_handler defines the name of the handler which is used for storing and retrieving data associated with a session. We also can create our own handler to manage sessions. In this post we’re going to create a custom handler to store sessions in a node.js service. Let’s start:
A simple usage of sessions with PHP. If we reload the page our counter will be incremented by one. We’re using the default session handler. It works without any problem.
The idea is create a custom handler to use a server with node.js to store the session information instead of filesystem. To create custom handlers we need to use the PHP function: session_set_save_handler and rewrite the callbacks for: open, close, read, write, destroy and gc. PHP’s documentation is great. My proposal is the following one:
In one of my previous posts I wrote about Real time notifications with PHP. I wanted to create a simple comet system fully written in PHP and JavaScript. It worked but as Scott Mattocks told me in a comment this implementation was still just doing short polling. The performance with this solution may be bad in a medium/hight traffic site. This days I’m playing with node.js and I want to create a simple test. I want to do exactly the same than the previous post but now with node.js instead of my PHP+js test. Let’s start
Now I want to use socket.io instead of pure web-sockets like my previous posts about node.js. For those who don’t know, socket.io is amazing library that allows us to use real-time technologies within every browsers (yes even with IE6). It uses one technology or another depending on the browser we are using, with the same interface for the developer. That’s means if we’re using Google Chrome we will use websockets, but if our browser does’t support them, socket.io will choose another supported transports. Definitely socket.io is the jQuery of the websockets. The supporter transports are:
WebSocket
Adobe Flash Socket
AJAX long polling
AJAX multipart streaming
Forever Iframe
JSONP Polling
First we create our node.js server. A really simple one.
var http = require('http');
var io = require('socket.io');
server = http.createServer(function(req, res){
});
server.listen(8080);
// socket.io
var socket = io.listen(server);
socket.on('connection', function(client){
client.on('message', function(msg){
socket.broadcast(msg);
})
});
This server will broadcast the message received from the browser to all connected clients.
As we can see we are including the js script called socket.io/socket.io.js. This script is served by our node server.
In fact we can use our node.js to serve everything (HTML, js, CSS) but in our example we will use only node.js for real-time stuff. Apache will serve the rest of the code (only a HTML file in this case).
And that’s all. Those few lines perform the same thing than our PHP and js code in the other post’s example. Our node.js implementation is definitely smarter than the PHP one. The socket.io library also allows us to use the example with all browser. Same code and without any browser nightmare (just like jQuery when we work with DOM).
Here I have a little screencast with the working example. As we will see there We will connect to the node server with firefox and chrome. Firefox will use xhr multipart and Chrome will use Websokets.
Another important issue of socket.io library is that we forget about the reconnection to the web-socket server, if something wrong happens (as we can see in Real time monitoring PHP applications with web-sockets and node.js). If we use raw WebSocket implementations and our connection with the web-socket server crashes or if we stop the server, our application will raise disconnect event and we need to create something to reconnect to the server. socket.io does it for us. With our small piece of JavaScript code we will get a high performance real-time applicatrion. Node is cool. Really cool. Kinda wierd at the beginning but the learning effort will be worthwhile. A few js lines and a real time applications.
I’ve got a problem within our node.js application. If we’ve got some kind of security within our application (imagine for example it’s behind a session based auth form) we need to share this security layer with our node.js server to ensure that non authenticated users aren’t allowed to use our websockets. I don’t know how to do it just now, but I’m investigating. Do you have any idea?
Full code Code at github. Ensure you’re using the stable version of node.js. With the last versión available on github of node.js there’s a bug and server dies when we connect with Google Chrome.
Continuing with my experiments of node.js, this time I want to create a Web console. The idea is simple. I want to send a few command to the server and I display the output inside the browser. I can do it entirely with PHP but I want to send the output to the browser as fast as they appear without waiting for the end of the command. OK we can do it flushing the output in the server but this solution normally crashes if we keep the application open for a long time. WebSockets again to the rescue. If we need a cross-browser implementation we need the socket.io library. Let’s start:
The node server is a simple websocket server. In this example we will launch each command with spawn function (require(‘child_process’).spawn) and send the output within the websoket. Simple and pretty straightforward.
var sys = require('sys'),
http = require('http'),
url = require('url'),
spawn = require('child_process').spawn,
ws = require('./ws.js');
var availableComands = ['ls', 'ps', 'uptime', 'tail', 'cat'];
ws.createServer(function(websocket) {
websocket.on('connect', function(resource) {
var parsedUrl = url.parse(resource, true);
var rawCmd = parsedUrl.query.cmd;
var cmd = rawCmd.split(" ");
if (cmd[0] == 'help') {
websocket.write("Available comands: \n");
for (i=0;i<availableComands.length;i++) {
websocket.write(availableComands[i]);
if (i< availableComands.length - 1) {
websocket.write(", ");
}
}
websocket.write("\n");
websocket.end();
} else if (availableComands.indexOf(cmd[0]) >= 0) {
if (cmd.length > 1) {
options = cmd.slice(1);
} else {
options = [];
}
try {
var process = spawn(cmd[0], options);
} catch(err) {
console.log(err);
websocket.write("ERROR");
}
websocket.on('end', function() {
process.kill();
});
process.stdout.on('data', function(data) {
websocket.write(data);
});
process.stdout.on('end', function() {
websocket.end();
});
} else {
websocket.write("Comand not available. Type help for available comands\n");
websocket.end();
}
});
}).listen(8880, '127.0.0.1');
The web client is similar than the example of my previous post
var timeout = 5000;
var wsServer = '127.0.0.1:8880';
var ws;
function cleanString(string) {
return string.replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">");
}
function pad(n) {
return ("0" + n).slice(-2);
}
var cmdHistory = [];
function send(msg) {
if (msg == 'clear') {
$('#log').html('');
return;
}
try {
ws = new WebSocket('ws://' + wsServer + '?cmd=' + msg);
$('#toolbar').css('background', '#933');
$('#socketStatus').html("working ... [<a href='#' onClick='quit()'>X</a>]");
cmdHistory.push(msg);
$('#log').append("<div class='cmd'>" + msg + "</div>");
console.log("startWs:");
} catch (err) {
console.log(err);
setTimeout(startWs, timeout);
}
ws.onmessage = function(event) {
$('#log').append(util.toStaticHTML(event.data));
window.scrollBy(0, 100000000000000000);
};
ws.onclose = function(){
//console.log("ws.onclose");
$('#toolbar').css('background', '#65A33F');
$('#socketStatus').html('Type your comand:');
}
}
function quit() {
ws.close();
window.scrollBy(0, 100000000000000000);
}
util = {
urlRE: /https?:\/\/([-\w\.]+)+(:\d+)?(\/([^\s]*(\?\S+)?)?)?/g,
// html sanitizer
toStaticHTML: function(inputHtml) {
inputHtml = inputHtml.toString();
return inputHtml.replace(/&/g, "&")
.replace(/</g, "<")
.replace("/n", "<br/>")
.replace(/>/g, ">");
},
//pads n with zeros on the left,
//digits is minimum length of output
//zeroPad(3, 5); returns "005"
//zeroPad(2, 500); returns "500"
zeroPad: function (digits, n) {
n = n.toString();
while (n.length < digits)
n = '0' + n;
return n;
},
//it is almost 8 o'clock PM here
//timeString(new Date); returns "19:49"
timeString: function (date) {
var minutes = date.getMinutes().toString();
var hours = date.getHours().toString();
return this.zeroPad(2, hours) + ":" + this.zeroPad(2, minutes);
},
//does the argument only contain whitespace?
isBlank: function(text) {
var blank = /^\s*$/;
return (text.match(blank) !== null);
}
};
$(document).ready(function() {
//submit new messages when the user hits enter if the message isnt blank
$("#entry").keypress(function (e) {
console.log(e);
if (e.keyCode != 13 /* Return */) return;
var msg = $("#entry").attr("value").replace("\n", "");
if (!util.isBlank(msg)) send(msg);
$("#entry").attr("value", ""); // clear the entry field.
});
});
And that’s all. In fact we don’t need any line of PHP to perform this web console. Last year I tried to do something similar with PHP but it was a big mess. With node those kind of jobs are trivial. I don’t know if node.js is the future or is just another hype, but it’s easy. And cool. Really cool.
You can see the full code at Github here. Anyway you must take care if you run this application on your host. You are letting user to execute raw unix commands. A bit of security layer would be necessary.
The inspection of the error logs is a common way to detect errors and bugs. We also can show errors on-screen within our developement server, or we even can use great tools like firePHP to show our PHP errors and warnings inside our firebug console. That’s cool, but we only can see our session errors/warnings. If we want to see another’s errors we need to inspect the error log. tail -f is our friend, but we need to surf against all the warnings of all sessions to see our desired ones. Because of that I want to build a tool to monitor my PHP applications in real-time. Let’s start:
What’s the idea? The idea is catch all PHP’s errors and warnings at run time and send them to a node.js HTTP server. This server will work similar than a chat server but our clients will only be able to read the server’s logs. Basically the applications have three parts: the node.js server, the web client (html5) and the server part (PHP). Let me explain a bit each part:
The node Server
Basically it has two parts: a http server to handle the PHP errors/warnings and a websocket server to manage the realtime communications with the browser. When I say that I’m using websockets that’s means the web client will only work with a browser with websocket support like chrome. Anyway it’s pretty straightforward swap from a websocket sever to a socket.io server to use it with every browser. But websockets seems to be the future, so I will use websockets in this example.
The http server:
http.createServer(function (req, res) {
var remoteAdrress = req.socket.remoteAddress;
if (allowedIP.indexOf(remoteAdrress) >= 0) {
res.writeHead(200, {
'Content-Type': 'text/plain'
});
res.end('Ok\n');
try {
var parsedUrl = url.parse(req.url, true);
var type = parsedUrl.query.type;
var logString = parsedUrl.query.logString;
var ip = eval(parsedUrl.query.logString)[0];
if (inspectingUrl == "" || inspectingUrl == ip) {
clients.forEach(function(client) {
client.write(logString);
});
}
} catch(err) {
console.log("500 to " + remoteAdrress);
res.writeHead(500, {
'Content-Type': 'text/plain'
});
res.end('System Error\n');
}
} else {
console.log("401 to " + remoteAdrress);
res.writeHead(401, {
'Content-Type': 'text/plain'
});
res.end('Not Authorized\n');
}
}).listen(httpConf.port, httpConf.host);
and the web socket server:
var inspectingUrl = undefined;
ws.createServer(function(websocket) {
websocket.on('connect', function(resource) {
var parsedUrl = url.parse(resource, true);
inspectingUrl = parsedUrl.query.ip;
clients.push(websocket);
});
websocket.on('close', function() {
var pos = clients.indexOf(websocket);
if (pos >= 0) {
clients.splice(pos, 1);
}
});
}).listen(wsConf.port, wsConf.host);
If you want to know more about node.js and see more examples, have a look to the great site: http://nodetuts.com/. In this site Pedro Teixeira will show examples and node.js tutorials. In fact my node.js http + websoket server is a mix of two tutorials from this site.
The web client.
The web client is a simple websockets application. We will handle the websockets connection, reconnect if it dies and a bit more. I’s based on node.js chat demo
var timeout = 5000;
var wsServer = '192.168.2.2:8880';
var unread = 0;
var focus = false;
var count = 0;
function updateCount() {
count++;
$("#count").text(count);
}
function cleanString(string) {
return string.replace(/&/g,"&").replace(/</g,"<").replace(/>/g,">");
}
function updateUptime () {
var now = new Date();
$("#uptime").text(now.toRelativeTime());
}
function updateTitle(){
if (unread) {
document.title = "(" + unread.toString() + ") Real time " + selectedIp + " monitor";
} else {
document.title = "Real time " + selectedIp + " monitor";
}
}
function pad(n) {
return ("0" + n).slice(-2);
}
function startWs(ip) {
try {
ws = new WebSocket("ws://" + wsServer + "?ip=" + ip);
$('#toolbar').css('background', '#65A33F');
$('#socketStatus').html('Connected to ' + wsServer);
//console.log("startWs:" + ip);
//listen for browser events so we know to update the document title
$(window).bind("blur", function() {
focus = false;
updateTitle();
});
$(window).bind("focus", function() {
focus = true;
unread = 0;
updateTitle();
});
} catch (err) {
//console.log(err);
setTimeout(startWs, timeout);
}
ws.onmessage = function(event) {
unread++;
updateTitle();
var now = new Date();
var hh = pad(now.getHours());
var mm = pad(now.getMinutes());
var ss = pad(now.getSeconds());
var timeMark = '[' + hh + ':' + mm + ':' + ss + '] ';
logString = eval(event.data);
var host = logString[0];
var line = "<table class='message'><tr><td width='1%' class='date'>" + timeMark + "</td><td width='1%' valign='top' class='host'><a href=?ip=" + host + ">" + host + "</a></td>";
line += "<td class='msg-text' width='98%'>" + logString[1]; + "</td></tr>";
if (logString[2]) {
line += "<tr><td> </td><td colspan='3' class='msg-text'>" + logString[2] + "</td></tr>";
}
$('#log').append(line);
updateCount();
window.scrollBy(0, 100000000000000000);
};
ws.onclose = function(){
//console.log("ws.onclose");
$('#toolbar').css('background', '#933');
$('#socketStatus').html('Disconected');
setTimeout(function() {startWs(selectedIp)}, timeout);
}
}
$(document).ready(function() {
startWs(selectedIp);
});
The server part:
The server part will handle silently all PHP warnings and errors and it will send them to the node server. The idea is to place a minimal PHP line of code at the beginning of the application that we want to monitor. Imagine the following piece of PHP code
$a = $var[1];
$a = 1/0;
class Dummy
{
static function err()
{
throw new Exception("error");
}
}
Dummy1::err();
it will throw:
A notice: Undefined variable: var
A warning: Division by zero
An Uncaught exception ‘Exception’ with message ‘error’
So we will add our small library to catch those errors and send them to the node server
include('client/NodeLog.php');
NodeLog::init('192.168.2.2');
$a = $var[1];
$a = 1/0;
class Dummy
{
static function err()
{
throw new Exception("error");
}
}
Dummy1::err();
The script will work in the same way than the fist version but if we start our node.js server in a console:
$ node server.js
HTTP server started at 192.168.2.2::5672
Web Socket server started at 192.168.2.2::8880
We will see those errors/warnings in real-time when we start our browser
Here we can see a small screencast with the working application:
