Category Archives: node.js

Building a simple TCP proxy server with node.js

Today we are going to build a simple TCP proxy server. The scenario is the following one. We have got one host (the client) that establishes a TCP connection to another one (the remote).

client —> remote

We want to set up a proxy server in the middle, so the client will establish the connection with the proxy and the proxy will forward it to the remote, keeping in mind the remote response also.
With node.js is really simple to perform those kind of network operations.

client —> proxy -> remote

var net = require('net');

var LOCAL_PORT  = 6512;
var REMOTE_PORT = 6512;
var REMOTE_ADDR = "192.168.1.25";

var server = net.createServer(function (socket) {
    socket.on('data', function (msg) {
        console.log('  ** START **');
        console.log('<< From client to proxy ', msg.toString());
        var serviceSocket = new net.Socket();
        serviceSocket.connect(parseInt(REMOTE_PORT), REMOTE_ADDR, function () {
            console.log('>> From proxy to remote', msg.toString());
            serviceSocket.write(msg);
        });
        serviceSocket.on("data", function (data) {
            console.log('<< From remote to proxy', data.toString());
            socket.write(data);
            console.log('>> From proxy to client', data.toString());
        });
    });
});

server.listen(LOCAL_PORT);
console.log("TCP server accepting connection on port: " + LOCAL_PORT);

Simple, isn’t it?
Source code in github

Advertisements

Scroll desktop’s Web pages remotely with our smartphone, using Node.js and WebSockets

Why this script? OK. It was a crazy idea. It started with one “Is it possible? Yes, let’s code it” in my mind. Let start. I want to scroll one web page in the TV’s web browser (or PC’s browser) using my smartphone lying on my couch. I’ve got a wireless mouse so I don’t really need it, but scroll the TV browser with the smartphone sounds cool, isn’t it?

The idea is the following one:

  • One QR code in our web page (added dinamically with JavaScrip with Google’s Chart API ). Write urls with the smartphone is hard and QR has a good hype, so we will add a QR code at the bottom of the web page with the link to the node.js server.
  • One socket.io server built with a node.js server for the real time communications. This node.js server will serve also a jQuery Mobile application with four buttons (with express and jade):
  • The server will register the WebSocket and send the real time commands to the browser (with one easy-to-hack security token).
  • The browser will handle the socket.io actions and controls the scroll of the web page.

The code is probably crowded by bugs and security problems, but it works and it was enough in my experiment 🙂 :

The node.js server:

var io = require('socket.io').listen(8080);

var app = require('express').createServer();
app.set('view engine', 'jade');

app.set('view options', {
    layout:false
});

app.get('/panel/:key', function (req, res) {
    var key = req.params.key;
    console.log(key);
    res.render('mobile.jade', {key:key});
});

app.get('/action/:key/:y/:action', function (req, res) {
    var key = req.params.key;
    var y = req.params.y;
    var action = req.params.action;
    sockets[key].emit('scrollTo', {y:y, action:action});
    res.send('OK');
});

app.listen(8000);

var sockets = {};
io.sockets.on('connection', function (socket) {
    socket.on('setKey', function (key) {
        sockets[key] = socket;
    });
});

The jade template with the jquery mobile application:

!!! 5
html
    head
        meta(charset="utf-8")
        meta(name="viewport", content="width=device-width, initial-scale=1")
        title title
        link(rel='stylesheet', href='http://code.jquery.com/mobile/1.1.0/jquery.mobile-1.1.0.min.css')
        script(src='http://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js')
        script(src='http://code.jquery.com/mobile/1.1.0/jquery.mobile-1.1.0.min.js')
        script(type='text/javascript')
            $(document).bind('pageinit', function() {
                $('#toTop').tap(function() {
                    $.get('/action/#{key}/0/go', function() {});
                });
                $('#toBotton').tap(function() {
                    $.get('/action/#{key}/max/go', function() {});
                });
                $('#toUp').tap(function() {
                    $.get('/action/#{key}/100/rew', function() {});
                });
                $('#toDown').tap(function() {
                    $.get('/action/#{key}/100/ffd', function() {});
                });
            });
    body
        #page1(data-role="page")
            #header(data-theme="a", data-role="header")
                h3 Header
            #content(data-role="content")
                a(data-role="button", data-transition="fade", data-theme="a", href="#", id="toTop", data-icon="minus", data-iconpos="left") Top
                a(data-role="button", data-transition="fade", href="#", id="toUp", data-icon="arrow-u", data-iconpos="left") Up
                a(data-role="button", data-transition="fade", href="#", id="toDown", data-icon="arrow-d", data-iconpos="left") Down
                a(data-r

Our Html client page

<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
        "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
<head>
    <title>jQuery Smooth Scroll - Design Woop</title>
    <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
</head>
<body>
<p>
    Lorem ipsum ….. put a big lorem ipsum here to make possible the scroll
</p>
<script src="http://localhost:8080/socket.io/socket.io.js"></script>
<script src="client.js"></script>
</body>
</html>

And finally our client magic with the QR code and the functions to handle the node.js actions

var key = "secret";
function getDocHeight() {
    var D = document;
    return Math.max(
            Math.max(D.body.scrollHeight, D.documentElement.scrollHeight),
            Math.max(D.body.offsetHeight, D.documentElement.offsetHeight),
            Math.max(D.body.clientHeight, D.documentElement.clientHeight)
    );
}
var socket = io.connect('http://localhost:8080');
var y = 0;

socket.emit('setKey', key);
socket.on('scrollTo', function (data) {
    if (data.y == 'max') {
        y = getDocHeight();
    } else {
        if (data.action == 'ffd') {
            y += parseInt(data.y);
        } else if (data.action == 'go') {
            y = parseInt(data.y);
        } else {
            y -= parseInt(data.y);
        }
    }
    window.scrollTo(0, y);

});
document.write('<img src="https://chart.googleapis.com/chart?chs=150x150&cht=qr&chl=http://192.168.2.3:8000/panel/' + key + '&choe=UTF-8" alt=""/>');

You can see the code in github too. We also can see the script in action here:

We can also add more features to our application but that’s enought for this experiment. What do you think?

Asynchronous queries to PostgreSql database from the browser with node.js and socket.io

Normally we perform our database connection at server side with PHP and PDO for example. In this post I will show a simple technique to send queries to our database (PostgreSql in this example) asynchronously using node.js and socket.io.

The idea is pretty straightforward. We will send the SQL string and the values with a WebSocket and we will execute a callback in the client when the server (node.js script) fetches the recordset.

Our server:

var pg = require('pg');

var conString = "tcp://user:password@localhost:5432/db";
var client = new pg.Client(conString);
client.connect();

var io = require('socket.io').listen(8888);

io.sockets.on('connection', function (socket) {
    socket.on('sql', function (data) {
        var query = client.query(data.sql, data.values);
        query.on('row', function(row) {
            socket.emit('sql', row);
        });
    });
});

And our client:

<script src="http://localhost:8888/socket.io/socket.io.js"></script>
<script>
var socket = io.connect('http://vmserver:8888');

function sendSql(sql, values, cbk) {
    socket.emit('sql', { sql: sql, values : values});
    socket.on('sql', function(data){
        console.log(data);
    });
}
</script>    
<p>
<a href="#" onclick="sendSql('select * from users', [], function(data) {console.log(data);})">select * from users</a>
</p>
<p>
<a href="#" onclick="sendSql('select * from users where uid=$1', [4], function(data) {console.log(data);})">select * from users where uid=$1</a>
</p>

Simple, isn’t it?
You must take care if you use this script at production. Our database is exposed to raw SQL sent from the client. It’s a concept example. Maybe it would be better not to send the SQL. Store them into key-value table in the server and send only an ID from the browser.

What do you think?

Coding Katas, TDD and Katayunos

2011 is about to finish, and I want to speak about my way through the world of TDD. In the beginning of the year appeared a new cool project called 12meses12katas (12 months – 12 katas). The aim of the project was to propose one coding kata per month, and allow to the people to publish their implementation of the kata over github. In the line of this project a crew of crazy coders started another cool project called Katayunos. Katayunos is a small pun with the word Desayuno (Breakfast) and Coding Kata. It’s something similar than a code retreat. The purpose of the katayunos was to meet together in one place one saturday morning, have a breakfast and play with pair programming and TDD with one coding kata. Something too geek to explain to non-geek people, I known, but if you are reading this, It’s probable that your understand this ;). Our first katayuno was in the cafeteria of one Hotel. One cold Saturday morning (a really cold one believe me). The main problem was that we didn’t have any electrical plug, so our pomodoros were marked by the laptop batteries.

After this cold first katayuno we achieve new places with wifi and those kind of luxuries. Those kind of events are really interesting because we meet together different people and different programming languages. I still remember when I played with c# one time (I still have the sulfur smell in my fingers 🙂 ).

Basically the idea is:

  • Choose a coding kata
  • Pomodoros of 25 minutes
  • Pair programming (change the couples with each iteration)
  • Have fun

And now I’m going to show my 12 implementations of the 12 katas from 12meses12katas’s project:

Jaunary: String-Calculator
A good kata for beginners. That’s a good one because the testing strategy is very clear and we can focus on Testing: Red (test NOK) -> implement code -> Green (test OK).
My solution: php + phpunit

February: Roman-Numerals
I really hate roman numerals after doing this kata 😉
My solution: python + pyunit

March: FizzBuzz
A famous kata. A Really simple one. Because of that we can focus on the TDD (baby steps, refactoring, …)
My solution: php + phpunit
I also wrote a post with a crazy dynamic classes and the implementation of FizzBuzz kata with those dinamic classes:

April: Bowling
My problem was that I didn’t know the rules of Bowling so I needed to learn how to play Bowling first.
My solution: php + phpunit

May: KataLonja
A strange kata. I like it because it’s not a theoretical one. It’s close to a real problem.
My solution: php + phpunit

June: Simple-Lists
It was my first kata with javascript and node.js.
My solution: node.js + nodeunit (with John Resig’s class implementation)

July: Prime-Factors
Another famous one: Prime factors.
My solution: node.js + nodeunit

August: Minesweeper
An implementation of the famous computer game Minesweeper.
My solution: php + phpunit

October: Karate-Chop
Dual solution. With php and JavaScript (with node.js)
My solution: php + phpunit and node.js + nodeunit

November: KataPotter
This time I will use coffeescript instead of pure js
My solution: node.js + coffeeScript + jasmine

December: PomodoroKata
CoffeeScript again. One pomodoro is one pomodoro. Let’s build a pomodoro timer in a pomodoro.
My solution: node.js + coffeeScript + jasmine

So, what are you waiting for to create a local group and clone the katayuno in your city? It’s very easy: broadcast the event over twitter, pick a place, pick a kata and enjoy.

Regards, Gonzalo

Talk about node.js and WebSockets

Last friday I spoke about node.js and Websockets with the people of The Mêlée. The talk was an introduction to node.js and focused in the new HTML5 feature: the WebSockets.

When I spoke about Websockets I also introduced the great library socket.io. The jQuery of WebSockets.

Using node.js to store PHP sessions

We use sessions when we want to preserve certain data across subsequent accesses. PHP allows us to use different handlers when we’re using sessions. The default one is filesystem, but we can change it with session.save_handler in the php.ini. session.save_handler defines the name of the handler which is used for storing and retrieving data associated with a session. We also can create our own handler to manage sessions. In this post we’re going to create a custom handler to store sessions in a node.js service. Let’s start:

Imagine we’ve got the following php script:

session_start();

if (!isset($_SESSION["gonzalo"])) $_SESSION["gonzalo"] = 0;
$_SESSION["gonzalo"]++;
$_SESSION["arr"] = array('key' => uniqid());
var_dump($_SESSION);

A simple usage of sessions with PHP. If we reload the page our counter will be incremented by one. We’re using the default session handler. It works without any problem.

The idea is create a custom handler to use a server with node.js to store the session information instead of filesystem. To create custom handlers we need to use the PHP function: session_set_save_handler and rewrite the callbacks for: open, close, read, write, destroy and gc. PHP’s documentation is great. My proposal is the following one:

Our custom handler:

class NodeSession
{
    const NODE_DEF_HOST = '127.0.0.1';
    const NODE_DEF_PORT = 5672;

    static function start($host = self::NODE_DEF_HOST, $port = self::NODE_DEF_PORT)
    {
        $obj = new self($host, $port);
        session_set_save_handler(
            array($obj, "open"),
            array($obj, "close"),
            array($obj, "read"),
            array($obj, "write"),
            array($obj, "destroy"),
            array($obj, "gc"));
        session_start();
        return $obj;
    }

    private function unserializeSession($data)
    {
        if(  strlen( $data) == 0) {
            return array();
        }

        // match all the session keys and offsets
        preg_match_all('/(^|;|\})([a-zA-Z0-9_]+)\|/i', $data, $matchesarray, PREG_OFFSET_CAPTURE);
        $returnArray = array();

        $lastOffset = null;
        $currentKey = '';
        foreach ( $matchesarray[2] as $value ) {
            $offset = $value[1];
            if(!is_null( $lastOffset)) {
                $valueText = substr($data, $lastOffset, $offset - $lastOffset );
                $returnArray[$currentKey] = unserialize($valueText);
            }
            $currentKey = $value[0];

            $lastOffset = $offset + strlen( $currentKey )+1;
        }

        $valueText = substr($data, $lastOffset );
        $returnArray[$currentKey] = unserialize($valueText);

        return $returnArray;
    }
    
    function __construct($host = self::NODE_DEF_HOST, $port = self::NODE_DEF_PORT)
    {
        $this->_host = $host;
        $this->_port = $port;
    }

    function open($save_path, $session_name)
    {
        return true;
    }

    function close()
    {
        return true;
    }

    public function read($id)
    {
        return (string) $this->send(__FUNCTION__, array('id' => $id));
    }

    public function write($id, $data)
    {
        try {
            $this->send(__FUNCTION__, array(
                'id'       => $id,
                'data'     => $data,
                'time'     => time(),
                'dataJSON' => json_encode($this->unserializeSession($data))));
            return true;
        } catch (Exception $e) {
            return false;
        }
    }

    public function destroy($id)
    {
        try {
            $this->send(__FUNCTION__, array('id' => $id));
        } catch (Exception $e) {
            return false;
        }
         return true;
    }

    function gc($maxlifetime)
    {
        try {
            $this->send(__FUNCTION__, array('maxlifetime' => $maxlifetime, 'time' => time()));
        } catch (Exception $e) {
            return false;
        }
        return true;
    }

    private function send($action, $params)
    {
        $params = array('action' => $action) + $params;
        return file_get_contents("http://{$this->_host}:{$this->_port}?" . http_build_query($params));
    }
}

Our node.js server:

var http = require('http'),
    url  = require('url'),
    session = require('nodePhpSessions').SessionHandler;

var sessionHandler = new session();

var server = http.createServer(function (req, res) {
    var parsedUrl = url.parse(req.url, true).query;
    res.writeHead(200, {'Content-Type': 'text/plain'});
    res.end(sessionHandler.run(parsedUrl));
});

server.listen(5672, "127.0.0.1", function() {
  var address = server.address();
  console.log("opened server on %j", address);
});

As we can see we need the node.js module nodePhpSessions. You can easily install with:

npm install nodePhpSessions

You can see nodePhpSessions library here.

The library is tested with nodeunit. Without TDD is very hard to test things such as garbage collector.:

var session = require('nodePhpSessions').SessionHandler;
var sessionHandler = new session();
var parsedUrl;

exports["testReadUndefinedSession"] = function(test){
    parsedUrl = { action: 'read', id: 'ts49vmf0p732iafr25mdu8gvg2' };
    test.equal(sessionHandler.run(parsedUrl), undefined);
    test.done();
};

exports["oneSessionShouldReturns1"] = function(test){
    parsedUrl = {
        action: 'write',
        id: 'ts49vmf0p732iafr25mdu8gvg2',
        data: 'gonzalo|i:1;arr|a:1:{s:3:"key";s:13:"4e2b1a40d136a";}',
        time: '1311447616',
        dataJSON: '{"gonzalo":1,"arr":{"key":"4e2b1a40d136a"}}' };
    sessionHandler.run(parsedUrl);

    parsedUrl = { action: 'readAsArray', id: 'ts49vmf0p732iafr25mdu8gvg2' };
    test.equal(sessionHandler.run(parsedUrl).gonzalo, 1);
    test.done();
};

exports["oneSessionShouldReturns2"] = function(test){
    parsedUrl = {
        action: 'write',
        id: 'ts49vmf0p732iafr25mdu8gvg2',
        data: 'gonzalo|i:2;arr|a:1:{s:3:"key";s:13:"4e2b1a40d136a";}',
        time: '1311447616',
        dataJSON: '{"gonzalo":2,"arr":{"key":"4e2b1a40d136a"}}' };
    sessionHandler.run(parsedUrl);
    parsedUrl = { action: 'readAsArray', id: 'ts49vmf0p732iafr25mdu8gvg2' };
    test.equal(sessionHandler.run(parsedUrl).gonzalo, 2);
    test.done();
};

exports["destroySession"] = function(test){
    parsedUrl = {
        action: 'destroy',
        id: 'ts49vmf0p732iafr25mdu8gvg2'};
    sessionHandler.run(parsedUrl);

    parsedUrl = { action: 'readAsArray', id: 'ts49vmf0p732iafr25mdu8gvg2' };
    test.equal(sessionHandler.run(parsedUrl), undefined);

	test.done();
};

exports["garbageColector"] = function(test){
    parsedUrl = {
        action: 'write',
        id: 'session1',
        data: 'gonzalo|i:1;arr|a:1:{s:3:"key";s:13:"4e2b1a40d136a";}',
        time: '1111111200',
        dataJSON: '{"gonzalo":1,"arr":{"key":"4e2b1a40d136a"}}' };
    sessionHandler.run(parsedUrl);

    parsedUrl = {
        action: 'write',
        id: 'session2',
        data: 'gonzalo|i:1;arr|a:1:{s:3:"key";s:13:"4e2b1a40d136a";}',
        time: '1111111100',
        dataJSON: '{"gonzalo":1,"arr":{"key":"4e2b1a40d136a"}}' };
    sessionHandler.run(parsedUrl);

    parsedUrl = { action: 'gc', maxlifetime: '100', time: '1111111210'};
    sessionHandler.run(parsedUrl);

    parsedUrl = { action: 'readAsArray', id: 'session2' };
    test.equal(sessionHandler.run(parsedUrl), undefined);

    parsedUrl = { action: 'readAsArray', id: 'session1' };
    test.equal(sessionHandler.run(parsedUrl).gonzalo, 1);

    test.done();
};

Here you can see the output of the tests:

nodeunit testNodeSessions.js 

testNodeSessions.js
✔ testReadUndefinedSession
✔ oneSessionShouldReturns1
✔ oneSessionShouldReturns2
✔ destroySession
✔ garbageColector

OK: 6 assertions (5ms)

Now we change the original PHP script to:

include_once 'NodeSessions.php';
NodeSession::start();

if (!isset($_SESSION["gonzalo"])) $_SESSION["gonzalo"] = 0;
$_SESSION["gonzalo"]++;
$_SESSION["arr"] = array('key' => uniqid());
var_dump($_SESSION);

We start the node.js server:

node serverSessions.js 

Now if we reload our script in the browser we will see the same behaviour, but now our sessions are stored in the node.js server.

array(2) {
  ["gonzalo"]=>
  int(16)
  ["arr"]=>
  array(1) {
    ["key"]=>
    string(13) "4e2a9f6a966f4"
  }
}

This kind of techniques are good when clustering PHP applications.

Full code is available on github (node server, PHP handler, tests and examples) here.

Real time notifications (part II). Now with node.js and socket.io

In one of my previous posts I wrote about Real time notifications with PHP. I wanted to create a simple comet system fully written in PHP and JavaScript. It worked but as Scott Mattocks told me in a comment this implementation was still just doing short polling. The performance with this solution may be bad in a medium/hight traffic site. This days I’m playing with node.js and I want to create a simple test. I want to do exactly the same than the previous post but now with node.js instead of my PHP+js test. Let’s start

Now I want to use socket.io instead of pure web-sockets like my previous posts about node.js. For those who don’t know, socket.io is amazing library that allows us to use real-time technologies within every browsers (yes even with IE6). It uses one technology or another depending on the browser we are using, with the same interface for the developer. That’s means if we’re using Google Chrome we will use websockets, but if our browser does’t support them, socket.io will choose another supported transports. Definitely socket.io is the jQuery of the websockets. The supporter transports are:

  • WebSocket
  • Adobe Flash Socket
  • AJAX long polling
  • AJAX multipart streaming
  • Forever Iframe
  • JSONP Polling

First we create our node.js server. A really simple one.

var http = require('http');
var io = require('socket.io');

server = http.createServer(function(req, res){
});
server.listen(8080);

// socket.io 
var socket = io.listen(server);

socket.on('connection', function(client){
  client.on('message', function(msg){
      socket.broadcast(msg);
  })
}); 

This server will broadcast the message received from the browser to all connected clients.

Our HTML page will look like that:

<html>
    <head>
        <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
        <title>Comet Test</title>
    </head>
    <body>
        <p><a id='customAlert' href="#" onclick='socket.send("customAlert")'>publish customAlert</a></p>
        <p><a id='customAlert2' href="#" onclick='socket.send("customAlert2")'>publish customAlert2</a></p>
        <script src="http://localhost:8080/socket.io/socket.io.js" type="text/javascript"></script>
        <script type="text/javascript">

// Start the socket
var socket = new io.Socket(null, {port: 8080});
socket.connect();

socket.on('message', function(msg){
    console.log(msg);
});
        </script>
    </body>
</html>

As we can see we are including the js script called socket.io/socket.io.js. This script is served by our node server.

In fact we can use our node.js to serve everything (HTML, js, CSS) but in our example we will use only node.js for real-time stuff. Apache will serve the rest of the code (only a HTML file in this case).

And that’s all. Those few lines perform the same thing than our PHP and js code in the other post’s example. Our node.js implementation is definitely smarter than the PHP one. The socket.io library also allows us to use the example with all browser. Same code and without any browser nightmare (just like jQuery when we work with DOM).

Here I have a little screencast with the working example. As we will see there We will connect to the node server with firefox and chrome. Firefox will use xhr multipart and Chrome will use Websokets.

Another important issue of socket.io library is that we forget about the reconnection to the web-socket server, if something wrong happens (as we can see in Real time monitoring PHP applications with web-sockets and node.js). If we use raw WebSocket implementations and our connection with the web-socket server crashes or if we stop the server, our application will raise disconnect event and we need to create something to reconnect to the server. socket.io does it for us. With our small piece of JavaScript code we will get a high performance real-time applicatrion. Node is cool. Really cool. Kinda wierd at the beginning but the learning effort will be worthwhile. A few js lines and a real time applications.

I’ve got a problem within our node.js application. If we’ve got some kind of security within our application (imagine for example it’s behind a session based auth form) we need to share this security layer with our node.js server to ensure that non authenticated users aren’t allowed to use our websockets. I don’t know how to do it just now, but I’m investigating. Do you have any idea?

Full code Code at github. Ensure you’re using the stable version of node.js. With the last versión available on github of node.js there’s a bug and server dies when we connect with Google Chrome.

Web console with node.js

Continuing with my experiments of node.js, this time I want to create a Web console. The idea is simple. I want to send a few command to the server and I display the output inside the browser. I can do it entirely with PHP but I want to send the output to the browser as fast as they appear without waiting for the end of the command. OK we can do it flushing the output in the server but this solution normally crashes if we keep the application open for a long time. WebSockets again to the rescue. If we need a cross-browser implementation we need the socket.io library. Let’s start:

The node server is a simple websocket server. In this example we will launch each command with spawn function (require(‘child_process’).spawn) and send the output within the websoket. Simple and pretty straightforward.

var sys   = require('sys'),
http  = require('http'),
url   = require('url'),
spawn = require('child_process').spawn,
ws    = require('./ws.js');

var availableComands = ['ls', 'ps', 'uptime', 'tail', 'cat'];
ws.createServer(function(websocket) {
    websocket.on('connect', function(resource) {
        var parsedUrl = url.parse(resource, true);
        var rawCmd = parsedUrl.query.cmd;
        var cmd = rawCmd.split(" ");
        if (cmd[0] == 'help') {
            websocket.write("Available comands: \n");
            for (i=0;i<availableComands.length;i++) {
                websocket.write(availableComands[i]);
                if (i< availableComands.length - 1) {
                    websocket.write(", ");
                }
            }
            websocket.write("\n");

            websocket.end();
        } else if (availableComands.indexOf(cmd[0]) >= 0) {
            if (cmd.length > 1) {
                options = cmd.slice(1);
            } else {
                options = [];
            }
            
            try {
                var process = spawn(cmd[0], options);
            } catch(err) {
                console.log(err);
                websocket.write("ERROR");
            }

            websocket.on('end', function() {
                process.kill();
            });

            process.stdout.on('data', function(data) {
                websocket.write(data);
            });

            process.stdout.on('end', function() {
                websocket.end();
            });
        } else {
             websocket.write("Comand not available. Type help for available comands\n");
             websocket.end();
        }
    });
  
}).listen(8880, '127.0.0.1');

The web client is similar than the example of my previous post

var timeout = 5000;
var wsServer = '127.0.0.1:8880';

var ws;


function cleanString(string) {
    return string.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/>/g,"&gt;");
}


function pad(n) {
    return ("0" + n).slice(-2);
}

var cmdHistory = [];
function send(msg) {
    if (msg == 'clear') {
        $('#log').html('');
        return;
    }
    try {
        ws = new WebSocket('ws://' + wsServer + '?cmd=' + msg);
        $('#toolbar').css('background', '#933');
        $('#socketStatus').html("working ... [<a href='#' onClick='quit()'>X</a>]");
        cmdHistory.push(msg);
        $('#log').append("<div class='cmd'>" + msg + "</div>");
        console.log("startWs:");
    } catch (err) {
        console.log(err);
        setTimeout(startWs, timeout);
    }

    ws.onmessage = function(event) {
        $('#log').append(util.toStaticHTML(event.data));
        window.scrollBy(0, 100000000000000000);
    };

    ws.onclose = function(){
        //console.log("ws.onclose");
        $('#toolbar').css('background', '#65A33F');
        $('#socketStatus').html('Type your comand:');
    }
}

function quit() {
    ws.close();
    window.scrollBy(0, 100000000000000000);
}
util = {
  urlRE: /https?:\/\/([-\w\.]+)+(:\d+)?(\/([^\s]*(\?\S+)?)?)?/g, 

  //  html sanitizer 
  toStaticHTML: function(inputHtml) {
    inputHtml = inputHtml.toString();
    return inputHtml.replace(/&/g, "&amp;")
                    .replace(/</g, "&lt;")
                    .replace("/n", "<br/>")
                    .replace(/>/g, "&gt;");
  }, 

  //pads n with zeros on the left,
  //digits is minimum length of output
  //zeroPad(3, 5); returns "005"
  //zeroPad(2, 500); returns "500"
  zeroPad: function (digits, n) {
    n = n.toString();
    while (n.length < digits) 
      n = '0' + n;
    return n;
  },

  //it is almost 8 o'clock PM here
  //timeString(new Date); returns "19:49"
  timeString: function (date) {
    var minutes = date.getMinutes().toString();
    var hours = date.getHours().toString();
    return this.zeroPad(2, hours) + ":" + this.zeroPad(2, minutes);
  },

  //does the argument only contain whitespace?
  isBlank: function(text) {
    var blank = /^\s*$/;
    return (text.match(blank) !== null);
  }
};
$(document).ready(function() {
  //submit new messages when the user hits enter if the message isnt blank
  $("#entry").keypress(function (e) {
    console.log(e);
    if (e.keyCode != 13 /* Return */) return;
    var msg = $("#entry").attr("value").replace("\n", "");
    if (!util.isBlank(msg)) send(msg);
    $("#entry").attr("value", ""); // clear the entry field.
  });
});

And that’s all. In fact we don’t need any line of PHP to perform this web console. Last year I tried to do something similar with PHP but it was a big mess. With node those kind of jobs are trivial. I don’t know if node.js is the future or is just another hype, but it’s easy. And cool. Really cool.

You can see the full code at Github here. Anyway you must take care if you run this application on your host. You are letting user to execute raw unix commands. A bit of security layer would be necessary.

Real time monitoring PHP applications with websockets and node.js

The inspection of the error logs is a common way to detect errors and bugs. We also can show errors on-screen within our developement server, or we even can use great tools like firePHP to show our PHP errors and warnings inside our firebug console. That’s cool, but we only can see our session errors/warnings. If we want to see another’s errors we need to inspect the error log. tail -f is our friend, but we need to surf against all the warnings of all sessions to see our desired ones. Because of that I want to build a tool to monitor my PHP applications in real-time. Let’s start:

What’s the idea? The idea is catch all PHP’s errors and warnings at run time and send them to a node.js HTTP server. This server will work similar than a chat server but our clients will only be able to read the server’s logs. Basically the applications have three parts: the node.js server, the web client (html5) and the server part (PHP). Let me explain a bit each part:

The node Server

Basically it has two parts: a http server to handle the PHP errors/warnings and a websocket server to manage the realtime communications with the browser. When I say that I’m using websockets that’s means the web client will only work with a browser with websocket support like chrome. Anyway it’s pretty straightforward swap from a websocket sever to a socket.io server to use it with every browser. But websockets seems to be the future, so I will use websockets in this example.

The http server:

http.createServer(function (req, res) {
    var remoteAdrress = req.socket.remoteAddress;
    if (allowedIP.indexOf(remoteAdrress) >= 0) {
        res.writeHead(200, {
            'Content-Type': 'text/plain'
        });
        res.end('Ok\n');
        try {
            var parsedUrl = url.parse(req.url, true);
            var type = parsedUrl.query.type;
            var logString = parsedUrl.query.logString;
            var ip = eval(parsedUrl.query.logString)[0];
            if (inspectingUrl == "" ||  inspectingUrl == ip) {
                clients.forEach(function(client) {
                    client.write(logString);
                });
            }
        } catch(err) {
            console.log("500 to " + remoteAdrress);
            res.writeHead(500, {
                'Content-Type': 'text/plain'
            });
            res.end('System Error\n');
        }
    } else {
        console.log("401 to " + remoteAdrress);
        res.writeHead(401, {
            'Content-Type': 'text/plain'
        });
        res.end('Not Authorized\n');
    }
}).listen(httpConf.port, httpConf.host);

and the web socket server:

var inspectingUrl = undefined;

ws.createServer(function(websocket) {
    websocket.on('connect', function(resource) {
        var parsedUrl = url.parse(resource, true);
        inspectingUrl = parsedUrl.query.ip;
        clients.push(websocket);
    });

    websocket.on('close', function() {
        var pos = clients.indexOf(websocket);
        if (pos >= 0) {
            clients.splice(pos, 1);
        }
    });

}).listen(wsConf.port, wsConf.host);

If you want to know more about node.js and see more examples, have a look to the great site: http://nodetuts.com/. In this site Pedro Teixeira will show examples and node.js tutorials. In fact my node.js http + websoket server is a mix of two tutorials from this site.

The web client.

The web client is a simple websockets application. We will handle the websockets connection, reconnect if it dies and a bit more. I’s based on node.js chat demo

<?php $ip = filter_input(INPUT_GET, 'ip', FILTER_SANITIZE_STRING); ?>

        Real time <?= $ip ?> monitor
<script type="text/javascript" src="http://ajax.googleapis.com/ajax/libs/jquery/1.4.3/jquery.min.js"></script><script type="text/javascript">// <![CDATA[
            selectedIp = '<?= $ip ?>';

// ]]></script>
<script type="text/javascript" src="js.js"></script>
</pre>
<div id="toolbar">
<ul id="status">
	<li>Socket status: <span id="socketStatus">Conecting ...</span></li>
	<li>IP: <!--?= $ip == '' ? 'all' : $ip . " <a href='?ip='-->[all]" ?></li>
	<li>count: <span id="count">0</span></li>
</ul>
</div>
<pre>


And the javascript magic

var timeout = 5000;
var wsServer = '192.168.2.2:8880';
var unread = 0;
var focus = false;

var count = 0;
function updateCount() {
    count++;
    $("#count").text(count);
}

function cleanString(string) {
    return string.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/>/g,"&gt;");
}

function updateUptime () {
    var now = new Date();
    $("#uptime").text(now.toRelativeTime());
}

function updateTitle(){
    if (unread) {
        document.title = "(" + unread.toString() + ") Real time " + selectedIp + " monitor";
    } else {
        document.title = "Real time " + selectedIp + " monitor";
    }
}

function pad(n) {
    return ("0" + n).slice(-2);
}

function startWs(ip) {
    try {
        ws = new WebSocket("ws://" + wsServer + "?ip=" + ip);
        $('#toolbar').css('background', '#65A33F');
        $('#socketStatus').html('Connected to ' + wsServer);
        //console.log("startWs:" + ip);
        //listen for browser events so we know to update the document title
        $(window).bind("blur", function() {
            focus = false;
            updateTitle();
        });

        $(window).bind("focus", function() {
            focus = true;
            unread = 0;
            updateTitle();
        });
    } catch (err) {
        //console.log(err);
        setTimeout(startWs, timeout);
    }

    ws.onmessage = function(event) {
        unread++;
        updateTitle();
        var now = new Date();
        var hh = pad(now.getHours());
        var mm = pad(now.getMinutes());
        var ss = pad(now.getSeconds());

        var timeMark = '[' + hh + ':' + mm + ':' + ss + '] ';
        logString = eval(event.data);
        var host = logString[0];
        var line = "<table class='message'><tr><td width='1%' class='date'>" + timeMark + "</td><td width='1%' valign='top' class='host'><a href=?ip=" + host + ">" + host + "</a></td>";
        line += "<td class='msg-text' width='98%'>" + logString[1]; + "</td></tr>";
        if (logString[2]) {
            line += "<tr><td>&nbsp;</td><td colspan='3' class='msg-text'>" + logString[2] + "</td></tr>";
        }

        $('#log').append(line);
        updateCount();
        window.scrollBy(0, 100000000000000000);
    };

    ws.onclose = function(){
        //console.log("ws.onclose");
        $('#toolbar').css('background', '#933');
        $('#socketStatus').html('Disconected');
        setTimeout(function() {startWs(selectedIp)}, timeout);
    }
}

$(document).ready(function() {
    startWs(selectedIp);
});

The server part:

The server part will handle silently all PHP warnings and errors and it will send them to the node server. The idea is to place a minimal PHP line of code at the beginning of the application that we want to monitor. Imagine the following piece of PHP code

$a = $var[1];
$a = 1/0;
class Dummy
{
    static function err()
    {
        throw new Exception("error");
    }
}
Dummy1::err();

it will throw:
A notice: Undefined variable: var
A warning: Division by zero
An Uncaught exception ‘Exception’ with message ‘error’

So we will add our small library to catch those errors and send them to the node server

include('client/NodeLog.php');
NodeLog::init('192.168.2.2');

$a = $var[1];
$a = 1/0;
class Dummy
{
    static function err()
    {
        throw new Exception("error");
    }
}
Dummy1::err();

The script will work in the same way than the fist version but if we start our node.js server in a console:

$ node server.js
HTTP server started at 192.168.2.2::5672
Web Socket server started at 192.168.2.2::8880

We will see those errors/warnings in real-time when we start our browser

Here we can see a small screencast with the working application:

This is the server side library:

class NodeLog
{
    const NODE_DEF_HOST = '127.0.0.1';
    const NODE_DEF_PORT = 5672;

    private $_host;
    private $_port;

    /**
     * @param String $host
     * @param Integer $port
     * @return NodeLog
     */
    static function connect($host = null, $port = null)
    {
        return new self(is_null($host) ? self::$_defHost : $host, is_null($port) ? self::$_defPort : $port);
    }

    function __construct($host, $port)
    {
        $this->_host = $host;
        $this->_port = $port;
    }

    /**
     * @param String $log
     * @return Array array($status, $response)
     */
    public function log($log)
    {
        list($status, $response) = $this->send(json_encode($log));
        return array($status, $response);
    }

    private function send($log)
    {
        $url = "http://{$this->_host}:{$this->_port}?logString=" . urlencode($log);
        $ch = curl_init();
        curl_setopt($ch, CURLOPT_URL, $url);
        curl_setopt($ch, CURLOPT_NOBODY, true);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);

        $response = curl_exec($ch);
        $status   = curl_getinfo($ch, CURLINFO_HTTP_CODE);
        curl_close($ch);

        return array($status, $response);
    }

    static function getip() {
        $realip = '0.0.0.0';
        if ($_SERVER) {
            if ( isset($_SERVER['HTTP_X_FORWARDED_FOR']) && $_SERVER['HTTP_X_FORWARDED_FOR'] ) {
                $realip = $_SERVER["HTTP_X_FORWARDED_FOR"];
            } elseif ( isset($_SERVER['HTTP_CLIENT_IP']) && $_SERVER["HTTP_CLIENT_IP"] ) {
                $realip = $_SERVER["HTTP_CLIENT_IP"];
            } else {
                $realip = $_SERVER["REMOTE_ADDR"];
            }
        } else {
            if ( getenv('HTTP_X_FORWARDED_FOR') ) {
                $realip = getenv('HTTP_X_FORWARDED_FOR');
            } elseif ( getenv('HTTP_CLIENT_IP') ) {
                $realip = getenv('HTTP_CLIENT_IP');
            } else {
                $realip = getenv('REMOTE_ADDR');
            }
        }
        return $realip;
    }

    public static function getErrorName($err)
    {
        $errors = array(
            E_ERROR             => 'ERROR',
            E_RECOVERABLE_ERROR => 'RECOVERABLE_ERROR',
            E_WARNING           => 'WARNING',
            E_PARSE             => 'PARSE',
            E_NOTICE            => 'NOTICE',
            E_STRICT            => 'STRICT',
            E_DEPRECATED        => 'DEPRECATED',
            E_CORE_ERROR        => 'CORE_ERROR',
            E_CORE_WARNING      => 'CORE_WARNING',
            E_COMPILE_ERROR     => 'COMPILE_ERROR',
            E_COMPILE_WARNING   => 'COMPILE_WARNING',
            E_USER_ERROR        => 'USER_ERROR',
            E_USER_WARNING      => 'USER_WARNING',
            E_USER_NOTICE       => 'USER_NOTICE',
            E_USER_DEPRECATED   => 'USER_DEPRECATED',
        );
        return $errors[$err];
    }

    private static function set_error_handler($nodeHost, $nodePort)
    {
        set_error_handler(function ($errno, $errstr, $errfile, $errline) use($nodeHost, $nodePort) {
            $err = NodeLog::getErrorName($errno);
            /*
            if (!(error_reporting() & $errno)) {
                // This error code is not included in error_reporting
                return;
            }
            */
            $log = array(
                NodeLog::getip(),
                "<strong class="{$err}">{$err}</strong> {$errfile}:{$errline}",
                nl2br($errstr)
            );
            NodeLog::connect($nodeHost, $nodePort)->log($log);
            return false;
        });
    }

    private static function register_exceptionHandler($nodeHost, $nodePort)
    {
        set_exception_handler(function($exception) use($nodeHost, $nodePort) {
            $exceptionName = get_class($exception);
            $message = $exception->getMessage();
            $file = $exception->getFile();
            $line = $exception->getLine();
            $trace = $exception->getTraceAsString();

            $msg = count($trace) > 0 ? "Stack trace:\n{$trace}" : null;
            $log = array(
                NodeLog::getip(),
                nl2br("<strong class="ERROR">Uncaught exception '{$exceptionName}'</strong> with message '{$message}' in {$file}:{$line}"),
                nl2br($msg)
            );
            NodeLog::connect($nodeHost, $nodePort)->log($log);
            return false;
        });
    }

    private static function register_shutdown_function($nodeHost, $nodePort)
    {
        register_shutdown_function(function() use($nodeHost, $nodePort) {
            $error = error_get_last();

            if ($error['type'] == E_ERROR) {
                $err = NodeLog::getErrorName($error['type']);
                $log = array(
                    NodeLog::getip(),
                    "<strong class="{$err}">{$err}</strong> {$error['file']}:{$error['line']}",
                    nl2br($error['message'])
                );
                NodeLog::connect($nodeHost, $nodePort)->log($log);
            }
            echo NodeLog::connect($nodeHost, $nodePort)->end();
        });
    }

    private static $_defHost = self::NODE_DEF_HOST;
    private static $_defPort = self::NODE_DEF_PORT;

    /**
     * @param String $host
     * @param Integer $port
     * @return NodeLog
     */
    public static function init($host = self::NODE_DEF_HOST, $port = self::NODE_DEF_PORT)
    {
        self::$_defHost = $host;
        self::$_defPort = $port;

        self::register_exceptionHandler($host, $port);
        self::set_error_handler($host, $port);
        self::register_shutdown_function($host, $port);

        $node = self::connect($host, $port);
        $node->start();
        return $node;
    }

    private static $time;
    private static $mem;

    public function start()
    {
        self::$time = microtime(TRUE);
        self::$mem = memory_get_usage();
        $log = array(NodeLog::getip(), "<strong class="OK">Start</strong> >>>> {$_SERVER['REQUEST_URI']}");
        $this->log($log);
    }

    public function end()
    {
        $mem = (memory_get_usage() - self::$mem) / (1024 * 1024);
        $time = microtime(TRUE) - self::$time;
        $log = array(NodeLog::getip(), "<strong class="OK">End</strong> <<<< mem: {$mem} time {$time}");         $this->log($log);
    }
}

And of course the full code on gitHub: RealTimeMonitor