Here is just a handy collection of my current javascript stuff.
Javascript Links
Basic Javascript
A Re-introduction to JavaScript
Reference Material
EMCA 262 (EMCAScript 5.1) Specs
Essential Blogs
Books
Secrets of a JavaScript Ninja (John Resig)
Pro JavaScript Design Patterns
Classical Inheritance Projects
Tools
If you’re using Socket.io and want to reverse proxy your web socket connections, you’ll quickly find it’s somewhat difficult. Since web sockets are done over HTTP 1.1 connections (where the handshake and upgrade are completed), your backend needs to support HTTP 1.1, and from what I have researched, they break the HTTP 1.0 spec (see this discussion at stackoverflow).
Some people have been able to get HAProxy to work for effective proxying of websocket connections, but I couldn’t get this to work reliably when I tried the latest version and TCP mode.
If you’re using nginx, you won’t be able to proxy web socket connections using the standard nginx proxy_pass directives. Fortunately, Weibin Yao has developed a tcp proxy module for nginx that allows you to reverse proxy general tcp connections, especially well suited for websockets.
Let’s get a simple web socket proxy up and running. Download nginx sources and tcp_proxy module:
export NGINX_VERSION=1.0.4
curl -O http://nginx.org/download/nginx-$NGINX_VERSION.tar.gz
git clone https://github.com/yaoweibin/nginx_tcp_proxy_module.git
tar -xvzf nginx-$NGINX_VERSION.tar.gz
cd nginx-$NGINX_VERSION
patch -p1 < ../nginx_tcp_proxy_module/tcp.patch
./configure --add-module=../nginx_tcp_proxy_module/
sudo make && make install
Create a simple vhost like the following (note tcp must be defined at the server directive level):
tcp {
upstream websockets {
## node processes
server 127.0.0.1:8001;
server 127.0.0.1:8002;
server 127.0.0.1:8003;
server 127.0.0.1:8004;
check interval=3000 rise=2 fall=5 timeout=1000;
}
server {
listen 127.0.0.1:80;
server_name _;
tcp_nodelay on;
proxy_pass websockets;
}
}
http {
## status check page for websockets
server {
listen 9000;
location /websocket_status {
check_status;
}
}
}
You’ll notice there is an additional http section, with a check_status directive. The tcp_proxy module provides a simple and convenient status check page which you can use to see if your backend node processes are up:
var http = require('http'), io = require('socket.io');
var server = http.createServer(function(req, res){
res.writeHead(200, {'Content-Type': 'text/html'});
res.end('Hello world');
});
server.listen(process.argv[2]);
// socket.io
var socket = io.listen(server);
socket.on('connection', function(client){
// new client is here!
console.log('client has connected');
client.on('message', function(){ })
});
Start four node processes, each listening on different ports:
node ./websocket-server.js 8001 &
node ./websocket-server.js 8002 &
node ./websocket-server.js 8003 &
node ./websocket-server.js 8004 &
You should now check your status page to verify all backends are up and running:
You can also go to our proxy via standard http (web browser) and correctly see “Hello World” to verify we’re hitting one of our node backends.
Now lets create a simple web socket client to test if we can actually create a websocket connection over the proxy:
<html>
<head>
<title>Websockets Proxy Test</title>
<script type="text/javascript" src="sio.js"></script>
<script type="text/javascript">
var socket = new io.Socket('ws://localhost');
socket.connect();
socket.on('connect', function() {
console.log('connected!');
});
</script>
</head>
<body>
<h1>Websockets Proxy Test</h1>
</body>
</html>
For simplicity, I used a node static page server to serve this test page from the same node process and attached my Socket.IO instance to it:
var io = require('./Socket.IO-node');
var http = require("http"),
url = require("url"),
path = require("path"),
fs = require("fs"),
port = process.argv[2] || 8888;
var server = http.createServer(function(request, response) {
var uri = url.parse(request.url).pathname
, filename = path.join(process.cwd(), uri);
path.exists(filename, function(exists) {
if(!exists) {
response.writeHead(404, {"Content-Type": "text/plain"});
response.write("404 Not Found\n");
response.end();
return;
}
if (fs.statSync(filename).isDirectory()) filename += '/index.html';
fs.readFile(filename, "binary", function(err, file) {
if(err) {
response.writeHead(500, {"Content-Type": "text/plain"});
response.write(err + "\n");
response.end();
return;
}
response.writeHead(200);
response.write(file, "binary");
response.end();
});
});
server.listen(parseInt(port, 10));
// socket.io
var socket = io.listen(server);
Now, when we run our node instances:
node ./websocket-server.js 8001
Static file server running at
=> http://localhost:8001/
CTRL + C to shutdown
info - socket.io started
And when we go to http://localhost, we can see successful websocket handshakes (which means we’re going over the nginx proxy):
debug - client authorized
info - handshake authorized
info - handshaken 5ec456d53d8dc0b43f61b5f3acdf8b8e
Using this method you can successfully balance a cluster of web socket nodes, with some simple failure provided by the tcp_proxy module. Depending on the need and placement of nginx within your specific application stack, the ability to use this instead of something like HAProxy or some other balancer strategy could allow scaling many more connections per server without introducing significant complexity.
One thing that should be noted is you can no longer guarantee that a client will always connect to the same node process, like in the event of disconnection, so your application will need to understand this and implement session management across the cluster (such as using a redis or memcache session store, for example).
Happy hacking!
What happens when you have half a million addresses and need to find out where in the world they actually are? You geocode them! Which isn’t the easiest thing to do — the difficulty increasing as your need for performance, quality and quantity goes up.
Geocoding Options
The first, and probably the most obvious, is to simply use an external service. There are plenty to choose from, with many major companies offering up their Geocoder API for your personal enjoyment (or dis-enjoyment). So pick your poison – Google, Yahoo, Bing all have nice API’s that will happily take an address and give you something that looks like a lat and long.
Of course, can you really make 500,000 web requests and no one be the wiser? Of course not. There are limitations, both on the service and potentially on your process to make these requests. For one, it’s pretty expensive for most API’s I looked at to purchase a commercial license suitable for bulk geocoding. But it’s also slow. And in slow I mean it takes time to make 500,000 web requests, wait for them to complete, and put the data some place. That’s not counting the fact you’ll need to somehow address failures, which probably means some sort of queue/worker system, that’ll need supervision, management, et al. Yeah, not fun. So even if you manage to find an awesome deal on 500k web requests to someone’s API, you still need to manage the carrier pigeon effect. At least until connections get faster and it suddenly becomes cheap to buy expensive online data and services.
It’s also worth mentioning that many of these providers rely on the same sources of data that are freely available. In my tests, there were only marginal differences in quality across various solutions (both commercial API, private sources of data and home-brew systems). The largest factor in geocoding appears to be address quality first, and your data source second (and how well your parser can understand bad addresses). Soundex helps with address quality, but some addresses data is just bad and I suspect Google and others use some form of dynamic analysis based upon past searches to obtain higher quality and suggestions for incorrect cities, zip codes, etc. Here’s an interesting report comparing various geocoding options and the effectiveness of each based upon testing with 100 addresses (pay close attention to shapefile based data sources).
With these facts of life in mind, I decided what any crazy person would — setup my own! Google be damned. I can do it, and in one week. Well, maybe two. I know what you’re thinking, pretty foolish and maybe even a little masochistic, and you’d be right on both counts.
Fortunately, PostGIS has in it’s latest version, buried under extras a quite comprehensive geocoder project based upon freely available US Census Shapefile data. I’ll walk you through setting up your own geocoder, complete with soundex support and ability to bulk geocode from a db table.
Download & Setup Postgres
I decided to compile Postgres from source, but most distributions also have a package available and Postgres maintains binary and graphical installer packages for some. To compile:
You’ll need build-essential, gcc, etc. packages to compile
export POSTGRES_VERSION=9.0.4
wget http://wwwmaster.postgresql.org/redir/180/f/source/v$POSTGRES_VERSION/postgresql-$POSTGRES_VERSION.tar.gz
tar -xvzf postgresql-$POSTGRES_VERSION.tar.gz
cd postgresql-$POSTGRES_VERSION
./configure --prefix=/usr/local
make
sudo make install
You’ll now need to compile and install PostGIS. Before you do this, it’s a good idea to spend some time setting up accounts on Postgres. The authentication model for Postgres relies on system accounts, specifically allowing root access to the postgres user by default, where you can administer other users as necessary. For my setup, I made sure to have a system account for every postgres user.
Create the postgres user:
sudo adduser postgres
You now need to create a directory to store the main Postgres DB files. I put mine under /usr/local, but you can put it generally anywhere as needed (the path is specified in configuration and/or during server startup).
sudo mkdir /usr/local/pgsql/data
sudo chown postgres /usr/local/pgsql/data
sudo chgrp postgres /usr/local/pgsql/data
Now you’ll need to create a postgres cluster and start the postmaster process. First, change to the postgres user:
su postgres
/usr/local/pgsql/bin/initdb -D /usr/local/pgsql/data
/usr/local/pgsql/bin/pg_ctl -D /usr/local/pgsql/data -l /var/log/postgres start
You can now login using the psql client (as the postgres system user):
/usr/local/pgsql/bin/psql
If all is well, you should be at a postgres command prompt like the following:
psql (9.0.4)
Type "help" for help.
postgres=#
To quit, type “\q”. Now we’re ready to install and setup PostGIS!
Note: You’ll need libxml2-dev, libgeos-dev and libproj-dev (xslt and convert are also needed for documentation). I had to manually compile and install geos (which you can get from http://trac.osgeo.org/geos/).
Since we’re interested in the Geocoder from 2.0, we’ll need to grab the latest development snapshot:
wget http://postgis.refractions.net/download/postgis-2.0.0SVN.tar.gz
tar -xvzf postgis-2.0.0SVN.tar.gz
./configure --with-pgconfig=/usr/local/pgsql/bin/pg_config
make
make install
ldconfig
Create a spatial database template
So we can easily create spatially enabled databases, we’ll create one as a template and apply the spatial data types to it:
su postgis
/usr/local/pgsql/bin/createdb spatial_db_template
/usr/local/pgsql/bin/createlang plpgsql spatial_db_template
/usr/local/pgsql/bin/psql -d spatial_db -f /usr/local/pgsql/share/contrib/postgis-2.0/postgis.sql
/usr/local/pgsql/bin/psql -d spatial_db -f /usr/local/pgsql/share/contrib/postgis-2.0/spatial_ref_sys.sql
Now to create a new postgis enabled spatial database, we just do the following:
/usr/local/pgsql/bin/createdb -T spatial_db_template tiger
Add soundex ability
To add soundex ability to the db (used in matching parsed address parts which includes soundex, metaphone and levenshtein abilities), first build the module under the contrib directory of your postgres source:
cd postgresql-$POSTGRES_VERSION
cd contrib/fuzzystrmatch
sudo make && make install
This will add the fuzzystrmatch under the contrib folder of your installation. To add it to your database:
/usr/local/pgsql/bin/psql -d tiger -f /usr/local/pgsql/share/contrib/fuzzystrmatch.sql
Create a geocoder database
Now, we’ll need to create a new geocoder database:
/usr/local/pgsql/bin/createdb -T spatial_db_template geocoder
/usr/local/pgsql/bin/psql -d tiger -f /usr/local/pgsql/share/contrib/fuzzystrmatch.sql
/usr/local/pgsql/bin/psql -d geocoder
CREATE SCHEMA tiger;
CREATE SCHEMA tiger_data;
ALTER DATABASE geocoder SET search_path=public, tiger;
Now, we can begin to setup our geocoder. We first need to populate some street abbreviation and state lookup tables, and the tiger_loader.sql, which is the actual code to load tiger data into the geocoder table structure:
(I had my PostGIS sources under /tmp in this case):
/usr/local/pgsql/bin/psql -d geocoder -f /tmp/postgis-2.0.0SVN/extras/tiger_geocoder/tiger_2010/tables/lookup_tables_2010.sql
/usr/local/pgsql/bin/psql -d geocoder -f /tmp/postgis-2.0.0SVN/extras/tiger_geocoder/tiger_2010/tiger_loader.sql
Next, create the actual geocode function in the db (note you need to be in the tiger_2010 directory):
/usr/local/pgsql/bin/psql -d geocoder -f ./create_geocode.sql
You’re now ready to load US Census shapefile data into the geocoder! There’s a slightly contrived way of generated scripts from the db, but I found this cumbersome for loading many states, so I wrote my own with perl. Someone from the PostGIS mailing list also wrote their own:
https://gist.github.com/885803#file_tiger_postgis_loader.sh
For the original loader script and instructions, check the original documentation.
I found it useful to get all the files into a single directory, and then just process them from there (instead of having a separate script to do each state):
wget http://www2.census.gov/geo/pvs/tiger2010st/ --no-parent --relative --recursive --level=2 --accept=zip,txt --mirror --reject=html
The loading process takes anywhere from 30 minutes to over an hour, depending on how many points of interest are in the shape file and the complexity of the generated sql. Once it’s finished, it’s quite simple to geocode addresses with pretty good quality:
/usr/local/pgsql/bin/psql -d geocoder
geocoder=# SELECT g.rating, ST_X(geomout) AS lon, ST_X(geomout) AS lat, (addy).* FROM geocode('2500 Farmers Rd Columbus, Ohio') as g;
rating | lon | lat | address | predirabbrev | streetname | streettypeabbrev | postdirabbrev | internal | location | stateabbrev | zip | parsed
--------+------------+------------+---------+--------------+------------+------------------+---------------+----------+--------------+-------------+-------+--------
8 | -83.784252 | -83.784252 | 2500 | | Farmers | Rd | | | Sabina | OH | 45177 | t
9 | -83.784252 | -83.784252 | 2500 | | Farmers | Rd | | | Wilmington | OH | 45177 | t
11 | -83.784252 | -83.784252 | 2500 | | Farmers | Rd | | | Port William | OH | 45177 | t
13 | -83.087185 | -83.087185 | 2500 | | Farmers | Dr | | | Columbus | OH | 43235 | t
(4 rows)
What’s even more interesting is bulk geocoding by loading an entire table of addresses and selecting them back out (or to another table) geocoded. Here’s some code that does that (modified from http://postgis.refractions.net/documentation/manual-svn/Geocode.html):
CREATE TABLE addresses_to_geocode(addid serial PRIMARY KEY, address text,
lon numeric, lat numeric, new_address text, rating integer);
INSERT INTO addresses_to_geocode(address)
VALUES ('529 Main Street, Boston MA, 02129'),
('77 Massachusetts Avenue, Cambridge, MA 02139'),
('28 Capen Street, Medford, MA'),
('124 Mount Auburn St, Cambridge, Massachusetts 02138'),
('950 Main Street, Worcester, MA 01610');
-- only update the first two addresses (850 ms) --
-- for large numbers of addresses you don't want to update all at once
-- since the whole geocode must commit at once
UPDATE addresses_to_geocode
SET (rating, new_address, lon, lat)
= (g.rating, pprint_addy(g.addy),
ST_X(g.geomout), ST_Y(g.geomout) )
FROM (SELECT DISTINCT ON (addid) addid, (g1.geo).*
FROM (SELECT addid, (geocode(address)) As geo
FROM addresses_to_geocode As ag
WHERE ag.rating IS NULL ) As g1
ORDER BY addid, rating LIMIT 2) As g
WHERE g.addid = addresses_to_geocode.addid;
result
-----
2 rows affected, 850 ms execution time.
SELECT * FROM addresses_to_geocode WHERE rating is not null;
addid | address | lon | lat | new_address | rating
------+----------------------------------------------+-----------+----------+-------------------------------------------+--------
1 | 529 Main Street, Boston MA, 02129 | -71.07187 | 42.38351 | 529 Main St, Boston, MA 02129 | 0
2 | 77 Massachusetts Avenue, Cambridge, MA 02139 | -71.09436 | 42.35981 | 77 Massachusetts Ave, Cambridge, MA 02139 | 0
What’s it like to use it for real life stuff? Well, your mileage may vary. I loaded all US States and performance ranged anywhere from 40 ms to over 3000 ms for some difficult addresses on a dual quad core machine with 32 GB of ram and fast disks. Part of the complexity is in the address parsing and attempting to match address parts to many tables, resulting in table scans and a lot of time and disk complexity. If you have many millions of addresses to geocode, this could be a problem.
I was able to increase performance by “pre-normalizing” my addresses, and then sorting them by zip code. This seemed to help the most, along with increases shared memory cache in Postgres configuration. I’m also looking at optimizing away some unnecessary table scans, and trying to make more use of partitioned tables and different indexes. So far, it’s worked for my purposes, and I haven’t tried to cluster it yet, which may be an option for some (e.g. partition multiple states or sets of data over many machines).
I couldn’t find a nice, clean and OOP javascript library/snippet to both read the query string and provide ability to modify it and navigate to new pages, so I made one . Of particular concern to me was a nice, simple & clean API.
Reading a parameter from the query string:
var somevar = new queryString.get('somevar');
Setting a parameter on the querystring and navigating to the page (while keeping the other query strings, useful for forms and such:
var qs = new queryString.set('somevar', 'somevalue').navigate();
Code:
I did a quick presentation on lazy loading/deferred loading javascript at the cbus.js meeting this week. One thing that surprised me was just how many frameworks/code people knew about that do some form of lazy loading or dependency management. I’ve reviewed several informally, but I probably need to take a closer look and do some actual testing and comparison (sounds like a fun weekend project?).
We had a great turnout at the meeting, with over 40 people in attendance! The last meeting, it was just me who showed up! I’m curious how many people will come to the next one, but I think the talks/presentations inspired some others to give presentations of their own, which has energized the group. Here’s hoping to building a nice, core javascript group in good old Columbus!
Over the past year, I’ve been on a unit testing kick. Yeah, I know :-). In the quest to become a better developer, I’ve been forcing myself to write unit tests. I’ve found it’s more of a behavior or habit more than anything else, with the eventual goal to go full tilt test driven development.
Since I’m mostly a Perl programmer, my typical test toolbox includes Test::More and Test::Harness, which provide for the various ok and assert operations you would expect, outputting their results in TAP format. It’s been nice and there is a certain satisfaction about writing tests for code and getting them all to pass. After doing unit testing for awhile, it feels like I’m only building one side of an arch when I code without them.
Recently, I’ve been developing with node.js and started looking into what was available, especially for the consideration of API development (but something more general purpose, e.g. testing classes and such would be desirable as well). I came across vows.
Vows bills itself as “behavior driven development for node applications”. BDD is somewhat a new concept, but suffice to say it’s basically unit testing higher level application concepts, so called the “behaviors” of the application. Going into exactly what BDD is, however, is not the topic for his post, so go read up about it on wikipedia or some rails/agile blog
So what is so cool about Vows? Well, it’s concept of “promises” or “vows” is very interesting and lends itself to its event-driven nature. Basically, since most tests are executed asynchronously, testing is handled via callbacks. That is, you don’t have to wait for the response of a distant resource, I/O or something else that isn’t instant to complete before you go on to the next test.
Not waiting is important for several reasons. What if you’re building a test suite that unit tests the requests/response of a large web service? It would take awhile for it to run, executing each test synchronously and waiting for the response. In TDD, it’s vital to make sure you can make a quick change, run your tests, and get back to work. Sure, you could test individual API’s, but why? Especially if you’re making changes to the model, or some library that many API’s rely on this issue becomes more complex. So you need to be able to run many tests at once, and this is what vows provides, largely because of the event-driven capabilities of node (it only makes sense for tests to be event driven as well).
My first experience with vows was a little murky, but once I figured out the taxonomy of the test suites (topics, batches and context) it became more clear and I was able to write some awesome tests for a web service API I’m working on. And how much easier can it get to unit test a JSON API using javascript?
I would check out vows especially if you’re working with node. Since it’s so simple and the docs are very good, I’m not going to bother to write a simple test. I would advise just jumping right in. Even if you’re not working with node directly, it’s still worth a look if you are writing a web services API and need a way to support unit testing.
Check it out:
Project Page: http://vowsjs.org/
Github: http://github.com/cloudhead/vows
Loading Javascript Source From a File
The next step in interacting with v8 is to load and compile a javascript file at runtime. This will allow us to make changes to the javascript portion of the application without having to recompile. Although not very impressive in itself, it illustrates the ability of allowing your application to be scriptable, which is a very powerful ability at such a low cost.
Since v8 already evaluates scripts at runtime, the changes to support this are pretty trivial:
#include <v8.h>
#include <string>
using namespace std;
using namespace v8;
// Reads a file into a v8 string.
Handle<String> ReadFile(const string& name) {
FILE* file = fopen(name.c_str(), "rb");
if (file == NULL) return Handle<String>();
fseek(file, 0, SEEK_END);
int size = ftell(file);
rewind(file);
char* chars = new char[size + 1];
chars[size] = '\0';
for (int i = 0; i < size;) {
int read = fread(&chars[i], 1, size - i, file);
i += read;
}
fclose(file);
Handle<String> result = String::New(chars, size);
delete[] chars;
return result;
}
int main(int argc, char* argv[]) {
string file;
if(argc > 1) {
file = argv[1];
} else {
fprintf(stderr, "Must specify script.\n");
return 1;
}
if (file.empty()) {
fprintf(stderr, "No script was specified.\n");
return 1;
}
HandleScope scope;
Handle<String> source = ReadFile(file);
if (source.IsEmpty()) {
fprintf(stderr, "Error reading '%s'.\n", file.c_str());
return 1;
}
Persistent<Context> context = Context::New();
Context::Scope context_scope(context);
Handle<Script> script = Script::Compile(source);
Handle<Value> result = script->Run();
context.Dispose();
String::AsciiValue ascii(result);
printf("%s\n", *ascii);
return 0;
}
We have added a new ReadFile function that gives us back a v8 string handle, and then proceed to create our context as normal. One thing to note about v8 is that you must create a HandleScope before you can create any handles (including string handles on a scope).
After compiling our new changes, the result is pretty cool:
$ ./testing2 test.js Hellooo World!!!
You can change test.js and rerun testing2 to see your changes. We’ve created our own little v8 interpreter! In the next go, we’ll bind some basic C functions to/from javascript! That’s when things will really start to get interesting! :-)
In this short (but sweet!) post I’ll detail how to compile your javascript into a native, ultra-fast C ++ application using the Google V8 javascript engine. By the end of everything, we’ll end up with a statically linked native binary that runs our own little javascript program!
The first step is to download the V8 sources and compile them. To do this, we’ll need to install some dependancies. I built everything on Ubuntu 64 bit, but you should be able to get packages or libraries for OSX and other Linux distributions with some digging around. MacPorts has all of these packages, and I’ve successfully built V8 for it as well.
To begin, Google is using Scons as their build platform (instead of make or similar). You can install it via apt:
$ apt-get install scons
If you’re compiling on 64 bit and want to create 32 bit compatible binaries, you’ll also need the following:
$ apt-get install ia32-libs libc6-dev-i386 g++-4.4-multilib g++-multilib
Finally, lets clone the v8 repo:
$ git clone git://github.com/v8/v8.git v8
You can build a 64 bit library static library with the following command:
$ cd v8 && scons sample=shell mode=release snapshot=on arch=x64
Now with v8 built, you should end up with libv8.a and some include files. I couldn’t find an installation target or script, so I just copied these to my include and /usr/lib paths, so the compiler could find them.
Now we can try to build a hello world example:
#include <v8.h>
using namespace v8;
int main(int argc, char* argv[]) {
HandleScope handle_scope;
Persistent<Context> context = Context::New();
Context::Scope context_scope(context);
Handle<String> source = String::New("var a = 'Hello'; var b = 'World'; a + ' ' + b + '!'");
Handle<Script> script = Script::Compile(source);
Handle<Value> result = script->Run();
context.Dispose();
String::AsciiValue ascii(result);
printf("%s\n", *ascii);
return 0;
}
To compile:
g++ -o testing2 testing.cpp -I/usr/include/v8 -lv8 -L/usr/lib/lib64 -lpthread
If you get compile errors, check your include paths and make sure you’re building with either a 32 bit/64 bit compatible library.
$ ./testing2 Hello World!
Pretty simple, huh?
Practical Applications
So what can we do with this? There are many applications where an integrated javascript engine makes sense. For one, you can implement certain portions of your application in C or C++, such as I/O and buffer handling, which may not be well suited for javascript. Since scripts are compiled at run-time, sources can be loaded from a file or network connection, allowing user defined scripts to be executed and built on top of a C API you expose to V8′s engine, allowing your application to become scriptable. It’s also quite interesting to take this technique and apply it to the many mature and fast C libraries available, creating a glue layer using V8.
In the next post, I’ll go into more detail such as loading a javascript file from disk, accepting user input, and adding custom C functions to the javascript global namespace.
The Case for Javascript Templating
Most who have done some significant frontend work with javascript are well aware of the problem and eventual toss up over how to effectively implement some sort of templating. That is to say something other than string concatenation, and preferably something that cleanly separates content from the logic that creates it and any data structures involved (JSON/XML or other). Recognizing this need, most server side languages have invested great effort and thought into their templating systems. With the emergence of modern browsers and single page web applications, the need for clean separation is even more important as many views and widgets crave clean ways to display content. No one wants to go on a scavenger hunt through 100′s of lines of script to find a string concatenated mess. More importantly, a simple and consistent way of templating enforcing a standard interface and allows for easy separation of content. With the rise of MVC patterns on the frontend, a templating solution is a must. Several have recognized the need (besides myself) and have responded with some interesting (and great!) options for solving the templating problem. I began to review some of the popular solutions, before ultimatley deciding to hack together something myself.
var data = {'who':'BeeBole!', site:'http://beebole.com'},
directive = {
'a':'who', //look for the tag 'a' and place the value of the property 'who' in its node value
'a@href':'site' //look for the tag a, and set its attribute 'href' to the value of the property 'site'
}
//note the use of render instead of autoRender and the 2nd parameter with directive
$('div.template').render(data, directive);
Pure provides a selector based templating solution using a permise of directives, which provides for a selector (allowing you to specify where in the template to take action) and an action which allows you to select from providing a simple string, an object, or a custom javascript function. Since most of the templating is based on CSS selectors, you normally would be targeting actions using a class name on an element. Pure also supports autorendering, largely because the data structure is intermixed with template commands. Pure also supports iteration and dot notation, which is nice when templating arrays of objects into something like a list or table format. Overall, I found the pure framework an interesting choice with some clear benefits. However, I found the directive syntax a bit awkward, getting the feeling that it was designed for common cases, but as soon as corner cases come up (and they do!) I would have to resort to some ugly or otherwise unintuitive code. Part of what I was looking for was simplicity, and lack of features. In addition, the framework isn’t heavy at all but at 21k (unminifed/non-gzipped) it definitely qualifies as a dependency.
// load a template file, then render it with data
html = new EJS({url: '/template.ejs'}).render(data)
// update element 'todo' with the result of a
// template rendered with data from a JSON request
new EJS({url:'/todo.ejs'}).update('todo','/todo.json')
EJS takes a somewhat traditional approach to templating, and at first glance, it seemed to be just what I needed. Its homepage provides a nice, simple example of a very practical usage, along with a little area to test out changes live — we are doing this in the browser, aren’t we? I also liked the ability to put templates in seperate, cachable files, partial support and ulta-simple syntax/interface. EJS presents a convincing case for “clean ajax” so we can get the HTML out of our success functions, and really makes its point. However, I couldn’t find a callback for when the template was finished rendering, or how to attach a callback cleanly. Although with the use of live, the need to attach handlers can be somewhat mitigated, there are still plenty of other cases where you would need to do something to the markup (fade it in, apply styles, etc) after the content has been templated into the element. At 16k uncompressed it’s smaller than pure but still not what I would call ‘micro’.
jQuery templates is packaged as a plugin and is more in line with being micro than what we’ve seen thus far. Simple and straightforward, and ultra lightweight (under 10kb). However, it doesn’t support the ability to have templates in another file, separate from your javascript entirely or a callback function.
var view = {
title: "Joe",
calc: function() {
return 2 + 4;
}
}
var template = "{{title}} spends {{calc}}";
var html = Mustache.to_html(template, view);
Mustache.js boasts use by Twitter and LinkedIn, and has some nice features including the ability to have anonymous functions in the view data structure, conditionals and quick simple tags. In the end, I found it does too much and doesn’t have built in support to fetch templates from an external resource, although it does to caching and partials. I also disliked its use of double hitch-cocks everywhere, but I guess that’s where it gets its name from ;-) ?
var obj= {
name: 'Jabberwocky',
is_starving: function() {
return Math.random() > 0.5;
}
};
$('#template').jqote(obj).appendTo($('body'));
Simple enough and based off John Resig’s awesome micro-templating, jQuote is a nice rewrite/based off it and with much of the same idea. I liked the simple, straight-forward dot notation but disliked the need to use this and the lack of support for external loading (although it does support the nice script tag hack). The <%= syntax is very familiar to those from other languages and very natural, at least to me. Simple, I kept this one in mind but eventually started to look elsewhere when I couldn’t find callbacks again.
var data= {
user: {
login: "tomek",
first_name: "Thomas",
last_name: "Mazur",
account: {
status: "active",
expires_at: "2009-12-31"
}
}
};
$.nano("
Hello {user.first_name} {user.last_name}! Your account is {user.account.status}
", data)
Nano, “perfect for JSON parsing” which is exactly the type of data most are consuming with javascript. I really liked the ability to look deep into an object and the single hitch-cock syntax. Lacking are support for caching and no callbacks. Hacking Something Together Unable to find something that met all my requirements (with and without features), I started looking for blog posts about jquery templating — something very generic when I came across John Resig’s micro templating. Very easy to understand, and something I could easily build from. I also liked the syntax and felt it was flexible enough to be used in just about every case I could throw at it.
var results = document.getElementById("results");
results.innerHTML = tmpl("item_tmpl", dataObject);
Obviously, the first order of business is to convert this code into a jquery plugin, or at least something a little more jquery-ish. In John’s blog post, some already took the liberty of doing so, but I started with a simple plugin pattern and began adding the features I wanted, in a clean and performance conscience way as possible. My main goals were to get my callback that I so desperately wanted (needed! ha!), to support different types of manipulation on the target container (appending, replacing, prepending), and to allow templates to be loaded via a URL (with built-in GET support), or to pass in a template string. I also wanted caching for both compiled template regex and the remote templates. And, I was convinced I could do it all in only a few more lines of code. The hard part was allowing the asynchronous request to complete, and then do the templating and call the callback (if one was provided), and checking in the tpl_cache for a cached template, as well as the fn_cache for our compiled regex. Overall, I’m pretty happy how it turned out. It allows me to request separate files for my template (just static HTML files), caches them, compiles the template regex and caches that as well for subsequent calls. I can pass in one or more elements, and a jquery function to do the dom manipulation (e.g. prepend, append, etc.). And I have my callback! Certainly not perfect, but very lightweight, fast. I enjoyed coming up with a simple solution I hope to evolve as I work more with in-browser templating.
Usage:
$('#tabs').tmpl('/TabTemplate.html', obj,
function(data, el) {
alert('Added a tab!');
}, 'append'
);
(function($) {
var tpl_cache = {};
var fn_cache = {};
$.fn.tmpl = function(tpl, data, cb, fn, kd) {
var cb = typeof(cb) == 'function' ? cb : null;
var fn = fn || 'html';
return this.each(function() {
doTpl(tpl, data, $(this), cb, fn, kd);
});
}
var doTpl = function(tpl, data, el, cb, fn, kd) {
getTpl(tpl, function(tpl_data) {
fn_cache[tpl] = fn_cache[tpl] ||
new Function("obj",
"var p=[],print=function(){p.push.apply(p,arguments);};" +
"with(obj){p.push('" +
tpl_data
.replace(/[\r\t\n]/g, " ")
.split("<%").join("\t") .replace(/((^|%>)[^\t]*)'/g, "$1\r")
.replace(/\t=(.*?)%>/g, "',$1,'")
.split("\t").join("');")
.split("%>").join("p.push('")
.split("\r").join("\\'")
+ "');}return p.join('');");
var r = $(fn_cache[tpl].call(this, data));
if(kd) {
r.data(kd.key, kd.data);
}
el[fn](r);
if(cb) {
cb.call(el, data, r);
}
});
}
var getTpl = function(tpl, cb) {
tpl_cache[tpl] ? cb.call(this, tpl_cache[tpl]) :
$.get(tpl, function(data) {
tpl_cache[tpl] = data;
cb.call(this, tpl_cache[tpl]);
});
}
})(jQuery);
I'm a Software Engineer in San Francisco, CA. I am passionate about the open web, data driven products, and new technologies. Currently interested in javascript, node.js, HTML5, realtime apps, and other cool stuff.