Nodejs抓取非utf8字符编码的页面

时间：2011-05-15 来源：MK2

问题: nodejs目前无法处理非utf8编码以外的字符

在cnodejs用户组，有同学遇到抓取百度页面出现编码问题

由于Buffer.toString(encoding)中encoding只支持utf8编码，所以需要附加模块才能处理此问题

解决：node-iconv 模块

安装:

$ sudo npm install iconv

示例

var http = require('http');

var options = {
    host: 'www.baidu.com',
    port: 80,
    path: '/s?wd=nodejs'
};

var Iconv  = require('iconv').Iconv;

http.get(options, function(res) {
    console.log("Got response: " + res.statusCode, res.headers);
    var buffers = [], size = 0;
    res.on('data', function(buffer) {
        buffers.push(buffer);
        size += buffer.length;
    });
    res.on('end', function() {
        var buffer = new Buffer(size), pos = 0;
        for(var i = 0, len = buffers.length; i < len; i++) {
            buffers[i].copy(buffer, pos);
            pos += buffers[i].length;
        }
        // 'content-type': 'text/html;charset=gbk',
        var gb2312_to_utf8_iconv = new Iconv('GBK', 'UTF-8');
        var utf8_buffer = gb2312_to_utf8_iconv.convert(buffer);
        console.log(utf8_buffer.toString());
    });
}).on('error', function(e) {
    console.log("Got error: " + e.message);
});