使用node.js下載大文件以避免高內存消耗

Question

我試圖創建一個文件下載程序作爲後臺服務，但是當計劃一個大文件時，它首先放在內存中，然後在下載結束時文件被寫入磁盤。

考慮到我可能同時有很多文件正在下載，我該如何讓文件逐漸寫入磁盤保留內存？

這裏使用的代碼我真的：

var sys = require("sys"), 
    http = require("http"), 
    url = require("url"), 
    path = require("path"), 
    fs = require("fs"), 
    events = require("events"); 

var downloadfile = "http://nodejs.org/dist/node-v0.2.6.tar.gz"; 

var host = url.parse(downloadfile).hostname 
var filename = url.parse(downloadfile).pathname.split("/").pop() 

var theurl = http.createClient(80, host); 
var requestUrl = downloadfile; 
sys.puts("Downloading file: " + filename); 
sys.puts("Before download request"); 
var request = theurl.request('GET', requestUrl, {"host": host}); 
request.end(); 

var dlprogress = 0; 


setInterval(function() { 
    sys.puts("Download progress: " + dlprogress + " bytes"); 
}, 1000); 


request.addListener('response', function (response) { 
    response.setEncoding('binary') 
    sys.puts("File size: " + response.headers['content-length'] + " bytes.") 
    var body = ''; 
    response.addListener('data', function (chunk) { 
     dlprogress += chunk.length; 
     body += chunk; 
    }); 
    response.addListener("end", function() { 
     fs.writeFileSync(filename, body, 'binary'); 
     sys.puts("After download finished"); 
    }); 

}); 

Answer 1

我改變了回調：

request.addListener('response', function (response) { 
     var downloadfile = fs.createWriteStream(filename, {'flags': 'a'}); 
     sys.puts("File size " + filename + ": " + response.headers['content-length'] + " bytes."); 
     response.addListener('data', function (chunk) { 
      dlprogress += chunk.length; 
      downloadfile.write(chunk, encoding='binary'); 
     }); 
     response.addListener("end", function() { 
      downloadfile.end(); 
      sys.puts("Finished downloading " + filename); 
     }); 

    }); 

這完美地工作。

Answer 2

而不是拿着你應該寫在追加模式的文件"data"事件偵聽器的內容到內存中。

Answer 3

下載大文件時請使用fs.write而不是writeFile，因爲它會覆蓋以前的內容。

function downloadfile(res) { 
    var requestserver = http.request(options, function(r) { 
     console.log('STATUS: ' + r.statusCode); 
     console.log('HEADERS: ' + JSON.stringify(r.headers)); 

     var fd = fs.openSync('sai.tar.gz', 'w'); 

     r.on('data', function (chunk) { 
      size += chunk.length; 
      console.log(size+'bytes received'); 
      sendstatus(res,size); 
      fs.write(fd, chunk, 0, chunk.length, null, function(er, written) { 
      }); 
     }); 
     r.on('end',function(){ 
      console.log('\nended from server'); 
      fs.closeSync(fd); 
      sendendstatus(res); 
     }); 
    }); 
} 

Answer 4

請求包是否適合您的使用？

它可以讓你做這樣的事情：

request(downloadurl).pipe(fs.createWriteStream(downloadtohere)) 

Answer 5

像卡特·科爾使用流建議。這裏是一個更完整的例子

var inspect = require('eyespect').inspector(); 
var request = require('request'); 
var filed = require('filed'); 
var temp = require('temp'); 
var downloadURL = 'http://upload.wikimedia.org/wikipedia/commons/e/ec/Hazard_Creek_Kayaker.JPG'; 
var downloadPath = temp.path({prefix: 'singlePageRaw', suffix: '.jpg'}); 

var downloadFile = filed(downloadPath); 
var r = request(downloadURL).pipe(downloadFile); 


r.on('data', function(data) { 
    inspect('binary data received'); 
}); 
downloadFile.on('end', function() { 
    inspect(downloadPath, 'file downloaded to path'); 
}); 

downloadFile.on('error', function (err) { 
    inspect(err, 'error downloading file'); 
}); 

您可能需要安裝模塊，您可以通過 npm install filed request eyespect temp

Answer 6

做在http-request看看：

// shorthand syntax, buffered response 
http.get('http://localhost/get', function (err, res) { 
    if (err) throw err; 
    console.log(res.code, res.headers, res.buffer.toString()); 
}); 

// save the response to 'myfile.bin' with a progress callback 
http.get({ 
    url: 'http://localhost/get', 
    progress: function (current, total) { 
     console.log('downloaded %d bytes from %d', current, total); 
    } 
}, 'myfile.bin', function (err, res) { 
    if (err) throw err; 
    console.log(res.code, res.headers, res.file); 
});