iterate node.js請求功能

Question

這個問題是關於一個履帶在node.js。 A start_url給出了他抓取URL的地方，並將它們「推送」給了一個。 json -file（output.json）。 目前，他只使用start_url運行請求功能，並將收集的URL保存在output.json中。我想，他通過與第一收集網址替換start_url使用保存的網址，並再次收集鏈接...等等...

var fs = require('fs'); 
var request = require('request'); 
var cheerio = require('cheerio'); 

var start_url = ["http://stackoverflow.com/"] 

var req = function(url){ 
    request(url, function(error, response, html){ 
     var $ = cheerio.load(html); 

     var data = []; 

     $("a").each(function() { 
      var link = $(this); 
       var exurls = {exurl: new Array(link.attr("href"))} 

       data.push(exurls); 

       // Queue "exurls" for "start_url" and call the same function with the new URL (endless loop) 
       // save to "output.json" from time to time, so you can stop it anytime 
     }); 

     fs.writeFile("output.json", JSON.stringify(data, null, 4), function(err){ 
      if(err){ 
       console.log(err); 
      } else { 
       console.log("File successfully written!"); 
      } 
     }); 
    }); 
} 
for (var i = 0; i < start_url.length; i++){ 
    req(start_url[i]); 
} 

Answer 1

所以你可以做的是使函數調用遞歸。下面的例子應該工作：

var fs = require('fs'); 
 
var request = require('request'); 
 
var cheerio = require('cheerio'); 
 

 
var start_url = ["http://stackoverflow.com/"] 
 

 
var req = function(url){ 
 
    var count = 0; 
 
    
 
    request(url, function(error, response, html){ 
 
    var $ = cheerio.load(html); 
 

 
    $("a").each(function() { 
 
     var link = $(this); 
 
     var exurls = {exurl: new Array(link.attr("href"))} 
 

 
     start_url.push(exurls); 
 

 
     // Queue "exurls" for "start_url" and call the same function with the new URL (endless loop) 
 
     // save to "output.json" from time to time, so you can stop it anytime 
 
    }); 
 

 
    try { 
 
     fs.writeFileSync("output.json"); 
 
     console.log("File successfully written!"); 
 
    }catch(err){ 
 
     console.log(err); 
 
    } 
 
     
 
     ++count; 
 
     
 
     if(start_url.length > count) { 
 
     req(start_url[count]); 
 
     } 
 
    }); 
 
} 
 

 
return req(start_url[0]);

這裏的問題是，你完全重寫每次文件。如果這種情況持續一段時間，你將耗盡內存。另一種選擇是創建一個寫流

var fs = require('fs'); 
 
    var request = require('request'); 
 
    var cheerio = require('cheerio'); 
 

 
    var start_url = ["http://stackoverflow.com/"] 
 
    
 
    var wstream = fs.createWriteStream("output.json"); 
 

 
    var req = function(url){ 
 
     
 
     request(url, function(error, response, html){ 
 
     var $ = cheerio.load(html); 
 

 
     $("a").each(function() { 
 
      var link = $(this); 
 
      var exurls = {exurl: new Array(link.attr("href"))} 
 

 
      start_url.push(exurls); 
 

 
      // Queue "exurls" for "start_url" and call the same function with the new URL (endless loop) 
 
      // save to "output.json" from time to time, so you can stop it anytime 
 
      wstream.write('"'+ exurls + '",'); 
 
     }); 
 
      
 
     start_url.shift(); 
 
     if(start_url.length > 0) { 
 
      return req(start_url[0]); 
 
     } 
 
      
 
      wstream.end(); 
 
     }); 
 
    } 
 

 
    req(start_url[0]);

編輯：切換到基本的隊列，以戰鬥的記憶問題

---

---