如何在dynamodb中超過1mb的掃描數據限制

Question

我正在使用dynamodb和nodejs，我有3000條記錄，並且我在代碼中編寫了60多個段，每段掃描1mb數據並顯示結果到60多個1MB的限制。因此，請提供解決方案，如何在單個步驟中獲得3000條記錄掃描，這意味着在一個分段中。請迅速提供解決方案，因爲我在項目中間受到了攻擊。請幫幫我。以下是我的代碼。

var AWS = require("aws-sdk"); 
var async = require("async"); 

    AWS.config.update({ 
     region: "---", 
     endpoint: "-----------", 
     accessKeyId: "-----------------", 
     secretAccessKey:"----------" 
    }); 


var db = new AWS.DynamoDB.DocumentClient() 

var table = "rets_property_all"; 
var pstart =new Date() .getTime(); 

async.parallel({ 
      0 : function(callback){ 
     db.scan ({TableName: table, 
      ProjectionExpression: "#cityname,ListingKey ", 
      FilterExpression: "#cityname = :v_id", 
      ExpressionAttributeNames: { 
       "#cityname": "CityName", 
      }, 
      ExpressionAttributeValues: {":v_id" : 'BALTIMORE'}, 
      TotalSegments: 63, 
      Segment: 0//by the worker who has been called 
     },function (err , res) { 
      callback (null , res.Items); 
     }); 
      }, 
      1 : function(callback){ 
     db.scan ({TableName: table, 
      ProjectionExpression: "#cityname,ListingKey ", 
      FilterExpression: "#cityname = :v_id", 
      ExpressionAttributeNames: { 
       "#cityname": "CityName", 
      }, 
      ExpressionAttributeValues: {":v_id" : 'BALTIMORE'}, 
      TotalSegments: 63, 
      Segment: 1//by the worker who has been called 
     }, function (err , res) { 
      callback (null , res.Items); 
     }); 
      }, 
     2 : function(callback){ 
     db.scan ({TableName: table, 
      ProjectionExpression: "#cityname,ListingKey ", 
      FilterExpression: "#cityname = :v_id", 
      ExpressionAttributeNames: { 
       "#cityname": "CityName", 
      }, 
      ExpressionAttributeValues: {":v_id" : 'BALTIMORE'}, 
      TotalSegments: 63, 
      Segment: 2//by the worker who has been called 
     }, function (err , res) { 
      callback (null , res.Items); 
     }); 
      }, 
-------- 
--------- 
------ 

     62 : function(callback){ 
     db.scan ({TableName: table, 
      ProjectionExpression: "#cityname,ListingKey ", 
      FilterExpression: "#cityname = :v_id", 
      ExpressionAttributeNames: { 
       "#cityname": "CityName", 
      }, 
      ExpressionAttributeValues: {":v_id" : 'BALTIMORE'}, 
      TotalSegments: 63, 
      Segment: 62//by the worker who has been called 
     }, function (err , res) { 
      callback (null , res.Items); 
     }); 
     }, 

     },function(err,results){ 
     if (err) {throw err; } 
     var pend = new Date() .getTime(); 

     console.log (results); 
     }) 

Answer 1

其實，有沒有辦法覆蓋掃描1 MB限制。這是DynamoDB設計限制，不能被任何API覆蓋。您可能需要了解體系結構或AWS服務設計的限制。

您可以在隨後的掃描中使用LastEvaluatedKey從上次掃描結束時開始。

掃描的結果集限於每次調用1 MB。您可以使用掃描響應中的LastEvaluatedKey 來檢索更多結果。

用例不清楚爲什麼你想在一次掃描中獲得所有3000條記錄。即使您有特定用例，也無法在DynamoDB掃描中實現。

即使在關係型數據庫中，您也可以獲取遊標並遍歷迭代以順序獲取所有記錄。同樣，在DynamoDB中，您必須遞歸使用Scan，直到LastEvaluatedKey爲空。單掃描

示例代碼遞歸直到LastEvaluatedKey爲空： -

var docClient = new AWS.DynamoDB.DocumentClient(); 
var params = { 
    TableName: table, 
    ProjectionExpression: "#cityname,ListingKey ", 
    FilterExpression: "#cityname = :v_id", 
    ExpressionAttributeNames: { 
     "#cityname": "CityName", 
    }, 
    ExpressionAttributeValues: { ":v_id": 'BALTIMORE' } 
}; 

docClient.scan(params, onScan); 

function onScan(err, data) { 
    if (err) { 
     console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2)); 
    } else { 
     // print all the movies 
     console.log("Scan succeeded."); 
     data.Items.forEach(function (movie) { 
      console.log("Item :", JSON.stringify(movie)); 
     }); 

     // continue scanning if we have more movies 
     if (typeof data.LastEvaluatedKey != "undefined") { 
      console.log("Scanning for more..."); 
      params.ExclusiveStartKey = data.LastEvaluatedKey; 
      docClient.scan(params, onScan); 
     } 
    } 
}