2017-06-22 132 views
0

我使用envContainerOverrides從stepfunctions讀取輸出,然後使用此nodejscode調用批處理作業:在這裏,我正在讀取傳遞給批量作業。如何在java類(lambda函數)中使用AWS批處理提交批作業

Stepfunctions輸出:

{"bucketName":"bucketName","filesList":["filelist-bucket/filelist1.txt","filelist-bucket/filelist2.txt"]} 

此代碼的NodeJS不能如果其抓取的陣列[文件清單:桶/ filelist1.txt「」文件列表鬥/ filelist2.txt]它完美如果其一個單一的價值。

我想這個轉換代碼的NodeJS成Java:

'use strict'; 

const AWS = require('aws-sdk'); 

console.log('Loading function'); 

exports.handler = (event, context, callback) => { 
    // Take the data from step 1 and modify, send to standard output 
    var comment = event.Comment ; 

    var envContainerOverrides ={ 
     "environment" :[ 
      { 
      "name":"s3Bucket", 
      "value":event.bucketName 
      }, 
      { 
      "name":"s3FileList", 
      "value":event.filesListUrl 
      } 
     ] 
    }; 
    const params = { 
     jobDefinition: process.env.JOB_DEFINITION, 
     jobName: process.env.JOB_NAME, 
     jobQueue:process.env.JOB_QUEUE, 
     containerOverrides: envContainerOverrides || null, 
     parameters: event.parameters || null, 
    }; 


    // Submit the Batch Job 
    new AWS.Batch().submitJob(params, (err, data) => { 
     if (err) { 
      console.error(err); 
      const message = `Error calling SubmitJob for: ${event.jobName}`; 
      console.error(message); 
      callback(message); 
     } else { 
      const jobId = data.jobId; 
      console.log('jobId:', jobId); 
      callback(null, "Job Id : "+jobId); 
     } 
    }); 



}; 

我做這樣的事情:

public class InitiateBatchJob1 { 

    public static BatchJobRequest process(BatchJobRequest batchJobRequest) throws Exception { 


     String s3Bucket = batchJobRequest.getBucketName(); 
     List<String> s3FileList = batchJobRequest.getFilesListUrl(); 

     Job job = new Job(); 
     job.setJobDefinition("testbatchjobenv:2"); 
     job.setJobQueue("nbatchjobqueue"); 
     job.setJobName("Filedownload"); 


     /*// Submit the Batch Job 
     new AWS.Batch().submitJob(params, (err, data) => { 
      if (err) { 
       console.error(err); 
       const message = `Error calling SubmitJob for: ${event.jobName}`; 
       console.error(message); 
       callback(message); 
      } else { 
       const jobId = data.jobId; 
       console.log('jobId:', jobId); 
       callback(null, "Job Id : "+jobId); 
      } 
     }); 
*/ 



     return null; 
    } 


} 

BatchJobRequest.java

import java.util.List; 

public class BatchJobRequest { 

    private String bucketName; 

    private List<String> filesListUrl; 

    public String getBucketName() { 
     return bucketName; 
    } 

    public void setBucketName(String bucketName) { 
     this.bucketName = bucketName; 
    } 

    public List<String> getFilesListUrl() { 
     return filesListUrl; 
    } 

    public void setFilesListUrl(List<String> filesListUrl) { 
     this.filesListUrl = filesListUrl; 
    } 

} 

Job.java:

public class Job { 

    private String jobDefinition; 
    private String jobName; 
    private String jobQueue; 
    private String containerOverrides; 
    private String parameters; 
    public String getJobDefinition() { 
     return jobDefinition; 
    } 
    public void setJobDefinition(String jobDefinition) { 
     this.jobDefinition = jobDefinition; 
    } 
    public String getJobName() { 
     return jobName; 
    } 
    public void setJobName(String jobName) { 
     this.jobName = jobName; 
    } 
    public String getJobQueue() { 
     return jobQueue; 
    } 
    public void setJobQueue(String jobQueue) { 
     this.jobQueue = jobQueue; 
    } 
    public String getContainerOverrides() { 
     return containerOverrides; 
    } 
    public void setContainerOverrides(String containerOverrides) { 
     this.containerOverrides = containerOverrides; 
    } 
    public String getParameters() { 
     return parameters; 
    } 
    public void setParameters(String parameters) { 
     this.parameters = parameters; 
    } 


} 

,我發現這個API AWSBatch http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/batch/AWSBatch.html#cancelJob-com.amazonaws.services.batch.model.CancelJobRequest-

但不知道它在我的class.I使用正確的API在網上搜心不是在Java中使用AWS批量多鏈接。

基本上我需要知道如何設置作業denfition,jobName,作業隊列和使用java類提交batchjob。任何人都可以幫助我這個。

回答

0

你必須做這樣的事情: http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/batch/AWSBatch.html#submitJob-com.amazonaws.services.batch.model.SubmitJobRequest-

AWSBatch client = AWSBatchClientBuilder.standard().build(); 
    SubmitJobRequest request = new SubmitJobRequest().withJobName("some-name") 
      .withJobQueue("job-queue-name") 
      .withJobDefinition("job-definition-name-with-revision-number:1"); 
    SubmitJobResult response = client.submitJob(request); 

我用這個代碼與lambda函數與AWSBatchFullAccess我在AWS控制檯創建的IAM角色。因此,在aws上構建並加載jar之後,我的'客戶'使用lambda中的數據進行初始化。在你的應用程序中,你似乎也需要添加數據來初始化客戶端。您必須使用該方法

AWSBatch build(AwsSyncClientParams params) 

查看AWSBatchClientBuilder類。

您仍然需要創建作業隊列和作業定義。 另外我會建議你開始使用aws控制檯。這裏有很好的教程來了解aws批處理工作流程http://technology.finra.org/code/enjoying-auto-scaling-integrated-authentication-low-host-cost.htmlhttp://www.awsomeblog.com/analysing-exif-data-with-aws-batch/。在本教程之後,您可以添加用於在myHandler方法中提交作業的代碼,並查看作業如何啓動。