我想用BigQuery UDF處理appengine_googleapis_com_request_log.protoPayload.line

Question

我正在將App Engine標準的請求日誌導出到BigQuery。

由於Application輸出的日誌是在protoPayload.line中的，我們試圖用UDF來處理它。

但是，由於類型不匹配錯誤，查詢無法執行。 我打算讓模具保持一致，但我該如何解決它？

Error: No matching signature for function JS:LINE_COUNT for argument types: ARRAY<STRUCT<time TIMESTAMP, severity STRING, logMessage STRING, ...>>. Supported signature: LINE_COUNT(ARRAY<STRUCT<time TIMESTAMP, severity STRING, logMessage STRING, ...>>) at [3:3] 

查詢：

#standardSQL 
CREATE TEMPORARY FUNCTION LINE_COUNT(lines ARRAY<STRUCT<time TIMESTAMP, 
    severity STRING, 
    logMessage STRING, 
    sourceLocation STRUCT<file STRING, 
    line FLOAT64, 
    functionName STRING>>>) 
RETURNS FLOAT64 
LANGUAGE js AS """ 
var count = 0; 
for (var i = 0; i < lines.length; i++) { 
    count++; 
} 
return count; 
"""; 
SELECT 
LINE_COUNT(protoPayload.line) 
FROM 
`gaelog_from_bqstreaming.appengine_googleapis_com_request_log_*` 
WHERE 
_TABLE_SUFFIX BETWEEN FORMAT_DATE("%Y%m%d", 
    DATE_SUB(CURRENT_DATE(), 
    INTERVAL 1 DAY)) 
AND FORMAT_DATE("%Y%m%d", 
    CURRENT_DATE()) 
LIMIT 
1000 

Answer 1

這個怎麼樣？您不需要指定整個類型：

#standardSQL 
CREATE TEMPORARY FUNCTION LINE_COUNT(
    lines_json STRING) 
RETURNS FLOAT64 
LANGUAGE js AS """ 
var lines = JSON.Parse(lines_json); 
var count = 0; 
for (var i = 0; i < lines.length; i++) { 
    count++; 
} 
return count; 
"""; 
SELECT 
LINE_COUNT(TO_JSON_STRING(protoPayload.line)) 
FROM 
`gaelog_from_bqstreaming.appengine_googleapis_com_request_log_*` 
WHERE 
_TABLE_SUFFIX BETWEEN FORMAT_DATE("%Y%m%d", 
    DATE_SUB(CURRENT_DATE(), 
    INTERVAL 1 DAY)) 
AND FORMAT_DATE("%Y%m%d", 
    CURRENT_DATE()) 
LIMIT 
1000 

請注意，您不需要JavaScript來進行這種計算。你可以用SQL直接表達出來，通常會更便宜，因爲你不會需要一個更高層次的計費：

SELECT 
    ARRAY_LENGTH(protoPayload.line) 
FROM 
`gaelog_from_bqstreaming.appengine_googleapis_com_request_log_*` 
WHERE 
_TABLE_SUFFIX BETWEEN FORMAT_DATE("%Y%m%d", 
    DATE_SUB(CURRENT_DATE(), 
    INTERVAL 1 DAY)) 
AND FORMAT_DATE("%Y%m%d", 
    CURRENT_DATE()) 
LIMIT 
1000 

Answer 2

讓我們終於讓我做的代碼。 我想通過ARRAY進行搜索並提取它並使其成爲STRUCT。

#standardSQL 
    CREATE TEMPORARY FUNCTION GetQueueStats(lines_json STRING) 
    RETURNS STRUCT<email STRING, executed1Minute FLOAT64> 
    LANGUAGE js AS """ 
    var lines = JSON.parse(lines_json); 
    var result = { 
    email : '', 
    executed1Minute : 0 
    }; 
    lines.forEach(function(element, index, array) { 
    var emailPrefix = 'userEmail='; 
    var queueStats = 'TaskQueue.QueueStats:'; 
    if (element.logMessage.startsWith(emailPrefix)) { 
     result.email = element.logMessage.substring(emailPrefix.length); 
    } else if (element.logMessage.startsWith(queueStats)) { 
     var vj = element.logMessage.substring(queueStats.length); 
     var v = JSON.parse(vj); 
     result.executed1Minute = v.Executed1Minute; 
    } 
    }); 

    return result; 
"""; 
SELECT 
    insertId, 
    timestamp, 
    GetQueueStats(TO_JSON_STRING(protoPayload.line)) AS queueStats 
FROM 
    `appengine.appengine_googleapis_com_request_log_20171014` 
ORDER BY 
    timestamp DESC 
LIMIT 
    1000