![\"image\"](\"https:\/\/www.hmouse.cn\/wp-content\/uploads\/2016\/06\/image.png\" "\\"image\\"")
<\/a> <\/p>\n\u7814\u53d1\u4f1a\u628a\u4e1a\u52a1\u65e5\u5fd7\u5199\u6210json\u683c\u5f0f\uff0c\u7136\u540e\u5199\u5165\u672c\u5730\u7684\u6587\u4ef6\uff0c\u6211\u4eec\u5728\u670d\u52a1\u5668\u4e0a\u5b89\u88c5logstash\u8f6f\u4ef6\uff0c\u8d1f\u8d23\u91c7\u96c6\uff0c\u7531\u4e8e\u662fjson\u683c\u5f0f\uff0c\u6240\u4ee5\u6307\u5b9acodec=\u300bjson\u683c\u5f0f\uff0c\u8fd9\u6837\u80fd\u5927\u5927\u63d0\u9ad8logstash\u7684\u91c7\u96c6\u901f\u5ea6\u3002<\/p>\n
\u7531\u4e8e\u53ef\u80fd\u62c5\u5fc3\u65e5\u5fd7\u6587\u4ef6\u7684\u5de8\u5927\uff0c\u540e\u7eed\u5904\u7406\u4e0d\u8fc7\u6765\uff0c\u6240\u4ee5\u6211\u4eec\u8bbe\u7f6e\u4e86\u4e00\u4e2a\u4e2d\u95f4\u4ef6kafka\uff0c\u4e34\u65f6\u5b58\u50a8\u6587\u4ef6\uff0ckafka\u771f\u7684\u5f88\u597d\u7528\uff0c\u4e3b\u8981\u5f97\u76ca\u4e8e\u4ed6\u7684\u65e5\u5fd7\u6587\u4ef6\u5b58\u50a8\uff0c\u4e00\u6bb5\u65f6\u95f4\u53c8\u53ef\u4ee5\u81ea\u52a8\u5220\u9664\u3002\u53c8\u5f15\u5165\u4e86Partition \u548c cousumer group\u7684\u6982\u5ff5\u3002partition \u7684\u51fa\u73b0 \u5bf9\u6d88\u606f\u7684\u987a\u5e8f\u4e0d\u662f\u8981\u6c42\u90a3\u4e48\u9ad8\u7684\u8bdd\uff0c\u53ef\u4ee5\u5206\u6210\u591a\u4e2apartition\u4f9b\u591a\u4e2a\u6d88\u8d39\u8005\u4f7f\u7528\u3002\u5927\u5927\u7684\u63d0\u9ad8\u6d88\u8d39\u901f\u5ea6\u3002cousumer group \u53ef\u4ee5\u8ba9 queue \u53d8\u6210 \u5e7f\u64ad\uff0c\u4e00\u4e2a\u6d88\u606f\u88ab\u591a\u6b21\u6d88\u8d39\u3002\u8fd9\u6837\u53ef\u4ee5\u89e3\u51b3\u6d88\u606f\u7684\u591a\u6b21\u5229\u7528\u5b58\u50a8\u3002<\/p>\n
hadoop \u4e3b\u8981\u505a\u6574\u4e2a\u65e5\u5fd7\u5e73\u53f0\u7684 \u6587\u4ef6\u5b58\u50a8\u3002\u6536\u96c6\u6240\u6709\u7684\u65e5\u5fd7\u5e76\u4ee5\u6587\u4ef6\u683c\u5f0f\u5b58\u50a8\u3002\u5728hadoop\u5b58\u50a8\u7684\u95ee\u9898\u6211\u4e3b\u8981\u662f\u9047\u5230\u4e86\uff0cjson\u683c\u5f0f\u7684\u4e1a\u52a1\u65e5\u5fd7\u8ddf\u7cfb\u7edf\u65e5\u5fd7\u5982\u679c\u5b58\u50a8\u5728\u4e00\u8d77\uff0c\u76ee\u524d\u7684\u65b9\u6848\u662f\u901a\u8fc7\u4ecekafka\u5206\u4e0d\u540c\u7684topic\u53bb\u5b9e\u73b0\uff0c\u5f53\u7136\u540e\u6765\u53d1\u73b0\u4e5f\u53ef\u4ee5\u901a\u8fc7\u5b57\u6bb5\u8fc7\u6ee4\u53bb\u89e3\u51b3\u3002\u73b0\u5728\u6211\u628a\u65e5\u5fd7\u5206\u4e3a rizhi \u548c xt_rizhi\u3002rizhi\u5b58\u50a8\u7684\u90fd\u662fjson \u800c xt_rizhi\u4e3b\u8981\u5b58\u50a8\u7684\u662f\u975ejson\u683c\u5f0f\u7684jboss\u65e5\u5fd7\u3002<\/p>\n
elasticsearch \u4e3b\u8981\u662f\u505a\u6211\u4eec\u7684\u641c\u7d22\u5f15\u8d77\u4f7f\u7528\u3002\u65b9\u4fbf\u6211\u4eec\u4ee5\u540e\u5bf9\u7ebf\u4e0a\u4e1a\u52a1\u7684\u7edf\u8ba1\u3002\u7531\u4e8e\u65e5\u5fd7\u5de8\u5927\uff0c\u53ef\u80fd\u65e5\u5fd7\u4fdd\u5b58\u7684\u5929\u6570\u4f1a\u6bd4\u8f83\u77ed\u3002<\/p>\n
influxdb \u4e3b\u8981\u662f\u4e3a\u4e86\u8bb0\u5f55\u7814\u53d1\u8981\u6c42\u7684 \u843d\u5e93\u7684\u6570\u636e\uff0cinfluxdb \u65e0\u8bba\u6027\u80fd\u4e0a \u8fd8\u662f \u8ddflogstash \u914d\u5408\u4e0a \u90fd\u6bd4mysql\u66f4\u597d\u3002\u4e3b\u8981\u662f\u901a\u8fc7\u5b57\u6bb5\u7684\u8fc7\u6ee4\uff0c\u5199\u5165\u6307\u5b9a\u6570\u636e\u3002<\/p>\n
output {
if [method] == “listByUser” {
# stdout {
# codec => rubydebug
# }
influxdb {
# data_points => {
# “host” => “%{host}”
# “hostname” => “%{hostname}”
# “uri” => “%{uri}”
# “timestamp” => “%{timestamp}”
# “remoteAddr” => “%{remoteAddr}”
# }
use_event_fields_for_data_points => true
exclude_fields => [“@version”,”@timestamp”,”kafka”]
allow_time_override => true
measurement => “%{logger}”
host => “10.25.85.52”
db => “test”
user => “test1”
password => “test123”
codec => “json”
# exclude_fields => [“@version”,”@timestamp”]
}
}
} <\/p>\n
<\/p>\n
\u8fd9\u6837\u4e0b\u6765\u4e5f\u6ee1\u8db3\u7814\u53d1\u548c\u8fd0\u7ef4\u7684\u9700\u6c42\uff0c\u6240\u6709\u7684\u65e5\u5fd7\u6c47\u603b\uff0c\u8fd0\u7ef4\u505a\u4e00\u4e0b\u7b80\u5355\u7684\u5206\u6790\uff0c\u6587\u4ef6\u5168\u90e8\u5b58\u50a8\u5728hadoop\u3002\u653e\u4e00\u90e8\u5206\u6570\u636e\u5230influxdb\u9632\u6b62\u4ee5\u540e\u8981\u67e5\u770b\u4f7f\u7528\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"
\u6700\u8fd1\u51e0\u4e2a\u6708\u90fd\u5728\u5fd9\u7740\u65e5\u5fd7\u7cfb\u7edf\u7684\u67b6\u6784\u3002\u8bbe\u8ba1\u51fa\u4e00\u5957\u6bd4\u8f83\u9ad8\u53ef\u7528\uff0c\u9ad8\u541e\u5410\u7684\u65e5\u5fd7\u7cfb\u7edf\uff0c\u5e76\u80fd\u7b26\u5408\u6211\u4eec\u7684\u4e1a\u52a1\u9700\u6c42\u3002\u6700\u7ec8\u7684\u67b6\u6784\u5982\u4e0b\uff1a<\/p>\n