{"id":4876,"date":"2018-01-04T17:14:19","date_gmt":"2018-01-04T09:14:19","guid":{"rendered":"https:\/\/www.hmouse.tk\/?p=4876"},"modified":"2018-01-04T17:20:38","modified_gmt":"2018-01-04T09:20:38","slug":"filebeatlogstaheskinbanahdfs%e7%9a%84%e5%9d%91","status":"publish","type":"post","link":"https:\/\/www.hmouse.cn\/?p=4876","title":{"rendered":"filebeat+logstah+es+kinbana+hdfs\u7684\u5751"},"content":{"rendered":"

\u6700\u8fd1\u7814\u7a76elasticsearch\uff0c\u5176\u5b9e\u662ffilebeat+logstah+es+kinbana+hdfs\u96c6\u7fa4\uff0c\u53d1\u73b0\u4e86\u5f88\u591a\u7684\u5751\uff0c\u8b6c\u5982\u65f6\u533a\u95ee\u9898\uff0c\u5b57\u6bb5\u95ee\u9898\uff0c\u5199\u5165hdfs\u95ee\u9898\uff0c\u7b49\u7b49\u3002\u73b0\u5728\u505a\u4e0b\u603b\u7ed3\uff0c\u65b9\u4fbf\u4ee5\u540e\u67e5\u9605\uff1a<\/p>\n

1\uff0c\u65f6\u533a\u95ee\u9898\uff1a<\/h1>\n

\u9ed8\u8ba4\u89e3\u6790\u65f6\u95f4\u662fUTC0\uff0c\u5176\u5b9e\u7528es\u6ca1\u5565\u95ee\u9898\uff0c\u4f46\u662fhdfs\u91c7\u96c6\u7684\u65f6\u5019\u9700\u8981\u4e89\u53d6\u7684\u65f6\u95f4\uff0c\u6240\u4ee5\u8981\u5bf9\u65f6\u95f4\u8fdb\u884c\u5904\u7406\uff0c\u8c03\u6362\u5230UTC+8\uff0c\u914d\u7f6e\u5982\u4e0b\uff08\u89c4\u5219\u4e00\u822c\u90fd\u662f\u5728filter\u91cc\u9762\u6dfb\u52a0\uff09\uff1a<\/p>\n

date{
               match=>[“tmstp”, “UNIX_MS”]
               target=>”@timestamp”
}
ruby {  
  code => “event.set(‘timestamp’, event.get(‘@timestamp’).time.localtime + 8*60*60)”  
     } <\/p>\n

\u8fd9\u6837’timestamp’\u83b7\u53d6\u7684\u65f6\u95f4\u5c31\u662f\u6b63\u786e8\u65f6\u533a\u7684\u65f6\u95f4\u3002 <\/p>\n

\u5982\u679c\u60f3\u628a@timestamp \u4e5f\u6362\u6389\uff0c\u5c31\u53ef\u4ee5\u66ff\u6362\u6389\uff1a <\/p>\n

ruby {
   code => “event.set(‘@timestamp’,event.get(‘timestamp’))”
} <\/p>\n

\u7531\u4e8e\u6211\u4e0d\u5e0c\u671b@timestamp \u8bbe\u7f6e\u62108\u65f6\u533a\uff0c\u4f46\u662f\u53c8\u60f3\u8981\u6b63\u786e\u7684\u65f6\u95f4\u8def\u5f84\uff0c\u6240\u4ee5\u81ea\u884c\u8bbe\u7f6e\u4e86\u53d8\u91cf\uff1a<\/p>\n

date{
                match=>[“tmstp”, “UNIX_MS”]
                target=>”@timestamp”
  }
ruby {  
   code => “event.set(‘timestamp’, event.get(‘@timestamp’).time.localtime + 8*60*60)”  
      } <\/p>\n

ruby {
   code => “event.set(‘timestamp1’, event.get(‘@timestamp’))”
}
ruby {
   code => “event.set(‘@timestamp’,event.get(‘timestamp’))”
} <\/p>\n

mutate {
          add_field => {
            “datapath”=>”%{+yy-MM}\/%{+dd}\/%{+HH}”
            “service_type” => “%{service_type1[6]}_access”
          } <\/p>\n

       } <\/p>\n

ruby {
   code => “event.set(‘@timestamp’, event.get(‘timestamp1’))”
}
mutate {
          remove_field => [“service_type1″,”timestamp1″,”timestamp”]
       } <\/p>\n

\u4ee5\u4e0a\u5c31\u6ee1\u8db3hdfs\u83b7\u53d6\u7684\u5199\u5165\u8def\u5f84\u662f\u5317\u4eac\u65f6\u95f4\u7684\u683c\u5f0f\uff0c@timestamp\u8fd8\u662fUTC\u76840\u65f6\u533a\u3002<\/p>\n

2\uff1ajson\u5904\u7406<\/h1>\n

\u7531\u4e8e\u6211\u4eec\u7684\u65e5\u5fd7\u683c\u5f0f\u662f:<\/p>\n

2018-01-04 12:17:59,872 AM GMT+08:00 INFO  – {“p”:”SP”,”ser”:3,”tmstp”:1514996279872,”serip”:”172.23.89.7″,”logtp”:2,”fip”:”\/172.23.89.5:58111″}<\/p>\n

\u9700\u8981\u5bf9\u6570\u636e\u8fdb\u884c\u5904\u7406\u622a\u53d6{}\u4f5c\u4e3ajson\u683c\u5f0f\uff0c\u524d\u9762\u90fd\u9700\u8981\u4e22\u5f03\u3002<\/p>\n

\u9700\u8981\u81ea\u5b9a\u4e49\u89c4\u5219\uff0c\u8ddf\u4e4b\u524d\u7684nginx\u65e5\u5fd7\u4e00\u6837\uff1a<\/p>\n

[root@emr-worker-1 logstash-6.1.1]# cat vendor\/bundle\/jruby\/2.3.0\/gems\/logstash-patterns-core-4.1.2\/patterns\/chat
CHATLOG \\{(.*)\\}
\u5b9a\u4e49\u4e86CHATLOG \u683c\u5f0f\u65e5\u5fd7\u3002 <\/p>\n

grok {
      match => { “message” => “%{CHATLOG:chatlog}” }
  } <\/p>\n

\u8fd9\u6837\u5c31\u5bf9message\u505a\u4e86json\u622a\u53d6\u3002\u7136\u540e\u89e3\u6790\u4e0b<\/p>\n

json {
       source => “chatlog”
   }<\/p>\n

\u8fd9\u6837\u5c31\u53d8\u6210\u4e86\u7eafjson\u683c\u5f0f\u7684\u65e5\u5fd7\u3002<\/p>\n

2\uff1a\u5199\u5165hdfs\u7684\u95ee\u9898<\/h1>\n

\u5199\u5165\u7ecf\u5e38\u51b2\u7a81\uff0c\u540e\u6765\u53d1\u73b0\u5e94\u8be5\u6bcf\u4e2alogstash\u5199\u5165\u5230hdfs\u6700\u597d\u4e0d\u662f\u540c\u4e00\u4e2a\u6587\u4ef6\uff0c<\/p>\n

output {
#     stdout { codec => rubydebug }
if [p] {
     webhdfs {
        host => [“172.23.89.33”]              
        port => 50070                     
        path => “\/liaobei\/logdata\/%{service_type}\/%{datapath}\/data01<\/font>-access.log”
        user => “hadoop”                     
        compression => “snappy”
        codec => line {format => “%{chatlog}”} 
      }
     }
} <\/p>\n

\u6700\u597d\u629b\u51fa\u6389json\u5f02\u5e38\uff08\u4e2a\u4eba\u89c9\u5f97\u8fd9\u4e2a\u5176\u5b9e\u6ca1\u7528\uff09 <\/p>\n

if  “_jsonparsefailure” in [tags] {
    drop { }
  } <\/p>\n

\u4e0d\u8fc7\u540e\u9762\u5176\u5b9e\u53d1\u73b0\u662f\u56e0\u4e3a\u6211\u8865\u5145\u7684\u65e5\u5fd7\uff0c\u6ca1\u5bf9service_type\u505a\u8bf4\u660e\uff0c\u9020\u6210service_type\u6ca1\u529e\u6cd5\u83b7\u53d6\uff0c\u624d\u9020\u6210\u7684hdfs\u7684\u5199\u5165\u62a5\u9519\u3002\u8fd9\u70b9\u4e5f\u8981\u6ce8\u610f\uff0c\u56e0\u4e3a\u8865\u5145\u65e5\u5fd7\u7684\u65f6\u5019\uff0c\u6211\u968f\u4fbf\u5f04\u4e2a\u6587\u4ef6\uff0c\u9020\u6210\u4e86\u8def\u5f84\u83b7\u53d6\u9519\u8bef\uff0c\u9020\u6210\u4e86service_type\u6ca1\u529e\u6cd5\u83b7\u53d6\u3002 <\/p>\n

mutate {
         add_field=>{“service_type1″=>”%{source}”}
       } <\/p>\n

mutate {
          split => [“service_type1” , “\/”]
} <\/p>\n

mutate {
          add_field => {
            “datapath”=>”%{+yy-MM}\/%{+dd}\/%{+HH}”
            “service_type” => “%{service_type1[6]}_access”
          } <\/p>\n

       } <\/p>\n

\u6240\u4ee5\u4e4b\u540e\u7684php\u4e00\u76f4\u6ca1\u62a5\u9519\uff0cPHP\u662f\u56e0\u4e3aservice_type\u5199\u6b7b\u7684\uff0c\u6240\u4ee5\u8865\u5145\u65e5\u5fd7\u4e5f\u5199\u6b7b\u4e86\u3002\u6240\u4ee5\u6ca1\u6709\u62a5\u9519\u3002\u53ea\u6709chat\u7684\u65e5\u5fd7\u62a5\u9519\u3002<\/p>\n

\u5f53\u7136\u7531\u4e8e\u540e\u9762\u6ca1\u6709\u65f6\u95f4\u9a8c\u8bc1\uff0c\u5c31\u6ca1\u505a\u91cd\u73b0\u3002\u5927\u6982\u5c31\u662f\u8fd9\u4e2a\u95ee\u9898\u3002<\/p>\n

<\/h1>\n

3\uff1akafka\u548ces\u7684\u5751<\/h1>\n

kafka\u4e3b\u8981\u662f php\u90a3\u8fb9\u6709\u5927\u7684\u65e5\u5fd7\u5199\u5165\uff0c\u4f1a\u62a5The request included a message larger than the max message size the server will accept\u3002\u8fd9\u4e2a\u4e0d\u7b97\u5751\uff0c\u53ea\u80fd\u8bf4\u6211\u4eec\u7684\u65e5\u5fd7\u5b9e\u5728\u662f\u592a\u5927\u4e86\uff0c\u4e0d\u5408\u7406\u3002<\/p>\n

\u4fee\u6539\u914d\u7f6e<\/p>\n

message.max.bytes=10240000
fetch.message.max.bytes=10240000 <\/p>\n

\u8ba9kafka\u63a5\u53d7\u7684\u6d88\u606f\u66f4\u5927\u3002<\/p>\n

 <\/p>\n

es\u7684\u8bdd\uff0c\u662f\u4e00\u4e2aindex\u7684\u8bc6\u522b\u521b\u5efa\u6709\u95ee\u9898\u3002\u539f\u56e0\u8fd8\u6ca1\u67e5\u660e\uff0c\u4f30\u8ba1\u662f\u7a0b\u5e8f\u7684BUG\u3002\u6211\u53d1\u73b0es5.5\u548c6.1\u90fd\u6709\u8fd9\u4e2a\u62a5\u9519\u3002<\/p>\n

\u62a5\u9519\u5982\u4e0b:<\/p>\n

[2018-01-04T14:15:24,625][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>[“index”, {:_id=>nil, :_index=>”logstash-2018.01.04″, :_type=>”chat_log”, :_routing=>nil}, #<LogStash::Event:0x44960714>], :response=>{“index”=>{“_index”=>”logstash-2018.01.04”, “_type”=>”chat_log”, “_id”=>”AWC_z1Sxlwnz255tEaZZ”, “status”=>400, “error”=>{“type”=>”mapper_parsing_exception”, “reason”=>”failed to parse [verbose]”, “caused_by”=>{“type”=>”illegal_state_exception”, “reason”=>”Can’t get text on a START_OBJECT at 1:174”}}}}} <\/p>\n

\u662f\u8bf4verbose\u65e0\u6cd5json\u89e3\u6790\uff08failed to parse [verbose]\uff09\u3002\u4f46\u662fflume\u4f20\u8fc7\u6765\u7684\u90fd\u662fjson\u683c\u5f0f\uff0c\u800c\u4e14\u6709\u7684\u66f4\u52a0\u590d\u6742\u7684json\u683c\u5f0f\u90fd\u6ca1\u62a5\u9519\uff0c\u6211\u5f53\u65f6\u6000\u7591\u662f\u4e0d\u662fverbose\u5b57\u6bb5\u7684\u95ee\u9898\u3002\u6240\u4ee5\u6211\u66f4\u6362\u4e86\u540d\u5b57\u53d8\u6210php_verbose\u3002\u5c31\u4e0d\u518d\u62a5\u9519\u3002\u672c\u6765\u6000\u7591\u662f\u4e0d\u662f\u5386\u53f2\u521b\u5efa\u7684\u7d22\u5f15\u6709\u95ee\u9898\uff0c\u4f46\u662f\u5c1d\u8bd5\u8fc7\u5220\u9664\u6240\u6709\u7684\u6570\u636e\u548c\u7d22\u5f15\uff0c\u5373\u4f7f\u6ca1\u6709verbose\u7684index\uff0c\u4e5f\u4e00\u6837\u62a5\u9519\u3002\u6211\u4e5f\u6000\u7591\u7248\u672c\u95ee\u9898\uff0c\u4f46\u662f\u5347\u7ea7logstash \u548c es\u90fd\u65e0\u6cd5\u89e3\u51b3\u95ee\u9898\u3002\u76ee\u524d\u63a8\u6d4b\u5c31\u662f  \u8fd9\u4e2a\u5b57\u6bb5\u662f\u7cfb\u7edf\u7684\u5b57\u6bb5\uff0c\u6709\u7279\u6b8a\u5904\u7406\u3002\u4ee5\u524d\u9047\u5230\u8fc7\u7c7b\u4f3c\u60c5\u51b5\uff0c\u8b6c\u5982@timestamp\u3002\u540e\u9762\u6709\u7a7a\u53ef\u4ee5\u67e5\u4e0b\uff0c\u76ee\u524d\u5c31\u662f\u66f4\u6362\u5b57\u6bb5\uff0c\u5c31\u4e0d\u518d\u62a5\u9519\u3002<\/p>\n

\"image\"<\/a> <\/p>\n

if [verbose]  {
     mutate {
         add_field => { “php_verbose”=>”%{verbose}” }
         remove_field => [“verbose”]
            }
} <\/p>\n

\u7531\u4e8echat\u65e5\u5fd7\u6ca1\u6709verbose\u5b57\u6bb5\uff0c\u6240\u4ee5\u5728\u6539\u540d\u4e4b\u524d\uff0c\u8fd8\u505a\u4e86\u5b57\u6bb5\u5b58\u5728\u5224\u65ad\u3002<\/p>\n","protected":false},"excerpt":{"rendered":"

\u6700\u8fd1\u7814\u7a76elasticsearch\uff0c\u5176\u5b9e\u662ffilebeat+logstah+es+kinbana+hdfs\u96c6\u7fa4\uff0c\u53d1\u73b0\u4e86\u5f88\u591a\u7684\u5751\uff0c\u8b6c\u5982\u65f6\u533a\u95ee\u9898\uff0c\u5b57\u6bb5\u95ee\u9898\uff0c\u5199\u5165hdfs\u95ee\u9898\uff0c\u7b49\u7b49\u3002\u73b0\u5728\u505a\u4e0b\u603b\u7ed3\uff0c\u65b9\u4fbf\u4ee5\u540e\u67e5\u9605\uff1a<\/p>\n

1\uff0c\u65f6\u533a\u95ee\u9898\uff1a<\/h1>\n

\u9ed8\u8ba4\u89e3\u6790\u65f6\u95f4\u662fUTC0\uff0c\u5176\u5b9e\u7528es\u6ca1\u5565\u95ee\u9898\uff0c\u4f46\u662fhdfs\u91c7\u96c6\u7684\u65f6\u5019\u9700\u8981\u4e89\u53d6\u7684\u65f6\u95f4\uff0c\u6240\u4ee5\u8981\u5bf9\u65f6\u95f4\u8fdb\u884c\u5904\u7406\uff0c\u8c03\u6362\u5230UTC+8\uff0c\u914d\u7f6e\u5982\u4e0b\uff08\u89c4\u5219\u4e00\u822c\u90fd\u662f\u5728filter\u91cc\u9762\u6dfb\u52a0\uff09\uff1a<\/p>\n

date{
               match=>[“tmstp”, “UNIX_MS”]
               target=>”@timestamp”
}
ruby {  
  code => “event.set(‘timestamp’, event.get(‘@timestamp’).time.localtime + 8*60*60)”  
     } <\/p>\n

\u8fd9\u6837’timestamp’\u83b7\u53d6\u7684\u65f6\u95f4\u5c31\u662f\u6b63\u786e8\u65f6\u533a\u7684\u65f6\u95f4\u3002 <\/p>\n

\u5982\u679c\u60f3\u628a@timestamp \u4e5f\u6362\u6389\uff0c\u5c31\u53ef\u4ee5\u66ff\u6362\u6389\uff1a <\/p>\n

ruby {
   code => “event.set(‘@timestamp’,event.get(‘timestamp’))”
} <\/p>\n

\u7531\u4e8e\u6211\u4e0d\u5e0c\u671b@timestamp \u8bbe\u7f6e\u62108\u65f6\u533a\uff0c\u4f46\u662f\u53c8\u60f3\u8981\u6b63\u786e\u7684\u65f6\u95f4\u8def\u5f84\uff0c\u6240\u4ee5\u81ea\u884c\u8bbe\u7f6e\u4e86\u53d8\u91cf\uff1a<\/p>\n

date{
                match=>[“tmstp”, “UNIX_MS”]
                target=>”@timestamp”
  }
ruby {  
   code => “event.set(‘timestamp’, event.get(‘@timestamp’).time.localtime + 8*60*60)”  
      } <\/p>\n

ruby {
   code => “event.set(‘timestamp1’, event.get(‘@timestamp’))”
}
ruby {
   code => “event.set(‘@timestamp’,event.get(‘timestamp’))”
} <\/p>\n

mutate {
          add_field => {
            “datapath”=>”%{+yy-MM}\/%{+dd}\/%{+HH}”
            “service_type” => “%{service_type1[6]}_access”
          } <\/p>\n

       } <\/p>\n

ruby {
   code => “event.set(‘@timestamp’, event.get(‘timestamp1’))”
}
mutate {
          remove_field => [“service_type1″,”timestamp1″,”timestamp”]
       } <\/p>\n

\u4ee5\u4e0a\u5c31\u6ee1\u8db3hdfs\u83b7\u53d6\u7684\u5199\u5165\u8def\u5f84\u662f\u5317\u4eac\u65f6\u95f4\u7684\u683c\u5f0f\uff0c@timestamp\u8fd8\u662fUTC\u76840\u65f6\u533a\u3002<\/p>\n

2\uff1ajson\u5904\u7406<\/h1>\n

\u7531\u4e8e\u6211\u4eec\u7684\u65e5\u5fd7\u683c\u5f0f\u662f:<\/p>\n

2018-01-04 12:17:59,872 AM GMT+08:00 INFO  – {“p”:”SP”,”ser”:3,”tmstp”:1514996279872,”serip”:”172.23.89.7″,”logtp”:2,”fip”:”\/172.23.89.5:58111″}<\/p>\n

\u9700\u8981\u5bf9\u6570\u636e\u8fdb\u884c\u5904\u7406\u622a\u53d6{}\u4f5c\u4e3ajson\u683c\u5f0f\uff0c\u524d\u9762\u90fd\u9700\u8981\u4e22\u5f03\u3002<\/p>\n

\u9700\u8981\u81ea\u5b9a\u4e49\u89c4\u5219\uff0c\u8ddf\u4e4b\u524d\u7684nginx\u65e5\u5fd7\u4e00\u6837\uff1a<\/p>\n

[root@emr-worker-1 logstash-6.1.1]# cat vendor\/bundle\/jruby\/2.3.0\/gems\/logstash-patterns-core-4.1.2\/patterns\/chat
CHATLOG \\{(.*)\\}
\u5b9a\u4e49\u4e86CHATLOG \u683c\u5f0f\u65e5\u5fd7\u3002 <\/p>\n

grok {
      match => { “message” => “%{CHATLOG:chatlog}” }
  } <\/p>\n

\u8fd9\u6837\u5c31\u5bf9message\u505a\u4e86json\u622a\u53d6\u3002\u7136\u540e\u89e3\u6790\u4e0b<\/p>\n

json {
       source => “chatlog”
   }<\/p>\n

\u8fd9\u6837\u5c31\u53d8\u6210\u4e86\u7eafjson\u683c\u5f0f\u7684\u65e5\u5fd7\u3002<\/p>\n

2\uff1a\u5199\u5165hdfs\u7684\u95ee\u9898<\/h1>\n

\u5199\u5165\u7ecf\u5e38\u51b2\u7a81\uff0c\u540e\u6765\u53d1\u73b0\u5e94\u8be5\u6bcf\u4e2alogstash\u5199\u5165\u5230hdfs\u6700\u597d\u4e0d\u662f\u540c\u4e00\u4e2a\u6587\u4ef6\uff0c<\/p>\n

output {
#     stdout { codec => rubydebug }
if [p] {
     webhdfs {
        host => [“172.23.89.33”]              
        port => 50070                     
        path => “\/liaobei\/logdata\/%{service_type}\/%{datapath}\/data01<\/font>-access.log”
        user => “hadoop”                     
        compression => “snappy”
        codec => line {format => “%{chatlog}”} 
      }
     }
} <\/p>\n

\u6700\u597d\u629b\u51fa\u6389json\u5f02\u5e38\uff08\u4e2a\u4eba\u89c9\u5f97\u8fd9\u4e2a\u5176\u5b9e\u6ca1\u7528\uff09 <\/p>\n

if  “_jsonparsefailure” in [tags] {
    drop { }
  } <\/p>\n

\u4e0d\u8fc7\u540e\u9762\u5176\u5b9e\u53d1\u73b0\u662f\u56e0\u4e3a\u6211\u8865\u5145\u7684\u65e5\u5fd7\uff0c\u6ca1\u5bf9service_type\u505a\u8bf4\u660e\uff0c\u9020\u6210service_type\u6ca1\u529e\u6cd5\u83b7\u53d6\uff0c\u624d\u9020\u6210\u7684hdfs\u7684\u5199\u5165\u62a5\u9519\u3002\u8fd9\u70b9\u4e5f\u8981\u6ce8\u610f\uff0c\u56e0\u4e3a\u8865\u5145\u65e5\u5fd7\u7684\u65f6\u5019\uff0c\u6211\u968f\u4fbf\u5f04\u4e2a\u6587\u4ef6\uff0c\u9020\u6210\u4e86\u8def\u5f84\u83b7\u53d6\u9519\u8bef\uff0c\u9020\u6210\u4e86service_type\u6ca1\u529e\u6cd5\u83b7\u53d6\u3002 <\/p>\n

mutate {
         add_field=>{“service_type1″=>”%{source}”}
       } <\/p>\n

mutate {
          split => [“service_type1” , “\/”]
} <\/p>\n

mutate {
          add_field => {
            “datapath”=>”%{+yy-MM}\/%{+dd}\/%{+HH}”
            “service_type” => “%{service_type1[6]}_access”
          } <\/p>\n

       } <\/p>\n

\u6240\u4ee5\u4e4b\u540e\u7684php\u4e00\u76f4\u6ca1\u62a5\u9519\uff0cPHP\u662f\u56e0\u4e3aservice_type\u5199\u6b7b\u7684\uff0c\u6240\u4ee5\u8865\u5145\u65e5\u5fd7\u4e5f\u5199\u6b7b\u4e86\u3002\u6240\u4ee5\u6ca1\u6709\u62a5\u9519\u3002\u53ea\u6709chat\u7684\u65e5\u5fd7\u62a5\u9519\u3002<\/p>\n

\u5f53\u7136\u7531\u4e8e\u540e\u9762\u6ca1\u6709\u65f6\u95f4\u9a8c\u8bc1\uff0c\u5c31\u6ca1\u505a\u91cd\u73b0\u3002\u5927\u6982\u5c31\u662f\u8fd9\u4e2a\u95ee\u9898\u3002<\/p>\n

<\/h1>\n

3\uff1akafka\u548ces\u7684\u5751<\/h1>\n

kafka\u4e3b\u8981\u662f php\u90a3\u8fb9\u6709\u5927\u7684\u65e5\u5fd7\u5199\u5165\uff0c\u4f1a\u62a5The request included a message larger than the max message size the server will accept\u3002\u8fd9\u4e2a\u4e0d\u7b97\u5751\uff0c\u53ea\u80fd\u8bf4\u6211\u4eec\u7684\u65e5\u5fd7\u5b9e\u5728\u662f\u592a\u5927\u4e86\uff0c\u4e0d\u5408\u7406\u3002<\/p>\n

\u4fee\u6539\u914d\u7f6e<\/p>\n

message.max.bytes=10240000
fetch.message.max.bytes=10240000 <\/p>\n

\u8ba9kafka\u63a5\u53d7\u7684\u6d88\u606f\u66f4\u5927\u3002<\/p>\n

 <\/p>\n

es\u7684\u8bdd\uff0c\u662f\u4e00\u4e2aindex\u7684\u8bc6\u522b\u521b\u5efa\u6709\u95ee\u9898\u3002\u539f\u56e0\u8fd8\u6ca1\u67e5\u660e\uff0c\u4f30\u8ba1\u662f\u7a0b\u5e8f\u7684BUG\u3002\u6211\u53d1\u73b0es5.5\u548c6.1\u90fd\u6709\u8fd9\u4e2a\u62a5\u9519\u3002<\/p>\n

\u62a5\u9519\u5982\u4e0b:<\/p>\n

[2018-01-04T14:15:24,625][WARN ][logstash.outputs.elasticsearch] Could not index event to Elasticsearch. {:status=>400, :action=>[“index”, {:_id=>nil, :_index=>”logstash-2018.01.04″, :_type=>”chat_log”, :_routing=>nil}, #<LogStash::Event:0x44960714>], :response=>{“index”=>{“_index”=>”logstash-2018.01.04”, “_type”=>”chat_log”, “_id”=>”AWC_z1Sxlwnz255tEaZZ”, “status”=>400, “error”=>{“type”=>”mapper_parsing_exception”, “reason”=>”failed to parse [verbose]”, “caused_by”=>{“type”=>”illegal_state_exception”, “reason”=>”Can’t get text on a START_OBJECT at 1:174”}}}}} <\/p>\n

\u662f\u8bf4verbose\u65e0\u6cd5json\u89e3\u6790\uff08failed to parse [verbose]\uff09\u3002\u4f46\u662fflume\u4f20\u8fc7\u6765\u7684\u90fd\u662fjson\u683c\u5f0f\uff0c\u800c\u4e14\u6709\u7684\u66f4\u52a0\u590d\u6742\u7684json\u683c\u5f0f\u90fd\u6ca1\u62a5\u9519\uff0c\u6211\u5f53\u65f6\u6000\u7591\u662f\u4e0d\u662fverbose\u5b57\u6bb5\u7684\u95ee\u9898\u3002\u6240\u4ee5\u6211\u66f4\u6362\u4e86\u540d\u5b57\u53d8\u6210php_verbose\u3002\u5c31\u4e0d\u518d\u62a5\u9519\u3002\u672c\u6765\u6000\u7591\u662f\u4e0d\u662f\u5386\u53f2\u521b\u5efa\u7684\u7d22\u5f15\u6709\u95ee\u9898\uff0c\u4f46\u662f\u5c1d\u8bd5\u8fc7\u5220\u9664\u6240\u6709\u7684\u6570\u636e\u548c\u7d22\u5f15\uff0c\u5373\u4f7f\u6ca1\u6709verbose\u7684index\uff0c\u4e5f\u4e00\u6837\u62a5\u9519\u3002\u6211\u4e5f\u6000\u7591\u7248\u672c\u95ee\u9898\uff0c\u4f46\u662f\u5347\u7ea7logstash \u548c es\u90fd\u65e0\u6cd5\u89e3\u51b3\u95ee\u9898\u3002\u76ee\u524d\u63a8\u6d4b\u5c31\u662f  \u8fd9\u4e2a\u5b57\u6bb5\u662f\u7cfb\u7edf\u7684\u5b57\u6bb5\uff0c\u6709\u7279\u6b8a\u5904\u7406\u3002\u4ee5\u524d\u9047\u5230\u8fc7\u7c7b\u4f3c\u60c5\u51b5\uff0c\u8b6c\u5982@timestamp\u3002\u540e\u9762\u6709\u7a7a\u53ef\u4ee5\u67e5\u4e0b\uff0c\u76ee\u524d\u5c31\u662f\u66f4\u6362\u5b57\u6bb5\uff0c\u5c31\u4e0d\u518d\u62a5\u9519\u3002<\/p>\n

\"image\"<\/a> <\/p>\n

if [verbose]  {
     mutate {
         add_field => { “php_verbose”=>”%{verbose}” }
         remove_field => [“verbose”]
            }
} <\/p>\n

\u7531\u4e8echat\u65e5\u5fd7\u6ca1\u6709verbose\u5b57\u6bb5\uff0c\u6240\u4ee5\u5728\u6539\u540d\u4e4b\u524d\uff0c\u8fd8\u505a\u4e86\u5b57\u6bb5\u5b58\u5728\u5224\u65ad\u3002<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[5],"tags":[229,226,227,193,230,228],"class_list":{"0":"post-4876","1":"post","2":"type-post","3":"status-publish","4":"format-standard","5":"hentry","6":"category-5","7":"tag-es","9":"tag-filebeat","10":"tag-hdfs","11":"tag-kinbana","12":"tag-logstah","13":"category-5-id","14":"post-seq-1","15":"post-parity-odd","16":"meta-position-corners","17":"fix"},"amp_enabled":true,"_links":{"self":[{"href":"https:\/\/www.hmouse.cn\/index.php?rest_route=\/wp\/v2\/posts\/4876","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.hmouse.cn\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.hmouse.cn\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.hmouse.cn\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"https:\/\/www.hmouse.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=4876"}],"version-history":[{"count":2,"href":"https:\/\/www.hmouse.cn\/index.php?rest_route=\/wp\/v2\/posts\/4876\/revisions"}],"predecessor-version":[{"id":4880,"href":"https:\/\/www.hmouse.cn\/index.php?rest_route=\/wp\/v2\/posts\/4876\/revisions\/4880"}],"wp:attachment":[{"href":"https:\/\/www.hmouse.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=4876"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.hmouse.cn\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=4876"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.hmouse.cn\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=4876"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}