{"id":854,"date":"2020-11-05T19:36:14","date_gmt":"2020-11-05T11:36:14","guid":{"rendered":"https:\/\/www.specialwu.com\/?p=854"},"modified":"2024-01-24T01:58:59","modified_gmt":"2024-01-23T17:58:59","slug":"%e6%95%b0%e6%8d%ae%e4%bc%a0%e8%be%93%e4%b9%8b-flume","status":"publish","type":"post","link":"http:\/\/www.specialwu.com\/?p=854","title":{"rendered":"\u6570\u636e\u4f20\u8f93\u4e4b-Flume"},"content":{"rendered":"<ul>\n<li>\u5b98\u7f51\u7ed9\u51fa\u7684\u4ecb\u7ecd<\/li>\n<\/ul>\n<p>Flume is a distributed, reliable, and available service for efficiently collecting, aggregating, and moving large amounts of log data. It has a simple and flexible architecture based on streaming data flows. It is robust and fault tolerant with tunable reliability mechanisms and many failover and recovery mechanisms. It uses a simple extensible data model that allows for online analytic application.<br \/>\n<a class=\"wp-editor-md-post-content-link\" href=\"http:\/\/flume.apache.org\/releases\/content\/1.9.0\/FlumeUserGuide.html\" title=\"flume\u914d\u7f6e\">\u70b9\u51fb\u67e5\u770bflume\u5b98\u65b9\u914d\u7f6e\u6587\u6863<\/a><\/p>\n<div id=\"ez-toc-container\" class=\"ez-toc-v2_0_61 ez-toc-wrap-center counter-hierarchy ez-toc-counter ez-toc-custom ez-toc-container-direction\">\n<div class=\"ez-toc-title-container\">\n<p class=\"ez-toc-title \" >\u76ee\u5f55<\/p>\n<span class=\"ez-toc-title-toggle\"><a href=\"#\" class=\"ez-toc-pull-right ez-toc-btn ez-toc-btn-xs ez-toc-btn-default ez-toc-toggle\" aria-label=\"Toggle Table of Content\"><span class=\"ez-toc-js-icon-con\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #000000;color:#000000\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #000000;color:#000000\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/span><\/a><\/span><\/div>\n<nav><ul class='ez-toc-list ez-toc-list-level-1 ' ><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"http:\/\/www.specialwu.com\/?p=854\/#%E5%89%8D%E8%A8%80\" title=\"\u524d\u8a00\">\u524d\u8a00<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"http:\/\/www.specialwu.com\/?p=854\/#%E4%B8%80%E3%80%81%E6%98%AF%E4%BB%80%E4%B9%88\" title=\"\u4e00\u3001\u662f\u4ec0\u4e48\">\u4e00\u3001\u662f\u4ec0\u4e48<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-3\" href=\"http:\/\/www.specialwu.com\/?p=854\/#%E4%BA%8C%E3%80%81%E6%80%8E%E4%B9%88%E7%94%A8\" title=\"\u4e8c\u3001\u600e\u4e48\u7528\">\u4e8c\u3001\u600e\u4e48\u7528<\/a><ul class='ez-toc-list-level-2' ><li class='ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-4\" href=\"http:\/\/www.specialwu.com\/?p=854\/#%E5%AE%89%E8%A3%85%E8%BF%87%E7%A8%8B\" title=\"\u5b89\u88c5\u8fc7\u7a0b\">\u5b89\u88c5\u8fc7\u7a0b<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-5\" href=\"http:\/\/www.specialwu.com\/?p=854\/#%E6%80%8E%E4%B9%88%E4%BD%BF%E7%94%A8\" title=\"\u600e\u4e48\u4f7f\u7528\">\u600e\u4e48\u4f7f\u7528<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-6\" href=\"http:\/\/www.specialwu.com\/?p=854\/#%E8%A2%AB%E6%89%A7%E8%A1%8C%E7%9A%84%E8%84%9A%E6%9C%AC\" title=\"\u88ab\u6267\u884c\u7684\u811a\u672c\">\u88ab\u6267\u884c\u7684\u811a\u672c<\/a><\/li><\/ul><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-7\" href=\"http:\/\/www.specialwu.com\/?p=854\/#%E4%B8%89%E3%80%81%E9%9C%80%E8%A6%81%E9%87%8D%E7%82%B9%E5%85%B3%E6%B3%A8%E7%9A%84%E5%9C%B0%E6%96%B9\" title=\"\u4e09\u3001\u9700\u8981\u91cd\u70b9\u5173\u6ce8\u7684\u5730\u65b9\">\u4e09\u3001\u9700\u8981\u91cd\u70b9\u5173\u6ce8\u7684\u5730\u65b9<\/a><ul class='ez-toc-list-level-2' ><li class='ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-8\" href=\"http:\/\/www.specialwu.com\/?p=854\/#Flume%E7%9A%84%E9%85%8D%E7%BD%AE%E5%AE%9E%E4%BE%8B\" title=\"Flume\u7684\u914d\u7f6e\u5b9e\u4f8b\">Flume\u7684\u914d\u7f6e\u5b9e\u4f8b<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-9\" href=\"http:\/\/www.specialwu.com\/?p=854\/#source\" title=\"source\">source<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-10\" href=\"http:\/\/www.specialwu.com\/?p=854\/#sink\" title=\"sink\">sink<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-11\" href=\"http:\/\/www.specialwu.com\/?p=854\/#channel\" title=\"channel\">channel<\/a><\/li><\/ul><\/li><\/ul><\/li><\/ul><\/nav><\/div>\n<h1><span class=\"ez-toc-section\" id=\"%E5%89%8D%E8%A8%80\"><\/span>\u524d\u8a00<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<p>flume\u4f5c\u4e3a\u4e00\u4e2a\u6570\u636e\u91c7\u96c6\u5de5\u5177\uff0c\u5f88\u660e\u663e\u7684\u4e00\u4e2a\u7279\u70b9\u5c31\u662f\u91c7\u96c6\u6570\u636e\u6bd4\u8f83\u7075\u6d3b\uff0c\u901a\u5e38\u6211\u4eec\u5e38\u628aflume\u7528\u6765\u548ckafka\u8fdb\u884c\u6bd4\u8f83\uff0c\u4f46\u5176\u5b9eflume\u548cflume\u548clogstash\u6bd4\u8f83\u662f\u6bd4\u8f83\u5408\u9002\u7684\u3002\u6211\u4eec\u4f9d\u7136\u6309\u7167\uff0c\u662f\u4ec0\u4e48\uff0c\u600e\u4e48\u7528\u3002\u5b83\u7684\u6838\u5fc3\u65b9\u9762\u6709\u54ea\u4e9b\u8fdb\u884c\u5c55\u5f00\u3002<\/p>\n<h1><span class=\"ez-toc-section\" id=\"%E4%B8%80%E3%80%81%E6%98%AF%E4%BB%80%E4%B9%88\"><\/span>\u4e00\u3001\u662f\u4ec0\u4e48<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<p>Flume\u662f\u4e00\u79cd\u5206\u5e03\u5f0f\uff0c\u53ef\u9760\u4e14\u53ef\u7528\u7684\u670d\u52a1\uff0c\u7528\u4e8e\u6709\u6548\u5730\u6536\u96c6\uff0c\u805a\u5408\u548c\u79fb\u52a8\u5927\u91cf\u65e5\u5fd7\u6570\u636e\u3002\u5b83\u5177\u6709\u57fa\u4e8e\u6d41\u6570\u636e\u6d41\u7684\u7b80\u5355\u7075\u6d3b\u7684\u4f53\u7cfb\u7ed3\u6784\u3002\u5b83\u5177\u6709\u53ef\u8c03\u6574\u7684\u53ef\u9760\u6027\u673a\u5236\u4ee5\u53ca\u8bb8\u591a\u6545\u969c\u8f6c\u79fb\u548c\u6062\u590d\u673a\u5236\uff0c\u5177\u6709\u5f3a\u5927\u7684\u529f\u80fd\u548c\u5bb9\u9519\u80fd\u529b\u3002\u5b83\u4f7f\u7528\u4e00\u4e2a\u7b80\u5355\u7684\u53ef\u6269\u5c55\u6570\u636e\u6a21\u578b\uff0c\u5141\u8bb8\u5728\u7ebf\u5206\u6790\u5e94\u7528\u7a0b\u5e8f\u3002\u4e8b\u4ef6\u4f5c\u4e3aFlume\u5185\u90e8\u6570\u636e\u4f20\u8f93\u7684\u6700\u57fa\u672c\u5355\u5143.\u5b83\u662f\u7531\u4e00\u4e2a\u8f6c\u8f7d\u6570\u636e\u7684\u5b57\u8282\u6570\u7ec4(\u8be5\u6570\u636e\u7ec4\u662f\u4ece\u6570\u636e\u6e90\u63a5\u5165\u70b9\u4f20\u5165\uff0c\u5e76\u4f20\u8f93\u7ed9\u4f20\u8f93\u5668\uff0c\u4e5f\u5c31\u662fHDFS\/HBase)\u548c\u4e00\u4e2a\u53ef\u9009\u5934\u90e8\u6784\u6210.<\/p>\n<p>flume\u662f\u4e00\u4e2a\u6570\u636e\u91c7\u96c6\u5de5\u5177\uff0cagent\u662fflume\u7684\u4e00\u4e2a\u57fa\u672c\u5355\u5143\uff1bflume\u7684\u6838\u5fc3\u5c31\u662f\u628a\u6570\u636e\u4ece\u6570\u636e\u6e90\uff08source\uff09\u6536\u96c6\u8fc7\u6765\uff0c\u518d\u5c06\u6536\u96c6\u5230\u7684\u6570\u636e\u9001\u5230\u6307\u5b9a\u7684\u76ee\u7684\u5730\uff08sink\uff09\u3002\u4e3a\u4e86\u4fdd\u8bc1\u8f93\u9001\u7684\u8fc7\u7a0b\u4e00\u5b9a\u6210\u529f\uff0c\u5728\u9001\u5230\u76ee\u7684\u5730\u4e4b\u524d\uff0c\u4f1a\u5148\u7f13\u5b58\u6570\u636e\uff08channel\uff09\uff0c\u5f85\u6570\u636e\u771f\u6b63\u7684\u9001\u5230\u76ee\u7684\u5730\uff08sink\uff09\u540e\uff0cflume\u518d\u5220\u9664\u7f13\u5b58\u4e2d\u7684\u6570\u636e\uff08channel\u4e2d\u7684\u6570\u636e\uff09\u3002\u4e00\u884c\u6587\u672c\u5185\u5bb9\u88ab\u53cd\u5e8f\u5217\u5316\u6210\u4e00\u4e2aevent\u3002event\u7684\u6700\u5927\u5b9a\u4e49\u4e3a2048\u5b57\u8282\uff0c\u8d85\u8fc7\u4f1a\u88ab\u5207\u5272\u653e\u5230\u4e0b\u4e00\u4e2aevent\u4e2d\u3002<br \/>\n<code>Source<\/code>\u7528\u4e8e\u91c7\u96c6\u6570\u636e\uff0cSource \u662f\u4ea7\u751f\u6570\u636e\u6d41\u7684\u5730\u65b9\uff0c\u540c\u65f6 Source \u4f1a\u5c06\u4ea7\u751f\u7684\u6570\u636e\u6d41\u4f20\u8f93\u5230 Channel\uff0c\u8fd9\u4e2a\u6709\u70b9\u7c7b\u4f3c\u4e8eJava IO \u90e8\u5206\u7684 Channel\u3002<br \/>\n<code>Channel<\/code> \u88ab\u8bbe\u8ba1\u4e3a Event \u4e2d\u8f6c\u4e34\u65f6\u7f13\u51b2\u533a\uff0c\u5b58\u50a8 Source \u6536\u96c6\u5e76\u4e14\u6ca1\u6709\u88abSink \u8bfb\u53d6\u7684 Event\uff0c\u4e3a\u5e73\u8861 Source \u6536\u96c6\u548c Sink \u8bfb\u53d6\u7684\u901f\u5ea6\uff0c\u53ef\u89c6\u4e3a Flume\u5185\u90e8\u7684\u6d88\u606f\u961f\u5217\u3002<br \/>\n<code>Sink<\/code>\u4eceChannel \u6536\u96c6\u6570\u636e\uff0c\u5c06\u6570\u636e\u5199\u5230\u76ee\u6807(\u53ef\u4ee5\u662f\u4e0b\u4e00\u4e2a Source\uff0c\u4e5f\u53ef\u4ee5\u662f HDFS \u6216\u8005 HBase)\u3002<\/p>\n<h1><span class=\"ez-toc-section\" id=\"%E4%BA%8C%E3%80%81%E6%80%8E%E4%B9%88%E7%94%A8\"><\/span>\u4e8c\u3001\u600e\u4e48\u7528<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<p>\u7528\u4e4b\u524d\u5927\u6982\u4e86\u89e3\u5176\u662f\u600e\u4e48\u88ab\u88c5\u4e0a\u7684\u518d\u4e86\u89e3\u662f\u5982\u4f55\u4f7f\u7528\u7684\u3002<\/p>\n<h2><span class=\"ez-toc-section\" id=\"%E5%AE%89%E8%A3%85%E8%BF%87%E7%A8%8B\"><\/span>\u5b89\u88c5\u8fc7\u7a0b<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u7b2c\u4e00\u6b65\u4fee\u6539\u914d\u7f6e\u6587\u4ef6\uff1a\u5728flume-env.sh\u52a0\u5165\u5982\u4e0b\u5185\u5bb9\uff08\u5e95\u5c42\u662fJava\uff09<br \/>\n<code>export JAVA_HOME=\/opt\/jdk1.8<\/code><\/p>\n<pre><code class=\"language-bash line-numbers\">[root@wq1 conf]# pwd\n\/opt\/flume-1.9.0\/conf\n[root@wq1 conf]# ll\n-rw-r--r--. 1 2020 2020 3254 9\u6708   8 11:06 flume-env.sh\n-rw-rw-r--. 1 2020 2020 3107 12\u6708 10 2018 log4j.properties\ndrwxr-xr-x. 2 root     root       78 11\u6708  2 00:30 my_conf\n<\/code><\/pre>\n<p>\u7b2c\u4e8c\u6b65\u6dfb\u52a0\u73af\u5883\u53d8\u91cf(\u76ee\u7684\u662f\u4f7f\u5f97Linux\u7cfb\u7edf\u80fd\u8ba4\u8bc6\u8be5flume\u7ec4\u4ef6)<\/p>\n<pre><code class=\"language-java line-numbers\">echo 'export FLUME_HOME=\/opt\/flume-1.9.0' &gt;&gt; \/etc\/profile\necho 'export PATH=.:<span class=\"katex math inline\">FLUME_HOME\/bin:<\/span>PATH' &gt;&gt; \/etc\/profile\nsource \/etc\/profile\n<\/code><\/pre>\n<h2><span class=\"ez-toc-section\" id=\"%E6%80%8E%E4%B9%88%E4%BD%BF%E7%94%A8\"><\/span>\u600e\u4e48\u4f7f\u7528<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<ul>\n<li>\u4e86\u89e3flume\u5b89\u88c5\u76ee\u5f55<\/li>\n<\/ul>\n<p>\u4f7f\u7528\u524d\u4f60\u9700\u8981\u5173\u6ce8\u4e0b\u9762\u8fd9\u51e0\u4e2a\u76ee\u5f55<br \/>\n<code>bin<\/code>\u76ee\u5f55\u4e0b\u8d1f\u8d23\u6267\u884cflume\u7684\u811a\u672c<br \/>\n<code>conf<\/code>\u6587\u4ef6\u5939\u4e0b\u53c8flume\u7684\u914d\u7f6e\u6587\u4ef6\uff0c\u81ea\u5b9a\u4e49\u7684\u811a\u672c<br \/>\n<code>lib<\/code>\u4e0b\u653e\u7684\u662f\u4e00\u4e9bjar\u5305,\u770b\u5230\u8fd9\u4e9bjar\u5305\u4f60\u4e5f\u5c31\u6e05\u695a\u4e86flume\u7684\u5e95\u5c42\u4ecd\u662fJava\uff0c\u540e\u9762\u81ea\u5b9a\u4e49\u7684\u62e6\u622a\u5668\u6253\u597djar\u5305\u540e\u653e\u5728\u8fd9\u4e2a\u4e0b\u9762\u3002<\/p>\n<pre><code class=\"language-bash line-numbers\">[root@wq1 flume-1.9.0]# ll\n\u603b\u7528\u91cf 168\ndrwxr-xr-x.  2 2020 2020    62 3\u6708  24 16:10 bin\ndrwxr-xr-x.  3 2020 2020   134 3\u6708  24 16:20 conf\ndrwxr-xr-x.  2 root     root      8192 3\u6708  24 21:21 lib\n[root@wq1 bin]# ll\n\u603b\u7528\u91cf 36\n-rwxr-xr-x. 1 2020 2020 12387 11\u6708 16 2017 flume-ng\n-rw-r--r--. 1 2020 2020   936 11\u6708 16 2017 flume-ng.cmd\n-rwxr-xr-x. 1 2020 2020 14176 11\u6708 16 2017 flume-ng.ps1\n[root@wq1 conf]# ll\n\u603b\u7528\u91cf 16\n-rw-r--r--. 1 2020 2020 1661 11\u6708 16 2017 flume-conf.properties.template\n-rw-r--r--. 1 2020 2020 1455 11\u6708 16 2017 flume-env.ps1.template\n-rw-r--r--. 1 2020 2020 1551 3\u6708  24 16:16 flume-env.sh\n-rw-rw-r--. 1 2020 2020 3127 3\u6708  25 15:23 log4j.properties\ndrwxr-xr-x. 2 root     root       29 3\u6708  25 14:39 selfconf\n[root@wq1 lib]# ll\n\u603b\u7528\u91cf 61984\n-rw-r--r--. 1 2020 wuqi2020  346729 9\u6708  12 2018 apache-log4j-extras-1.1.jar\n-rw-rw-r--. 1 2020 2020   18031 9\u6708  12 2018 async-1.4.0.jar\n-rw-rw-r--. 1 2020 2020 1300395 9\u6708  12 2018 asynchbase-1.7.0.jar\n-rw-rw-r--. 1 2020 2020  303139 9\u6708  13 2018 avro-1.7.4.jar\n...\n<\/code><\/pre>\n<ul>\n<li>\u6267\u884c\u547d\u4ee4<\/li>\n<\/ul>\n<pre><code class=\"language-java line-numbers\">#\u5728\u8fd9\u4e2abin\u76ee\u5f55\u4e0b\u6267\u884c\n[root@wq1 bin]# pwd\n\/opt\/flume-1.9.0\/bin\n# \u4f7f\u7528\u957f\u53c2\u6570\n flume-ng agent\n --conf conf # \u6307\u5b9aflume\u5b89\u88c5\u76ee\u5f55\u4e0b\u7684conf\u76ee\u5f55\n --conf-file example.conf # \u7528\u6237\u7f16\u5199\u7684agent\u7684\u914d\u7f6e\u811a\u672c\n --name a1 # \u6307\u5b9a\u542f\u52a8\u7684agent\u7684\u540d\u79f0\n -Dflume.root.logger=INFO,console #\u65e5\u5fd7\u6253\u5230\u63a7\u5236\u53f0\n# \u4f7f\u7528\u7b80\u5199\u7684\u77ed\u53c2\u6570\nflume-ng agent \\\n-c \/opt\/flume-1.9.0\/conf \\\n-f \/opt\/flume-1.9.0\/conf\/my_conf\/natcat_logger.conf \\\n-n a1 \\\n-Dflume.root.logger=INFO,console\n<\/code><\/pre>\n<h3><span class=\"ez-toc-section\" id=\"%E8%A2%AB%E6%89%A7%E8%A1%8C%E7%9A%84%E8%84%9A%E6%9C%AC\"><\/span>\u88ab\u6267\u884c\u7684\u811a\u672c<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<ol>\n<li>flume\u91c7\u96c6\u5355\u4e2a\u6587\u4ef6\u7684\u6570\u636e\u5230kafka\u96c6\u7fa4\u4e0a<\/li>\n<\/ol>\n<pre><code class=\"language-java line-numbers\"># a1\u662f\u6211\u4eec\u81ea\u5b9a\u4e49\u7684agent\u7684\u540d\u5b57\n# a1.sources  a1\u8fd9\u4e2aagent\u5305\u542b\u7684\u591a\u4e2asource\u7684\u540d\u5b57\n# \u5305\u542b\u51e0\u4e2a\u7ec4\u4ef6\u5c31\u5199\u4e00\u4e2a\u540d\u5b57\u7a7a\u683c\u9694\u5f00\na1.sources = r1\na1.sinks = k1\na1.channels = c1\n# \u914d\u7f6esource\na1.sources.r1.type = exec\n#\u8981\u91c7\u96c6\u7684\u6587\u4ef6\na1.sources.r1.command = tail -F \/root\/cdhdata\/products\/dt=26\/products-.1603703609450\n# Describe the sink\n# \u914d\u7f6esink\na1.sinks.k1.channel = c1\na1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink\na1.sinks.k1.kafka.topic = products\na1.sinks.k1.kafka.bootstrap.servers = wq1:9092,wq2:9092,wq3:9092\n# \u6bcf\u4e2a\u6279\u6b21\u5237\u65b0\u5230HDFS\u4e0a\u7684events\u6570\u91cf\uff1b\na1.sinks.k1.kafka.flumeBatchSize = 100\na1.sinks.k1.kafka.producer.acks = 1\na1.sinks.k1.kafka.producer.linger.ms = 1\na1.sinks.k1.kafka.producer.compression.type = snappy\n# Use a channel which buffers events in memory\n# \u914d\u7f6echannel\na1.channels.c1.type = memory\na1.channels.c1.capacity = 1000\n#\u4e8b\u52a1\u5bb9\u91cf\uff0c\u5b83\u5c31\u662fputList\u548ctakeList\u7684\u5bb9\u91cf\u5927\u5c0f\na1.channels.c1.transactionCapacity = 100\n# \u5c06\u7ec4\u4ef6\u8fde\u63a5\u8d77\u6765\na1.sources.r1.channels = c1\na1.sinks.k1.channel = c1\n<\/code><\/pre>\n<ol start=\"2\">\n<li>flume\u91c7\u96c6kafka\u96c6\u7fa4\u7684\u6570\u636e\u5230kafka\u96c6\u7fa4\u4e0a<\/li>\n<\/ol>\n<pre><code class=\"language-properties line-numbers\">a1.sources=r1  \na1.sinks=k1        \na1.channels=c1\n\n# configure source \n# source \u4e3akafka\u7c7b\u578b  \na1.sources.r1.type = org.apache.flume.source.kafka.KafkaSource\n\/\/\u6279\u91cf\u5927\u5c0f-\u4e00\u6b21\na1.sources.r1.batchSize = 5000\n\/\/\u6279\u5904\u7406\u65f6\u95f4-\u6bcf\u9694\u591a\u5c11\u6beb\u79d2\u5904\u7406\u7684\u65f6\u95f4\na1.sources.r1.batchDurationMillis = 2000\na1.sources.r1.kafka.bootstrap.servers = cdh0203:9092,cdh0204:9092,cdh0205:9092\n\/topic\na1.sources.r1.kafka.topics = topic01-categories\n#1earliest \u5f53\u5404\u5206\u533a\u4e0b\u6709\u5df2\u63d0\u4ea4\u7684offset\u65f6\uff0c\u4ece\u63d0\u4ea4\u7684offset\u5f00\u59cb\u6d88\u8d39\uff1b\u65e0\u63d0\u4ea4\u7684offset\u65f6\uff0c\u4ece\u5934\u5f00\u59cb\u6d88\u8d39\n#latest \u5f53\u5404\u5206\u533a\u4e0b\u6709\u5df2\u63d0\u4ea4\u7684offset\u65f6\uff0c\u4ece\u63d0\u4ea4\u7684offset\u5f00\u59cb\u6d88\u8d39\uff1b\u65e0\u63d0\u4ea4\u7684offset\u65f6\uff0c\u6d88\u8d39\u65b0\u4ea7\u751f\u7684\u8be5\u5206\u533a\u4e0b\u7684\u6570\u636e\na1.sources.r1.kafka.consumer.auto.offset.reset = earliest\n\/\/\u7ec4id,\u540d\u5b57\u968f\u610f\uff0c\u540c\u7ec4\u53ef\u6d88\u8d39\u4e00\u4e2atopic\na1.sources.r1.kafka.consumer.group.id = ds001\n# configure sink\na1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink\n\/\/\u81ea\u5df1\u96c6\u7fa4\u7684topic\uff0c\u4f46\u5c3d\u91cf\u4e0e\u6570\u636e\u6e90topic\u4e00\u81f4\na1.sinks.k1.kafka.topic = topic01-categories\na1.sinks.k1.kafka.bootstrap.servers = cdh0053:9092,cdh0054:9092\n#\u4e00\u6b21\u5904\u7406\u6d88\u606f\u7684\u6570\u91cf\na1.sinks.k1.kafka.flumeBatchSize = 2000\n#\u8bbe\u7f6e\u4e3a1\u5219leader\u63a5\u6536\u5230\u6570\u636e\u540e\u5c31\u5f00\u59cb\u4f20\u8f93\na1.sinks.k1.kafka.producer.acks = 1\n#\u6bcf\u96941\u79d2\u63a5\u6536\u4e00\u6b21\u6570\u636e\na1.sinks.k1.kafka.producer.linger.ms = 1\n#\u4e0d\u652f\u6301\u538b\u7f29\na1.sinks.k1.kafka.producer.compression.type = snappy\n#\u6279\u5904\u7406\u5927\u5c0f\na1.sinks.kafka-sink.kafka.producer.batch.size = 1048576\n# configure channels  \n#\u57fa\u4e8e\u5185\u5b58\u4f20\u8f93\u6570\u636e\na1.channels.c1.type = memory\na1.channels.c1.capacity = 10000\na1.channels.c1.byteCapacityBufferPercentage = 20\na1.channels.c1.transactionCapacity = 10000\na1.channels.c1.byteCapacity = 800000\na1.sources.r1.channels = c1\na1.sinks.k1.channel = c1\n<\/code><\/pre>\n<p>\u6211\u4eec\u5df2\u7ecf\u77e5\u9053\u4e86flume\u5728\u91c7\u96c6\u6570\u636e\u7684\u57fa\u672c\u5355\u4f4d\u662fevent\uff0c\u5b83\u53c8\u53ef\u5206\u4e09\u4e2a\u90e8\u5206\uff0csouce\u7aef\u7528\u6765\u91c7\u96c6\u6570\u636e\uff0cchanneld\u7684\u4f5c\u7528\u7c7b\u4f3c\u4e00\u4e2a\u7ba1\u9053\u8fde\u63a5\u8f93\u5165\u7aef\u4e0e\u8f93\u51fa\u7aef\uff0csink\u7aef\u8d1f\u8d23\u5c06\u91c7\u96c6\u5230\u7684\u6570\u636e\u6253\u5230\u6307\u5b9a\u5730\u65b9\u3002\u90a3\u4e48\u4e8b\u60c5\u5c31\u53d8\u7684\u7b80\u5355\u4e86\uff0c\u5982\u679c\u4f60\u7684\u6570\u636e\u6765\u6e90\u662fkafka\uff0c\u90a3\u4e48\u4f60\u9996\u5148\u8981\u627e\u5230kafka-source\uff0c\u5982\u679c\u4f60\u91c7\u96c6\u6765\u7684\u6570\u636e\u8981\u6253\u5230hdfs\u4e0a\uff0c\u90a3\u4e48\u4f60\u5c31\u5728flume\u7684\u5b98\u7f51\u4e0a\u627ehdfs-sink\u7684\u914d\u7f6e\u5373\u53ef\uff0c\u6700\u540e\u6309\u7167\u81ea\u5df1\u7684\u5177\u4f53\u7ec6\u8282\u8981\u6c42\u5bf9\u914d\u7f6e\u6587\u4ef6\u8fdb\u884c\u4fee\u6539\uff0c\u90a3\u4e48\u4f60\u7684\u6267\u884c\u811a\u672c\u5c31\u5199\u597d\u4e86\u3002<\/p>\n<p><a class=\"wp-editor-md-post-content-link\" href=\"http:\/\/flume.apache.org\/releases\/content\/1.9.0\/FlumeUserGuide.html#flume-sources\" title=\"\u70b9\u51fb\u67e5\u770bflume\u5b98\u7f51\u7684\u914d\u7f6e\">\u70b9\u51fb\u67e5\u770bflume\u5b98\u7f51\u7684\u914d\u7f6e<\/a><\/p>\n<blockquote><p>\n  \u81ea\u5b9a\u4e49flume\u62e6\u622a\u5668\n<\/p><\/blockquote>\n<p>\u62e6\u622a\u5668\u662f\u7b80\u5355\u7684\u63d2\u4ef6\u5f0f\u7ec4\u4ef6\uff0c\u8bbe\u7f6e\u5728source\u548cchannel\u4e4b\u95f4\u3002source\u63a5\u6536\u5230\u7684\u65f6\u95f4\uff0c\u5728\u5199\u5165channel\u4e4b\u524d\uff0c\u62e6\u622a\u5668\u90fd\u53ef\u4ee5\u8fdb\u884c\u8f6c\u6362\u6216\u8005\u5220\u9664\u8fd9\u4e9b\u4e8b\u4ef6\u3002\u6bcf\u4e2a\u62e6\u622a\u5668\u53ea\u5904\u7406\u540c\u4e00\u4e2asource\u63a5\u6536\u5230\u7684\u4e8b\u4ef6\u3002\u53ef\u4ee5\u81ea\u5b9a\u4e49\u62e6\u622a\u5668\u3002<br \/>\n\u5728ideal\u4e0a\u5f00\u59cb\u7f16\u5199\u62e6\u622a\u5668\u7684\u4ee3\u7801<\/p>\n<p>\u9700\u6c42\u8bf4\u660e\uff1a\u5bf9\u4e8e\u4e0b\u9762\u8fd9\u79cd\u6587\u4ef6\u6211\u4eec\u53ea\u8981\u72b6\u6001\u7801\u662f404\u7684\u6587\u4ef6\uff0c\u6211\u4eec\u6309\u7167\u7a7a\u683c\u8fdb\u884c\u62c6\u5206\u6570\u5230\u7b2c9\u4e2a\u5c31\u662f\u72b6\u6001\u7801<\/p>\n<pre><code class=\"language-text line-numbers\">1.1.1.1 - - [26\/Oct\/2020:11:57:02 +0800] \"GET \/ HTTP\/2.0\" 404 9631 \"-\" \"Mozilla\/5.0 (Windows NT 6.1; WOW64) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/78.0.3904.108 Safari\/537.36\" \"-\"\n<\/code><\/pre>\n<p>\u7b2c\u4e00\u6b65\uff1a\u5f15\u5165\u4f9d\u8d56<\/p>\n<pre data-language=XML><code class=\"language-markup line-numbers\"> &lt;!-- \u6839\u636eLinux\u4e0aflume\u7684\u7248\u672c\u6765 --&gt;\n&lt;dependency&gt;\n      &lt;groupId&gt;org.apache.flume&lt;\/groupId&gt;\n      &lt;artifactId&gt;flume-ng-core&lt;\/artifactId&gt;\n      &lt;version&gt;1.9.0&lt;\/version&gt;\n&lt;\/dependency&gt;\n<\/code><\/pre>\n<p>\u7b2c\u4e8c\u6b65\uff1a\u5b9e\u73b0Interceptor,\u5199\u5185\u90e8\u7c7b<\/p>\n<pre><code class=\"language-java line-numbers\">package com.flume.interceptor;\n\nimport org.apache.flume.Context;\nimport org.apache.flume.Event;\nimport org.apache.flume.interceptor.Interceptor;\nimport org.slf4j.Logger;\nimport org.slf4j.LoggerFactory;\n\n\nimport java.util.ArrayList;\nimport java.util.List;\n\n\/**\n * @Author:wuqi\n * @DATE: 2021\/3\/24 0024\n * @TIME: 18:36\n * @PROJECT_NAME: flumelanjie\n *\/\npublic class Mylanjieqi implements Interceptor {\n\n    \/\/\u6253\u5370\u65e5\u5fd7\uff0c\u4fbf\u4e8e\u6d4b\u8bd5\u65b9\u6cd5\u7684\u6267\u884c\u987a\u5e8f\n    private static final Logger logger = LoggerFactory.getLogger(Mylanjieqi.class);\n    @Override\n    public void initialize() {\n        logger.info(\"----------\u81ea\u5b9a\u4e49\u62e6\u622a\u5668\u521d\u59cb\u5316---------------------\");\n    }\n    \/\/\u8bfb\u53d6\u6570\u636e\uff0c\u5b9a\u4e49\u62e6\u622a\u89c4\u5219\n    @Override\n    public Event intercept(Event event) {\n        logger.info(\"----------\u81ea\u5b9a\u4e49\u62e6\u622a\u89c4\u5219---------------------\");\n        byte[] body = event.getBody();\n        String str = new String(body);\n        String[] split = str.split(\"\\\\s+\");\n        logger.info(\"----------\u65e7envent---------------------\"+event.hashCode());\n        if (split[8].equals(\"404\")){\n            event.setBody(str.getBytes());\n            logger.info(\"----------\u65b0\u7684envent---------------------\"+event.hashCode());\n            return event;\n        }\n        return null;\n    }\n    \/\/ \u63a5\u6536\u88ab\u8fc7\u6ee4\u4e8b\u4ef6\u96c6\u5408\n    @Override\n    public List&lt;Event&gt; intercept(List&lt;Event&gt; list) {\n        logger.info(\"----------\u63a5\u6536\u88ab\u8fc7\u6ee4\u4e8b\u4ef6\u96c6\u5408---------------------\");\n        List&lt;Event&gt; result = new ArrayList&lt;Event&gt;();\n        Event event;\n        for (Event e:list){\n           event = intercept(e);\n            if (event!=null){\n                result.add(event);\n            }\n        }\n        return result;\n    }\n    @Override\n    public void close() {\n        logger.info(\"----------\u62e6\u622a\u5668\u5173\u95ed---------------------\");\n    }\n   public static class Builder implements Interceptor.Builder {\n        \/\/ \u83b7\u53d6\u914d\u7f6e\u6587\u4ef6\u7684\u5c5e\u6027\n        @Override\n        public Interceptor build() {\n            logger.info(\"----------build()\u6267\u884c---------------------\");\n            return new Mylanjieqi();\n        }\n        @Override\n        public void configure(Context context) {\n            logger.info(\"----------configure(Context context)\u6267\u884c---------------------\");\n        }\n    }\n}\n<\/code><\/pre>\n<p>\u7b2c\u4e09\u6b65\uff1aideal\u6253jar\u5305<br \/>\n\u65b0\u5efamaven\u9879\u76ee&#8211;\u300bpom.xml\u6539\u8fd9\u4e2apackaging\u4e3ajar&#8211;\u300bLifeCycle.clean.install<\/p>\n<pre data-language=XML><code class=\"language-markup line-numbers\">&lt;groupId&gt;org.example&lt;\/groupId&gt;\n  &lt;artifactId&gt;laner&lt;\/artifactId&gt;\n  &lt;version&gt;1.0-SNAPSHOT&lt;\/version&gt;\n  &lt;packaging&gt;jar&lt;\/packaging&gt;\n<\/code><\/pre>\n<p>\u7b2c\u56db\u6b65\uff1aflume\u7684\u914d\u7f6e<br \/>\nflume-hdfs.conf<\/p>\n<pre><code class=\"language-properties line-numbers\"># Name the components on this agent\n# a1\u662f\u6211\u4eec\u81ea\u5b9a\u4e49\u7684agent\u7684\u540d\u5b57\n# a1.sources  a1\u8fd9\u4e2aagent\u5305\u542b\u7684\u591a\u4e2asource\u7684\u540d\u5b57\n# \u5305\u542b\u51e0\u4e2a\u7ec4\u4ef6\u5c31\u5199\u4e00\u4e2a\u540d\u5b57\u7a7a\u683c\u9694\u5f00\na1.sources = r1\na1.sinks = k1\na1.channels = c1\n# Describe\/configure the source\n# \u914d\u7f6esource\na1.sources.r1.type = exec\n#\u91c7\u96c6\u7684\u6570\u636e\na1.sources.r1.command = tail -F \/root\/w.text\n# Describe the sink\n# \u914d\u7f6esink\na1.sinks.k1.type = hdfs\na1.sinks.k1.hdfs.path = hdfs:\/\/192.168.236.236:9000\/laner\na1.sinks.k1.hdfs.filePrefix = laner\n#\u62e6\u622a\u5668\u540d\u79f0,\u67b6\u5305\u7684\u540d\na1.sources.r1.interceptors = laner\n#\u62e6\u622a\u5668\u5bf9\u8c61\u521b\u5efa                           \u7c7b\u8def\u5f84.\u7c7b\u540d<span class=\"katex math inline\">\u5185\u90e8\u7c7b\u540d\na1.sources.r1.interceptors.laner.type = com.flume.interceptor.Mylanjieqi<\/span>Builder\n#hdfs sink\u95f4\u9694\u591a\u957f\u5c06\u4e34\u65f6\u6587\u4ef6\u6eda\u52a8\u6210\u6700\u7ec8\u76ee\u6807\u6587\u4ef6\uff0c\u5355\u4f4d\uff1a\u79d2\uff1b\na1.sinks.k1.hdfs.rollInterval = 0\n#\u6bcf\u591a\u5c11M\u751f\u6210\u4e00\u4e2a\u6587\u4ef6\uff0c\u662f0\u7684\u8bdd\u5c31\u4e0e\u65e0\u5173\na1.sinks.k1.hdfs.rollSize = 0\n# \u5f53events\u6570\u636e\u8fbe\u5230\u8be5\u6570\u91cf\u65f6\u5019\uff0c\u5c06\u4e34\u65f6\u6587\u4ef6\u6eda\u52a8\u6210\u76ee\u6807\u6587\u4ef6\uff1b\na1.sinks.k1.hdfs.rollCount = 0\n# \u6587\u4ef6\u7684\u683c\u5f0f\u3002\u5305\u542b\uff1aText, Writable\uff08\u9ed8\u8ba4\uff09\na1.sinks.k1.hdfs.writeFormat = text\n#\u5199sequence\u6587\u4ef6\u7684\u683c\u5f0f\u3002\u5305\u542b\uff1aText, Writable\uff08\u9ed8\u8ba4\uff09\na1.sinks.k1.hdfs.fileType = DataStream\n#\u5305\u62ec\uff1aSequenceFile, DataStream,CompressedStream \u4f7f\u7528DataStream\u65f6\u5019\uff0c\u6587\u4ef6\u4e0d\u4f1a\u88ab\u538b\u7f29\n# Use a channel which buffers events in memory\n# \u914d\u7f6echannel\na1.channels.c1.type = memory\na1.channels.c1.capacity = 100000\na1.channels.c1.transactionCapacity = 10000\n# \u5c06\u7ec4\u4ef6\u8fde\u63a5\u8d77\u6765\na1.sources.r1.channels = c1\na1.sinks.k1.channel = c1\n<\/code><\/pre>\n<p>\u6253\u597d\u7684\u67b6\u5305\u653e\u5230flume\u7684lib\u4e0b\u5c31\u884c<br \/>\n\u672c\u7bc7\u6587\u7ae0\u672b\u4e5f\u6709flume\u7684\u914d\u7f6e\u5b9e\u4f8b<\/p>\n<blockquote><p>\n  flume\u91c7\u96c6\u591a\u4e2a\u6587\u4ef6\u5939\u4e0b\u7684\u6570\u636e\u5e76\u5728hdfs\u4ea7\u751f\u4e0d\u540c\u7684\u6587\u4ef6\u5939\n<\/p><\/blockquote>\n<p>\u73b0\u5728\u673a\u5668\u4e0a\u6709\u516d\u4e2a\u6587\u4ef6\u5939\uff0c\u73b0\u5728\u8981\u5c06\u8fd9\u4e9b\u6587\u4ef6\u91c7\u96c6\u5230hdfs\u4e0a\uff0c\u4e5f\u5c31\u662fhdfs\u4e0a\u4f1a\u6709\u516d\u4e2a\u6587\u4ef6\u5939\uff0c\u6bcf\u4e2a\u6587\u4ef6\u5939\u4e0b\u82e5\u5e72\u4e2a\u5757<\/p>\n<ol>\n<li>\u6570\u636e\u6e90\u5982\u4e0b<\/li>\n<\/ol>\n<pre><code class=\"language-bash line-numbers\">[root@wq1 ~]# du -h -d1 shoppingmallonline\/\n4.0K    shoppingmallonline\/categories\n146M    shoppingmallonline\/custom\n1.4G    shoppingmallonline\/employee\n1.2G    shoppingmallonline\/order\n398M    shoppingmallonline\/orderdetail\n416K    shoppingmallonline\/products\n<\/code><\/pre>\n<ol start=\"2\">\n<li>flume\u811a\u672c\u6587\u4ef6<\/li>\n<\/ol>\n<p>\u56e0\u4e3a\u6211\u60f3\u901a\u8fc7\u4e00\u4e2a\u811a\u672c\u628a\u516d\u4e2a\u6587\u4ef6\u5939\u4e0b\u7684\u6587\u4ef6\u901a\u8fc7flume\u4f20\u5230hdfs\u7684\u516d\u4e2a\u4e0d\u540c\u6587\u4ef6\u5939\u4e0b\uff0c\u6240\u4ee5\u8fd9\u4e48\u5199\uff0c\u5206\u522b\u5199\u516d\u4e2a\u811a\u672c\u8fd0\u884c\u516d\u6b21\u4e5f\u53ef\u4ee5\u8fbe\u5230\u76ee\u7684\uff1b<\/p>\n<pre><code class=\"language-bash line-numbers\"># Name the components on this agent\n#\u516d\u4e2asource\na2.sources = r1 r2 r3 r4 r5 r6\na2.sinks = k1\na2.channels = c1\n\n# Describe\/configure the source\na2.sources.r1.type=TAILDIR\n#\u5982\u679c\u8be5\u6587\u4ef6\u4f4d\u7f6e\u5185\u5bb9\u6ca1\u6709\u53d1\u751f\u53d8\u5316\uff0c\u5219flume\u5728\u65ad\u5f00\u540e\u53ef\u4ee5\u4ece\u4e0a\u6b21\u65ad\u5f00\u5904\u7ee7\u7eed\u91c7\u96c6\na2.sources.r1.positionFile = \/root\/opt\/flume-1.9.0\/data\/taildir1_position.json\n#\u6587\u4ef6\u7684\u7ec4\uff0c\u53ef\u4ee5\u5b9a\u4e49\u591a\u79cd,\u6b64\u65f6\u662f\u4e00\u4e2asource\u4e0b\u4e00\u4e2a\u7ec4\u5904\u7406\u4e00\u79cd\u7c7b\u578b\u7684\u6587\u4ef6\uff0c\u5b9a\u4e49\u591a\u4e2a\u7ec4\u7684\u8bdd\u5c31\u53ef\u4ee5\u5206\u522b\u5904\u7406\u591a\u79cd\u6587\u4ef6\na2.sources.r1.filegroups = f1\n#\u8be5\u7ec4\u5904\u7406\u7684\u662f\u6587\u4ef6\u5f00\u5934\u90fd\u4e3acategories-.\u7684\u6587\u4ef6\na2.sources.r1.filegroups.f1 = \/root\/shoppingmallonline\/categories\/dt=26\/categories-.*\n#\u62e6\u622a\u5668\u7684\u540d\uff0c\u4e0e\u4e0b\u9762\u8fd9\u4e2a\u540d\u4fdd\u6301\u4e00\u81f4\u5c31\u884c\na2.sources.r1.interceptors = i1\n#\u8868\u793a\u76ee\u524d\u4f7f\u7528\u7684\u662f\uff0c\u9759\u6001\u62e6\u622a\u5668\uff08\u5c5e\u4e8eflume\u5185\u7f6e\u62e6\u622a\u5668\u7684\u4e00\u79cd\uff09\uff1a\u5c06k\/v\u63d2\u5165\u5230\u4e8b\u4ef6\u7684\u62a5\u5934\u4e2d\uff0c\u6b64\u5904\u6211\u7528\u6765\u5b9e\u73b0hdfs\u4e0d\u540c\u6587\u4ef6\u5939\u540d\u7684\u76ee\u7684\na2.sources.r1.interceptors.i1.type = static\na2.sources.r1.interceptors.i1.key = type\na2.sources.r1.interceptors.i1.value = categories\n\na2.sources.r1.type=TAILDIR\na2.sources.r2.filegroups = f1\na2.sources.r2.filegroups.f1 = \/root\/shoppingmallonline\/custom\/dt=26\/customers-.*\na2.sources.r2.interceptors = i2\na2.sources.r2.interceptors.i2.type = static\na2.sources.r2.interceptors.i2.key = type\na2.sources.r2.interceptors.i2.value = custom\n\na2.sources.r1.type=TAILDIR\na2.sources.r3.filegroups = f1\na2.sources.r3.filegroups.f1 = \/root\/shoppingmallonline\/employee\/dt=26\/employees-.*\na2.sources.r3.interceptors = i3\na2.sources.r3.interceptors.i3.type = static\na2.sources.r3.interceptors.i3.key = type\na2.sources.r3.interceptors.i3.value = employee\n\na2.sources.r1.type=TAILDIR\na2.sources.r4.filegroups = f1\na2.sources.r4.filegroups.f1 = \/root\/shoppingmallonline\/order\/dt=26\/orders-.*\na2.sources.r4.interceptors = i4\na2.sources.r4.interceptors.i4.type = static\na2.sources.r4.interceptors.i4.key = type\na2.sources.r4.interceptors.i4.value = order\n\na2.sources.r1.type=TAILDIR\na2.sources.r5.filegroups = f1\na2.sources.r5.filegroups.f1 = \/root\/shoppingmallonline\/orderdetail\/dt=26\/orderdetails-.*\na2.sources.r5.interceptors = i5\na2.sources.r5.interceptors.i5.type = static\na2.sources.r5.interceptors.i5.key = type\na2.sources.r5.interceptors.i5.value = orderdetail\n\na2.sources.r1.type=TAILDIR\na2.sources.r6.filegroups = f1\na2.sources.r6.filegroups.f1 = \/root\/shoppingmallonline\/products\/dt=26\/products-.*\na2.sources.r6.interceptors = i6\na2.sources.r6.interceptors.i6.type = static\na2.sources.r6.interceptors.i6.key = type\na2.sources.r6.interceptors.i6.value = products\n\n\n#\u5177\u4f53\u5b9a\u4e49sink\na2.sinks.k1.type = hdfs\na2.sinks.k1.hdfs.path = hdfs:\/\/192.168.236.236:9000\/source-six\/%{type}\n#\u6307\u5b9a\u6587\u4ef6\u524d\u7f00\na2.sinks.k1.hdfs.filePrefix = %{type}-\n#\u4e0d\u538b\u7f29\na2.sinks.k1.hdfs.fileType = DataStream\n#\u5982\u679c\u538b\u7f29\u6307\u5b9a\u538b\u7f29\u7684\u65b9\u5f0f\n#\u5982\u679c\u538b\u7f29\u5b58\u50a8\u7684\u8bddHDFS\u4e0a\u7684\u6587\u4ef6\u8fbe\u5230128M\u65f6\u751f\u6210\u4e00\u4e2a\u6587\u4ef6\n#\u6ce8\u610f\u662f\u538b\u7f29\u524d\u5927\u5c0f\u4e3a128\u751f\u6210\u4e00\u4e2a\u6587\u4ef6\uff0c\u7136\u540e\u538b\u7f29\u5b58\u50a8\u3002\n#a2.sinks.k1.hdfs.fileType = CompressedStream\n#a2.sinks.k1.hdfs.codeC = gzip\n#\u4e0d\u6309\u7167\u6761\u6570\u751f\u6210\u6587\u4ef6\na2.sinks.k1.hdfs.rollCount = 0\na2.sinks.k1.hdfs.rollSize = 134217728\na2.sinks.k1.hdfs.rollInterval = 0\n\n\n# Use a channel which buffers events in memory\na2.channels.c1.type = memory\na2.channels.c1.capacity = 1000\na2.channels.c1.transactionCapacity = 100\n\n# Bind the source and sink to the channel\na2.sources.r1.channels = c1\na2.sources.r2.channels = c1\na2.sources.r3.channels = c1\na2.sources.r4.channels = c1\na2.sources.r5.channels = c1\na2.sources.r6.channels = c1\na2.sinks.k1.channel = c1\n<\/code><\/pre>\n<ol start=\"3\">\n<li>\u6267\u884c<\/li>\n<\/ol>\n<pre><code class=\"language-bash line-numbers\">flume-ng agent \\\n--conf \/opt\/flume-1.9.0\/conf \\\n--conf-file \/opt\/flume-1.9.0\/conf\/selfconf\/flume1-hdfs.conf \\\n--name a2 \\\n-Dflume.monitoring.type=http \\\n-Dflume.monitoring.port=5653 \\\n-Dflume.root.logger=INFO,console\n<\/code><\/pre>\n<ol start=\"4\">\n<li>\u8fc7\u7a0b\u89c2\u5bdf<\/li>\n<\/ol>\n<p>WebUi\u4e0a\u7684json\u6587\u4ef6<\/p>\n<pre><code class=\"language-json line-numbers\">{\"SINK.k1\":{\"ConnectionCreatedCount\":\"27\",\n            \"BatchCompleteCount\":\"346812\",\n            \"EventWriteFail\":\"0\",\n            \"BatchEmptyCount\":\"250\",\n            \"EventDrainAttemptCount\":\"34681272\",\n            \"StartTime\":\"1616938071561\",\n            \"BatchUnderflowCount\":\"1\",\n            \"ChannelReadFail\":\"0\",\n            \"ConnectionFailedCount\":\"0\",\n            \"ConnectionClosedCount\":\"22\",\n            \"Type\":\"SINK\",\n            \"EventDrainSuccessCount\":\"34681272\",\n            \"StopTime\":\"0\"},\n \"SOURCE.r3\":{\"AppendBatchAcceptedCount\":\"86932\",\n              \"GenericProcessingFail\":\"0\",\n              \"EventAcceptedCount\":\"8692898\",\n              \"AppendReceivedCount\":\"0\",\n              \"StartTime\":\"1616938071941\",\n              \"AppendBatchReceivedCount\":\"86932\",\n              \"ChannelWriteFail\":\"0\",\n              \"EventReceivedCount\":\"8692898\",\n              \"EventReadFail\":\"0\",\n              \"Type\":\"SOURCE\",\n              \"AppendAcceptedCount\":\"0\",\n              \"OpenConnectionCount\":\"0\",\n              \"StopTime\":\"0\"},\n \"SOURCE.r4\":{\"AppendBatchAcceptedCount\":\"99923\",\n              \"GenericProcessingFail\":\"0\",\n              \"EventAcceptedCount\":\"9992082\",\n              \"AppendReceivedCount\":\"0\",\n              \"StartTime\":\"1616938071858\",\n              \"AppendBatchReceivedCount\":\"99925\",\n              \"ChannelWriteFail\":\"2\",\"EventReceivedCount\":\"9992282\",\n              \"EventReadFail\":\"0\",\"Type\":\"SOURCE\",\n              \"AppendAcceptedCount\":\"0\",\n              \"OpenConnectionCount\":\"0\",\n              \"StopTime\":\"0\"},\n \"SOURCE.r5\":{\"AppendBatchAcceptedCount\":\"149937\",\n              \"GenericProcessingFail\":\"0\",\n              \"EventAcceptedCount\":\"14993597\",\n              \"AppendReceivedCount\":\"0\",\n              \"StartTime\":\"1616938071698\",\n              \"AppendBatchReceivedCount\":\"149938\",\n              \"ChannelWriteFail\":\"1\",\n              \"EventReceivedCount\":\"14993697\",\n              \"EventReadFail\":\"0\",\"Type\":\"SOURCE\",\n              \"AppendAcceptedCount\":\"0\",\n              \"OpenConnectionCount\":\"0\",\n              \"StopTime\":\"0\"},\n \"CHANNEL.c1\":{\"ChannelCapacity\":\"1000\",\n               \"ChannelFillPercentage\":\"0.0\",\n               \"Type\":\"CHANNEL\",\n               \"ChannelSize\":\"0\",\n               \"EventTakeSuccessCount\":\"34681272\",\n               \"EventTakeAttemptCount\":\"34681523\",\n               \"StartTime\":\"1616938071553\",\n               \"EventPutAttemptCount\":\"34681672\",\n               \"EventPutSuccessCount\":\"34681272\",\n               \"StopTime\":\"0\"},\n \"SOURCE.r6\":{\"AppendBatchAcceptedCount\":\"100\",\n              \"GenericProcessingFail\":\"0\",\n              \"EventAcceptedCount\":\"10000\",\n              \"AppendReceivedCount\":\"0\",\n              \"StartTime\":\"1616938071711\",\n              \"AppendBatchReceivedCount\":\"100\",\n              \"ChannelWriteFail\":\"0\",\n              \"EventReceivedCount\":\"10000\",\n              \"EventReadFail\":\"0\",\"Type\":\"SOURCE\",\n              \"AppendAcceptedCount\":\"0\",\n              \"OpenConnectionCount\":\"0\",\n              \"StopTime\":\"0\"},\n \"SOURCE.r1\":{\"AppendBatchAcceptedCount\":\"0\",\n              \"GenericProcessingFail\":\"0\",\n              \"EventAcceptedCount\":\"0\",\n              \"AppendReceivedCount\":\"0\",\"StartTime\":\"1616938071859\",\n              \"AppendBatchReceivedCount\":\"0\",\n              \"ChannelWriteFail\":\"0\",\n              \"EventReceivedCount\":\"0\",\n              \"EventReadFail\":\"0\",\n              \"Type\":\"SOURCE\",\"AppendAcceptedCount\":\"0\",\n              \"OpenConnectionCount\":\"0\",\n              \"StopTime\":\"0\"},\n \"SOURCE.r2\":{\"AppendBatchAcceptedCount\":\"9927\",\n              \"GenericProcessingFail\":\"0\",\n              \"EventAcceptedCount\":\"992695\",\n              \"AppendReceivedCount\":\"0\",\n              \"StartTime\":\"1616938071718\",\n              \"AppendBatchReceivedCount\":\"9928\",\n              \"ChannelWriteFail\":\"1\",\n              \"EventReceivedCount\":\"992795\",\n              \"EventReadFail\":\"0\",\n              \"Type\":\"SOURCE\",\n              \"AppendAcceptedCount\":\"0\",\n              \"OpenConnectionCount\":\"0\",\n              \"StopTime\":\"0\"}}\n<\/code><\/pre>\n<p>\u4e0a\u9762\u7684\u4e2a\u5c5e\u6027\u7684\u542b\u4e49\u5bf9\u7167<\/p>\n<pre><code class=\"language-json line-numbers\">{\n    \"SOURCE.src-1\":{\n        \"OpenConnectionCount\":\"0\",      \/\/\u76ee\u524d\u4e0e\u5ba2\u6237\u7aef\u6216sink\u4fdd\u6301\u8fde\u63a5\u7684\u603b\u6570\u91cf(\u76ee\u524d\u53ea\u6709avro source\u5c55\u73b0\u8be5\u5ea6\u91cf)\n        \"Type\":\"SOURCE\",                    \n        \"AppendBatchAcceptedCount\":\"1355\",  \/\/\u6210\u529f\u63d0\u4ea4\u5230channel\u7684\u6279\u6b21\u7684\u603b\u6570\u91cf\n        \"AppendBatchReceivedCount\":\"1355\",  \/\/\u63a5\u6536\u5230\u4e8b\u4ef6\u6279\u6b21\u7684\u603b\u6570\u91cf\n        \"EventAcceptedCount\":\"28286\",   \/\/\u6210\u529f\u5199\u51fa\u5230channel\u7684\u4e8b\u4ef6\u603b\u6570\u91cf\uff0c\u4e14source\u8fd4\u56desuccess\u7ed9\u521b\u5efa\u4e8b\u4ef6\u7684sink\u6216RPC\u5ba2\u6237\u7aef\u7cfb\u7edf\n        \"AppendReceivedCount\":\"0\",      \/\/\u6bcf\u6279\u53ea\u6709\u4e00\u4e2a\u4e8b\u4ef6\u7684\u4e8b\u4ef6\u603b\u6570\u91cf(\u4e0eRPC\u8c03\u7528\u4e2d\u7684\u4e00\u4e2aappend\u8c03\u7528\u76f8\u7b49)\n        \"StopTime\":\"0\",         \/\/source\u505c\u6b62\u65f6\u81eaEpoch\u4ee5\u6765\u7684\u6beb\u79d2\u503c\u65f6\u95f4\n        \"StartTime\":\"1442566410435\",    \/\/source\u542f\u52a8\u65f6\u81eaEpoch\u4ee5\u6765\u7684\u6beb\u79d2\u503c\u65f6\u95f4\n        \"EventReceivedCount\":\"28286\",   \/\/\u76ee\u524d\u4e3a\u6b62source\u5df2\u7ecf\u63a5\u6536\u5230\u7684\u4e8b\u4ef6\u603b\u6570\u91cf\n        \"AppendAcceptedCount\":\"0\"       \/\/\u5355\u72ec\u4f20\u5165\u7684\u4e8b\u4ef6\u5230Channel\u4e14\u6210\u529f\u8fd4\u56de\u7684\u4e8b\u4ef6\u603b\u6570\u91cf\n    },\n    \"CHANNEL.ch-1\":{\n        \"EventPutSuccessCount\":\"28286\", \/\/\u6210\u529f\u5199\u5165channel\u4e14\u63d0\u4ea4\u7684\u4e8b\u4ef6\u603b\u6570\u91cf\n        \"ChannelFillPercentage\":\"0.0\",  \/\/channel\u6ee1\u65f6\u7684\u767e\u5206\u6bd4\n        \"Type\":\"CHANNEL\",\n        \"StopTime\":\"0\",         \/\/channel\u505c\u6b62\u65f6\u81eaEpoch\u4ee5\u6765\u7684\u6beb\u79d2\u503c\u65f6\u95f4\n        \"EventPutAttemptCount\":\"28286\", \/\/Source\u5c1d\u8bd5\u5199\u5165Channe\u7684\u4e8b\u4ef6\u603b\u6570\u91cf\n        \"ChannelSize\":\"0\",          \/\/\u76ee\u524dchannel\u4e2d\u4e8b\u4ef6\u7684\u603b\u6570\u91cf\n        \"StartTime\":\"1442566410326\",    \/\/channel\u542f\u52a8\u65f6\u81eaEpoch\u4ee5\u6765\u7684\u6beb\u79d2\u503c\u65f6\u95f4\n        \"EventTakeSuccessCount\":\"28286\",    \/\/sink\u6210\u529f\u8bfb\u53d6\u7684\u4e8b\u4ef6\u7684\u603b\u6570\u91cf\n        \"ChannelCapacity\":\"1000000\",       \/\/channel\u7684\u5bb9\u91cf\n        \"EventTakeAttemptCount\":\"313734329512\" \/\/sink\u5c1d\u8bd5\u4ecechannel\u62c9\u53d6\u4e8b\u4ef6\u7684\u603b\u6570\u91cf\u3002\u8fd9\u4e0d\u610f\u5473\u7740\u6bcf\u6b21\u4e8b\u4ef6\u90fd\u88ab\u8fd4\u56de\uff0c\u56e0\u4e3asink\u62c9\u53d6\u7684\u65f6\u5019channel\u53ef\u80fd\u6ca1\u6709\u4efb\u4f55\u6570\u636e\n    },\n    \"SINK.sink-1\":{\n        \"Type\":\"SINK\",\n        \"ConnectionClosedCount\":\"0\",    \/\/\u4e0b\u4e00\u9636\u6bb5\u6216\u5b58\u50a8\u7cfb\u7edf\u5173\u95ed\u7684\u8fde\u63a5\u6570\u91cf(\u5982\u5728HDFS\u4e2d\u5173\u95ed\u4e00\u4e2a\u6587\u4ef6)\n        \"EventDrainSuccessCount\":\"28286\",   \/\/sink\u6210\u529f\u5199\u51fa\u5230\u5b58\u50a8\u7684\u4e8b\u4ef6\u603b\u6570\u91cf\n        \"KafkaEventSendTimer\":\"482493\",    \n        \"BatchCompleteCount\":\"0\",       \/\/\u4e0e\u6700\u5927\u6279\u91cf\u5c3a\u5bf8\u76f8\u7b49\u7684\u6279\u91cf\u7684\u6570\u91cf\n        \"ConnectionFailedCount\":\"0\",    \/\/\u4e0b\u4e00\u9636\u6bb5\u6216\u5b58\u50a8\u7cfb\u7edf\u7531\u4e8e\u9519\u8bef\u5173\u95ed\u7684\u8fde\u63a5\u6570\u91cf\uff08\u5982HDFS\u4e0a\u4e00\u4e2a\u65b0\u521b\u5efa\u7684\u6587\u4ef6\u56e0\u4e3a\u8d85\u65f6\u800c\u5173\u95ed\uff09\n        \"EventDrainAttemptCount\":\"0\",   \/\/sink\u5c1d\u8bd5\u5199\u51fa\u5230\u5b58\u50a8\u7684\u4e8b\u4ef6\u603b\u6570\u91cf\n        \"ConnectionCreatedCount\":\"0\",   \/\/\u4e0b\u4e00\u4e2a\u9636\u6bb5\u6216\u5b58\u50a8\u7cfb\u7edf\u521b\u5efa\u7684\u8fde\u63a5\u6570\u91cf\uff08\u5982HDFS\u521b\u5efa\u4e00\u4e2a\u65b0\u6587\u4ef6\uff09\n        \"BatchEmptyCount\":\"0\",      \/\/\u7a7a\u7684\u6279\u91cf\u7684\u6570\u91cf\uff0c\u5982\u679c\u6570\u91cf\u5f88\u5927\u8868\u793asouce\u5199\u6570\u636e\u6bd4sink\u6e05\u7406\u6570\u636e\u6162\u901f\u5ea6\u6162\u5f88\u591a\n        \"StopTime\":\"0\",         \n        \"RollbackCount\":\"9\",            \/\/\n        \"StartTime\":\"1442566411897\",\n        \"BatchUnderflowCount\":\"0\"       \/\/\u6bd4sink\u914d\u7f6e\u4f7f\u7528\u7684\u6700\u5927\u6279\u91cf\u5c3a\u5bf8\u66f4\u5c0f\u7684\u6279\u91cf\u7684\u6570\u91cf\uff0c\u5982\u679c\u8be5\u503c\u5f88\u9ad8\u4e5f\u8868\u793asink\u6bd4souce\u66f4\u5feb\n    }\n    }\n<\/code><\/pre>\n<p>\u63a7\u5236\u53f0\u4e0a\u5c55\u793a\u7684<\/p>\n<pre><code class=\"language-bash line-numbers\"><\/code><\/pre>\n<ol start=\"5\">\n<li>\u901a\u8fc7\u4e0a\u8ff0\u8fc7\u7a0b\u5efa\u7acb\u4e86\u4e00\u4e2a\u57fa\u672c\u6982\u5ff5<\/li>\n<\/ol>\n<p>\u67e5\u770b\u6211\u670d\u52a1\u5668\u7684\u914d\u7f6e<\/p>\n<pre><code class=\"language-bash line-numbers\">[root@wq1 ~]# cat \/proc\/cpuinfo | grep name | cut -f2 -d: | uniq -c\n4  Intel(R) Core(TM) i5-4210H CPU @ 2.90GHz\n[root@wq1 ~]# free -h\n              total        used        free      shared  buff\/cache   available\nMem:           1.8G        1.2G        147M        6.2M        451M        433M\nSwap:          2.0G         12M        2.0G\n<\/code><\/pre>\n<p>\u6b64\u6b21\u4f20\u8f93\u6570\u636e\u4f7f\u7528\u7684\u662f\u5355\u8282\u70b9\u7684flume\u4f20\u90123.1G\u7684\u6570\u636e\uff0c\u7528\u65f61\u4e2a\u5c0f\u65f6<\/p>\n<h1><span class=\"ez-toc-section\" id=\"%E4%B8%89%E3%80%81%E9%9C%80%E8%A6%81%E9%87%8D%E7%82%B9%E5%85%B3%E6%B3%A8%E7%9A%84%E5%9C%B0%E6%96%B9\"><\/span>\u4e09\u3001\u9700\u8981\u91cd\u70b9\u5173\u6ce8\u7684\u5730\u65b9<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<ul>\n<li>\u5b98\u7f51\u7ed9\u51fa\u7684\u56fe<\/li>\n<\/ul>\n<p>\u6709\u4e09\u79cd\u5f62\u5f0f\uff1a\u4e00\u4e2aagent\u7684sink\u5230\u53e6\u4e00\u4e2aagent\u7684source\u3001\u4e00\u4e2asource\u914d\u6709\u591a\u4e2achannel\u591a\u4e2asink\u7136\u540e\u628a\u6570\u636e\u6253\u5230\u4e0d\u540c\u7684\u5730\u65b9\u3001\u8fd8\u6709\u4e00\u79cd\u5c31\u662f\u4e0b\u56fe\u8fd9\u79cd\uff1b<\/p>\n<p><img src=\"http:\/\/flume.apache.org\/_images\/UserGuide_image02.png\" alt=\"\" \/><\/p>\n<blockquote><p>\n  \u5728\u5411kafka\u4f20\u6570\u636e\u65f6\u9700\u8981\u4e86\u89e3\u5230\u7684\u60c5\u5f62\n<\/p><\/blockquote>\n<pre data-language=XML><code class=\"language-markup line-numbers\">Kafka Sink: Flume \u901a\u8fc7 Kafka Sink \u5c06 Event \u5199\u2f0a\u5230 Kafka \u4e2d\u7684\u4e3b\u9898\uff0c\u5176\u4ed6\u5e94\u2f64\u901a\u8fc7\u8ba2\u9605\u4e3b\u9898\u6d88\u8d39\u6570\u636e\u3002\nkafka.producer.acks \u53ef\u4ee5\u8bbe\u7f6e Producer \u7aef\u53d1\u9001\u6d88\u606f\u5230 Broker \u4e4b\u540e\u4e0d\u9700\u8981\u7b49\u5f85 Broker \u8fd4\u56de\u6210\u529f\u9001\u8fbe\u7684\u4fe1\u53f7\u3002\n\n0\u8868\u793aProducer \u53d1\u9001\u6d88\u606f\u5230 Broker \u4e4b\u540e\u4e0d\u9700\u8981\u7b49\u5f85 Broker \u8fd4\u56de\u6210\u529f\u9001\u8fbe\u7684\u4fe1\u53f7\uff0c\u8fd9\u79cd\u2f45\u5f0f\u541e\u5410\u91cf\u2fbc\uff0c\u4f46\u5b58\u5728\u4e22\u5931\u6570\u636e\u7684\u98ce\u9669\u3002\n\n1\u8868\u793a Broker \u63a5\u6536\u5230\u6d88\u606f\u6210\u529f\u5199\u2f0a\u672c\u5730 log \u2f42\u4ef6\u540e\u5411 Producer \u8fd4\u56de\u6210\u529f \u63a5\u6536\u7684\u4fe1\u53f7\uff0c\u4e0d\u9700\u8981\u7b49\u5f85\u6240\u6709\u7684 Follower \u5168\u90e8\u540c\u6b65\u5b8c\u6d88\u606f\u540e\u518d\u505a\u56de\u5e94\uff0c\u8fd9\u79cd\u2f45\u5f0f\u5728\u6570\u636e\u4e22\u5931\u98ce\u9669\u548c\u541e\u5410\u91cf\u4e4b\u95f4\u505a\u4e86\u5e73\u8861\n\n-1\u8868\u793a Broker \u63a5\u6536\u5230 Producer \u7684\u6d88\u606f\u6210\u529f\u5199\u2f0a\u672c\u5730log \u5e76\u4e14\u7b49\u5f85\u6240\u6709\u7684 Follower \u6210\u529f\u5199\u2f0a\u672c\u5730 log \u540e\u5411 Producer \u8fd4\u56de\u6210\u529f\u63a5\u6536\u7684\u4fe1\u53f7\uff0c\u8fd9\u79cd\u2f45\u5f0f\u80fd\u591f\u4fdd\u8bc1\u6d88\u606f\u4e0d\u4e22\u5931\uff0c\u4f46\u662f\u6027\u80fd\u6700\u5dee\uff08\u5c42\u5c42\u9012\u8fdb\uff09\n<\/code><\/pre>\n<blockquote><p>\n  Flume \u7684\u8d1f\u8f7d\u5747\u8861\u548c\u6545\u969c\u8f6c\u79fb\n<\/p><\/blockquote>\n<p>\u91c7\u96c6Avro<br \/>\n&#8211; \u591a\u8def\u590d\u7528\u9009\u62e9\u5668<\/p>\n<p>\u673a\u5668wq1\u4e0a\u7684flume\u53d1\u9001\u7ed9wq2\u548cwq3\u4e0a\u540c\u6837\u7684\u6570\u636e\uff0c\u5982\u4f55\u4f60\u60f3\u91c7\u96c6\u540c\u4e00\u6570\u636e\u6e90\u5206\u522b\u505a\u4e0d\u540c\u7684\u5904\u7406\u53ef\u4ee5\u9009\u62e9\u8be5\u9009\u62e9\u5668<br \/>\n\u673a\u5668wq1\u4e0a\u7684\u811a\u672c<code>dir_flume.conf<\/code><\/p>\n<pre><code class=\"language-properties line-numbers\">#dir_flume.conf\na1.sources = r1 \na1.sinks = k1 k2 \na1.channels = c1 c2 \na1.sources.r1.selector.type = multiplexing\na1.sources.r1.selector.header= state\n#                       mapping.\u7684\u8fd9\u4e2a\u540d\u968f\u4fbf\u8d77\na1.sources.r1.selector.mapping.CZ= c1\na1.sources.r1.selector.mapping.US= c2\na1.sources.r1.selector.default= c1 c2 \n# Describe\/configure the source \na1.sources.r1.type = exec \na1.sources.r1.command = tail -F \/root\/w1.text\n\n# Describe the sink \n# sink \u7aef\u7684 avro \u662f\u4e00\u4e2a\u6570\u636e\u53d1\u9001\u8005 \na1.sinks.k1.type = avro \na1.sinks.k1.hostname = wq3\na1.sinks.k1.port = 4141 \n\na1.sinks.k2.type = avro \na1.sinks.k2.hostname = wq2 \na1.sinks.k2.port = 4142 \n\n# Describe the channel \na1.channels.c1.type = memory \na1.channels.c1.capacity = 1000 \na1.channels.c1.transactionCapacity = 100 \n\na1.channels.c2.type = memory \na1.channels.c2.capacity = 1000 \na1.channels.c2.transactionCapacity = 100 \n\n# Bind the source and sink to the channel \na1.sources.r1.channels = c1 c2 \na1.sinks.k1.channel = c1 \na1.sinks.k2.channel = c2 \n<\/code><\/pre>\n<p>\u673a\u5668wq2\u548cwq3\u811a\u672c\u5dee\u4e0d\u591a\uff08\u5c31\u673a\u5668\u540d\u548c\u7aef\u53e3\u53f7\u4e0d\u4e00\u6837\uff09<code>flume_avro2.conf<\/code><\/p>\n<pre><code class=\"language-properties line-numbers\">[root@wq3 selfconf]# cat flume_avro2.conf \n\n######flume_avro2.conf\n# Name the components on this agent \na2.sources = r1 \na2.sinks = k1 \na2.channels = c2 \n\n# Describe\/configure the source \na2.sources.r1.type = avro \na2.sources.r1.bind = wq3\na2.sources.r1.port = 4141 \n\n# Describe the sink \n#\u672c\u5730\u6587\u4ef6\u683c\u5f0f\na2.sinks.k1.type = file_roll \n#\u5728\u8be5\u6587\u4ef6\u5939\u4e0b\u5b58\u653e\u4f20\u8f93\u8fc7\u6765\u7684\u6570\u636e\na2.sinks.k1.sink.directory = \/root\/selector_flume_avro \n#\u4e0d\u6309\u7167\u65f6\u95f4\u8fdb\u884c\uff0c\u9ed8\u8ba4\u6bcf\u4e09\u79d2\u751f\u6210\u4e00\u4e2a\u6587\u4ef6\na2.sinks.k1.sink.rollInterval = 0 \n# Describe the channel \na2.channels.c2.type = memory \na2.channels.c2.capacity = 1000 \na2.channels.c2.transactionCapacity = 100 \n\n# Bind the source and sink to the channel \na2.sources.r1.channels = c2 \na2.sinks.k1.channel = c2 \n\n<\/code><\/pre>\n<ul>\n<li>\u8f6e\u8be2\u8d1f\u8f7d\u5747\u8861<\/li>\n<\/ul>\n<p>\u9996\u5148\u770b\u5b9e\u73b0\u6548\u679c<br \/>\n\u673a\u5668wq1\u4e0a\u7684flume\u91c7\u96c6\u6570\u636e\u96c6\u5206\u522b\u4f20\u8f93\u5230wq2,wq3,\u4f20\u8f93\u8fc7\u7a0b\u662f\u8f6e\u8be2\u7684\u65b9\u5f0f<\/p>\n<pre><code class=\"language-bash line-numbers\">[root@wq1 ~]# echo \u20181\u2019 &gt;&gt; w1.text          \n[root@wq1 ~]# echo \u20182\u2019 &gt;&gt; w1.text  \n[root@wq1 ~]# echo \u20183\u2019 &gt;&gt; w1.text  \n[root@wq1 ~]# echo \u20184\u2019 &gt;&gt; w1.text  \n[root@wq1 ~]# echo \u20185\u2019 &gt;&gt; w1.text\n[root@wq3 selector_flume_avro]# tail -f 1617453112325-1\n\u20181\u2019\n\u20183\u2019 \n\u20185\u2019\n[root@wq2 selector_flume_avro]# tail -f 1617453116741-1\n\u2018specialwu\u2019\n\u20182\u2019\n\u20184\u2019\n<\/code><\/pre>\n<p>\u673a\u5668wq1\u7684\u91c7\u96c6\u6570\u636e\u811a\u672clunxun.conf<\/p>\n<pre><code class=\"language-properties line-numbers\"># Name the components on this agent\na1.sources = r1\na1.sinks = k1 k2\na1.channels = c1\n\n# Describe\/configure the source\na1.sources.r1.type = exec\na1.sources.r1.channels=c1\na1.sources.r1.command=tail -F \/root\/w1.text\n\n#define sinkgroups\na1.sinkgroups=g1\na1.sinkgroups.g1.sinks=k1 k2\na1.sinkgroups.g1.processor.type=load_balance\na1.sinkgroups.g1.processor.backoff=true\na1.sinkgroups.g1.processor.selector=round_robin\n\n#define the sink 1\na1.sinks.k1.type=avro\na1.sinks.k1.hostname=wq3\na1.sinks.k1.port=4141 \n\n#define the sink 2\na1.sinks.k2.type=avro\na1.sinks.k2.hostname=wq2\na1.sinks.k2.port=4142\n\n\n# Use a channel which buffers events in memory\na1.channels.c1.type = memory\na1.channels.c1.capacity = 1000\na1.channels.c1.transactionCapacity = 100\n\n# Bind the source and sink to the channel\na1.sources.r1.channels = c1\na1.sinks.k1.channel = c1\na1.sinks.k2.channel=c1\n<\/code><\/pre>\n<p>\u6267\u884c<\/p>\n<pre><code class=\"language-bash line-numbers\">flume-ng agent \\\n--conf \/opt\/flume-1.9.0\/conf \\\n--conf-file \/opt\/flume-1.9.0\/conf\/selfconf\/lunxun.conf \\\n--name a1 \\\n-Dflume.root.logger=INFO,console\n<\/code><\/pre>\n<p>\u2f6c\u7684\u662f\u4e3a\u4e86\u63d0\u2fbc\u6574\u4e2a\u7cfb\u7edf\u7684\u5bb9\u9519\u80fd\u2f12\u548c\u7a33\u5b9a\u6027\u3002\u7b80\u5355\u914d\u7f6e\u5c31\u53ef\u4ee5\u8f7b\u677e\u5b9e\u73b0\uff0c\u2fb8\u5148\u9700\u8981\u8bbe\u7f6e Sink \u7ec4\uff0c\u540c\u2f00\u4e2a Sink \u7ec4\u5185\u6709\u591a\u4e2a\u2f26 Sink\uff0c\u4e0d\u540c Sink \u4e4b\u95f4\u53ef\u4ee5\u914d\u7f6e\u6210\u8d1f\u8f7d\u5747\u8861\u6216\u8005\u6545\u969c\u8f6c\u79fb\u3002<\/p>\n<blockquote><p>\n  Flume\u548cKafka\u9009\u53d6\n<\/p><\/blockquote>\n<ul>\n<li>Flume\u548cKafka\u7684\u533a\u522b\u4e0e\u8054\u7cfb<\/li>\n<\/ul>\n<p>flume\u8ffd\u6c42\u7684\u662f\u6570\u636e\u548c\u6570\u636e\u6e90\u3001\u6570\u636e\u6d41\u5411\u7684\u591a\u6837\u6027\uff0c\u9002\u5408\u591a\u4e2a\u751f\u4ea7\u8005\u7684\u573a\u666f\uff1b<br \/>\nkafka\u8ffd\u6c42\u7684\u662f\u9ad8\u541e\u5410\uff0c\u9ad8\u8d1f\u8f7d\uff0c\u540c\u4e00topic\u4e0b\u53ef\u4ee5\u6709\u591a\u4e2apartition\uff0c\u7531\u4e8e\u662fpull\u6a21\u5f0f\u62c9\u53d6\u6570\u636e\uff0c\u56e0\u6b64\u9002\u5408\u591a\u4e2a\u6d88\u8d39\u8005\u7684\u573a\u666f\uff1bkafka\u6ca1\u6709\u5185\u7f6e\u7684producer\u548cconsumer\u7ec4\u4ef6\uff0c\u9700\u8981\u81ea\u5df1\u7f16\u5199\u4ee3\u7801\u3002Flume\u53ef\u4ee5\u4f7f\u7528\u62e6\u622a\u5668\u5b9e\u65f6\u5904\u7406\u6570\u636e\u3002\u8fd9\u4e9b\u5bf9\u6570\u636e\u5c4f\u853d\u6216\u8005\u8fc7\u91cf\u662f\u5f88\u6709\u7528\u7684\u3002Kafka\u9700\u8981\u5916\u90e8\u7684\u6d41\u5904\u7406\u7cfb\u7edf\u624d\u80fd\u505a\u5230\u3002<\/p>\n<ul>\n<li>\u5982\u4f55\u9009\u53d6<\/li>\n<\/ul>\n<p>Kafka \u662f\u4e00\u4e2a\u975e\u5e38\u901a\u7528\u7684\u7cfb\u7edf\u3002\u4f60\u53ef\u4ee5\u6709\u8bb8\u591a\u751f\u4ea7\u8005\u548c\u5f88\u591a\u7684\u6d88\u8d39\u8005\u5171\u4eab\u591a\u4e2a\u4e3b\u9898Topics\u3002\u76f8\u6bd4\u4e4b\u4e0b\uff0cFlume\u662f\u4e00\u4e2a\u4e13\u7528\u5de5\u5177\u88ab\u8bbe\u8ba1\u4e3a\u65e8\u5728\u5f80HDFS\uff0cHBase\u53d1\u9001\u6570\u636e\u3002\u5b83\u5bf9HDFS\u6709\u7279\u6b8a\u7684\u4f18\u5316\uff0c\u5e76\u4e14\u96c6\u6210\u4e86Hadoop\u7684\u5b89\u5168\u7279\u6027\u3002\u6240\u4ee5\uff0cCloudera \u5efa\u8bae\u5982\u679c\u6570\u636e\u88ab\u591a\u4e2a\u7cfb\u7edf\u6d88\u8d39\u7684\u8bdd\uff0c\u4f7f\u7528kafka\uff1b\u5982\u679c\u6570\u636e\u88ab\u8bbe\u8ba1\u7ed9Hadoop\u4f7f\u7528\uff0c\u4f7f\u7528Flume\u3002Kafka\u548cFlume\u90fd\u662f\u53ef\u9760\u7684\u7cfb\u7edf,\u901a\u8fc7\u9002\u5f53\u7684\u914d\u7f6e\u80fd\u4fdd\u8bc1\u96f6\u6570\u636e\u4e22\u5931\u3002\u7136\u800c\uff0cFlume\u4e0d\u652f\u6301\u526f\u672c\u4e8b\u4ef6\u3002\u4e8e\u662f\uff0c\u5982\u679cFlume\u4ee3\u7406\u7684\u4e00\u4e2a\u8282\u70b9\u5954\u6e83\u4e86\uff0c\u5373\u4f7f\u4f7f\u7528\u4e86\u53ef\u9760\u7684\u6587\u4ef6\u7ba1\u9053\u65b9\u5f0f\uff0c\u4f60\u4e5f\u5c06\u4e22\u5931\u8fd9\u4e9b\u4e8b\u4ef6\u76f4\u5230\u4f60\u6062\u590d\u8fd9\u4e9b\u78c1\u76d8\u3002\u5982\u679c\u4f60\u9700\u8981\u4e00\u4e2a\u9ad8\u53ef\u9760\u884c\u7684\u7ba1\u9053\uff0c\u90a3\u4e48\u4f7f\u7528Kafka\u662f\u4e2a\u66f4\u597d\u7684\u9009\u62e9\u3002\u5728\u5b9e\u9645\u751f\u4ea7\u4e2d\u901a\u5e38\u662fflume+kafka\u642d\u914d\u4f7f\u7528\uff0c\u65e2\u80fd\u6ee1\u8db3\u53ef\u4ee5\u65b9\u4fbf\u7684\u5bfc\u6570\u636e\u5230hdfs\u4e0a\uff0c\u4e5f\u53ef\u505a\u6d41\u5f0f\u8ba1\u7b97\u65f6\u76f4\u63a5\u4ecekafka\u4e0a\u62ff\u6570\u636e\u3002<\/p>\n<p>flume\u548ckafka\u91c7\u96c6\u65e5\u5fd7\u533a\u522b\uff0c\u91c7\u96c6\u65e5\u5fd7\u65f6\u4e2d\u95f4\u505c\u4e86\uff0c\u600e\u4e48\u8bb0\u5f55\u4e4b\u524d\u7684\u65e5\u5fd7\u3002<\/p>\n<p>Flume\u91c7\u96c6\u65e5\u5fd7\u662f\u901a\u8fc7\u6d41\u7684\u65b9\u5f0f\u76f4\u63a5\u5c06\u65e5\u5fd7\u6536\u96c6\u5230\u5b58\u50a8\u5c42\uff0c\u800ckafka\u662f\u5c06\u7f13\u5b58\u5728kafka\u96c6\u7fa4\uff0c\u5f85\u540e\u671f\u53ef\u4ee5\u91c7\u96c6\u5230\u5b58\u50a8\u5c42\u3002<br \/>\nFlume\u91c7\u96c6\u4e2d\u95f4\u505c\u4e86\uff0c\u53ef\u4ee5\u91c7\u7528\u6587\u4ef6\u7684\u65b9\u5f0f\u8bb0\u5f55\u4e4b\u524d\u7684\u65e5\u5fd7\uff0c\u800ckafka\u662f\u91c7\u7528offset\u7684\u65b9\u5f0f\u8bb0\u5f55\u4e4b\u524d\u7684\u65e5\u5fd7\u3002<\/p>\n<h2><span class=\"ez-toc-section\" id=\"Flume%E7%9A%84%E9%85%8D%E7%BD%AE%E5%AE%9E%E4%BE%8B\"><\/span>Flume\u7684\u914d\u7f6e\u5b9e\u4f8b<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u6e05\u695a\u7684\u4e86\u89e3\u5404\u4e2a\u5b9e\u4f8b\u6709\u52a9\u4e8e\u5728\u4f7f\u7528\u8fc7\u7a0b\u4e2d\u5bf9\u53c2\u6570\u505a\u4e00\u4e2a\u5f88\u597d\u7684\u9009\u62e9\u3002<\/p>\n<p><code>\u5982\u679cHadoop\u548cflume\u4e0d\u5728\u540c\u4e00\u8282\u70b9\u9700\u8981\u590d\u5236Hadoop\u7684jar\u5305\u5230flume<\/code><\/p>\n<pre><code class=\"language-bash line-numbers\">docker cp \/opt\/hadoop-2.7.7\/share\/hadoop\/common\/hadoop-common-2.7.7.jar flume01:\/opt\/flume-1.9.0\/lib\ndocker cp \/opt\/hadoop-2.7.7\/share\/hadoop\/common\/lib\/commons-configuration-1.6.jar flume01:\/opt\/flume-1.9.0\/lib\ndocker cp \/opt\/hadoop-2.7.7\/share\/hadoop\/common\/lib\/hadoop-auth-2.7.7.jar flume01:\/opt\/flume-1.9.0\/lib\ndocker cp \/opt\/hadoop-2.7.7\/share\/hadoop\/common\/lib\/htrace-core-3.1.0-incubating.jar flume01:\/opt\/flume-1.9.0\/lib\ndocker cp \/opt\/hadoop-2.7.7\/share\/hadoop\/hdfs\/hadoop-hdfs-2.7.7.jar flume01:\/opt\/flume-1.9.0\/lib\ndocker cp \/opt\/hadoop-2.7.7\/share\/hadoop\/common\/lib\/commons-io-2.4.jar flume01:\/opt\/flume-1.9.0\/lib\n<\/code><\/pre>\n<h3><span class=\"ez-toc-section\" id=\"source\"><\/span>source<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>batchSize\u662f\u9488\u5bf9Source\u548cSink\u63d0\u51fa\u7684\u4e00\u4e2a\u6982\u5ff5\uff0c\u5b83\u7528\u6765\u9650\u5236source\u548csink\u5bf9event\u6279\u91cf\u5904\u7406\u7684\u3002<\/p>\n<p>\u5373\u4e00\u6b21\u6027\u4f60\u53ef\u4ee5\u5904\u7406batchSize\u4e2aevent\uff0c\u8fd9\u4e2a\u4e00\u6b21\u6027\u5c31\u662f\u6307\u5728\u4e00\u4e2a\u4e8b\u52a1\u4e2d\u3002<\/p>\n<p>\u5f53\u4f60\u5904\u7406\u7684event\u6570\u91cf\u8d85\u51fa\u4e86batchSize\uff0c\u90a3\u4e48\u4e8b\u52a1\u5c31\u4f1a\u63d0\u4ea4\u4e86\u3002<\/p>\n<p>\u6ce8\u610f\uff0c\u8fd9\u91cc\u6709\u4e00\u4e2a\u9690\u6666\u7684\u5730\u65b9\uff0c\u5c31\u662fbatchSize\u4e00\u5b9a\u4e0d\u80fd\u5927\u4e8etransactionCapacity<br \/>\n\u67e5\u770b\u4e0b\u9762\u7684\u914d\u7f6e\u524d\u660e\u786e\u51e0\u4e2a\u53c2\u6570<\/p>\n<pre><code class=\"language-bash line-numbers\">#source\u76f8\u5173\nbatch size:\u5411channel\u53d1\u9001\u6570\u636e\u7684\u6279\u6b21\u5927\u5c0f\nmaxBatchCount:\u4ece\u540c\u4e00\u4e2a\u6587\u4ef6\u4e2d\u8fde\u7eed\u8bfb\u53d6\u7684\u6700\u5927\u6279\u6b21\u6570\u91cf\nwritepostInterval:\u5411json\u6587\u4ef6\u8ffd\u51fb\u5185\u5bb9\u7684\u9891\u7387\uff08taidir source\u4e0b\u6709\uff0c\u9ed8\u8ba4\u4e09\u79d2\uff09\n#channel\u76f8\u5173\ncapacity\uff1a\u65f6\u95f4\u5bb9\u7eb3\u7684\u6570\u76ee\ntranscapacity:\u6bcf\u4e2a\u4e8b\u7269\u80fd\u591f\u5305\u542b\u7684\u4e8b\u4ef6\u6570\u76ee\uff08source\u4e00\u6b21\u7684put\u6216sink\u4e00\u6b21\u7684take\uff09\n#\u5728\u7ed9\u53c2\u6570\u5927\u5c0f\u65f6\u7b26\u5408\u4e0b\u9762\u8fd9\u8981\u6c42\nbatch size&lt;=transcapacity&lt;=capacity\n<\/code><\/pre>\n<ul>\n<li><strong>Taildir Source<\/strong>\uff08\u53ef\u7528\u6765\u8bfb\u53d6\u6587\u4ef6\u5939\u4e0b\u591a\u4e2a\u6587\u4ef6,\u53ef\u5b9e\u73b0\u65ad\u70b9\u7eed\u4f20\uff09<\/li>\n<\/ul>\n<p>Taildir Source \u53ef\u4ee5\u8bfb\u53d6\u591a\u4e2a\u6587\u4ef6\u6700\u65b0\u8ffd\u52a0\u5199\u5165\u7684\u5185\u5bb9\uff01<br \/>\nTaildir Source\u662f\u53ef\u9760\u7684\uff0c\u5373\u4f7fflume\u51fa\u73b0\u4e86\u6545\u969c\u6216\u6302\u6389\u3002Taildir Source\u5728\u5de5\u4f5c\u65f6\uff0c\u4f1a\u5c06\u8bfb\u53d6\u6587\u4ef6\u7684\u6700\u540e\u7684\u4f4d\u7f6e\u8bb0\u5f55\u5728\u4e00\u4e2ajson\u6587\u4ef6\u4e2d\uff0c\u4e00\u65e6agent\u91cd\u542f\uff0c\u4f1a\u4ece\u4e4b\u524d\u5df2\u7ecf\u8bb0\u5f55\u7684\u4f4d\u7f6e\uff0c\u7ee7\u7eed\u6267\u884ctail\u64cd\u4f5c\uff01<br \/>\nJson\u6587\u4ef6\u4e2d\uff0c\u4f4d\u7f6e\u662f\u53ef\u4ee5\u4fee\u6539\uff0c\u4fee\u6539\u540e\uff0cTaildir Source\u4f1a\u4ece\u4fee\u6539\u7684\u4f4d\u7f6e\u8fdb\u884ctail\u64cd\u4f5c\uff01\u5982\u679cJSON\u6587\u4ef6\u4e22\u5931\u4e86\uff0c\u6b64\u65f6\u4f1a\u91cd\u65b0\u4ece\u6bcf\u4e2a\u6587\u4ef6\u7684\u7b2c\u4e00\u884c\uff0c\u91cd\u65b0\u8bfb\u53d6\uff0c\u8fd9\u4f1a\u9020\u6210\u6570\u636e\u7684\u91cd\u590d\uff01<br \/>\nTaildir Source\u76ee\u524d\u53ea\u80fd\u8bfb\u6587\u672c\u6587\u4ef6<br \/>\n\u76d1\u89c6\u6307\u5b9a\u7684\u6587\u4ef6\uff0c\u4e00\u65e6\u68c0\u6d4b\u5230\u9644\u52a0\u5230\u6bcf\u4e2a\u6587\u4ef6\u7684\u65b0\u884c\uff0c\u5c31\u51e0\u4e4e\u5b9e\u65f6\u8ddf\u8e2a\u5b83\u4eec\u3002\u5982\u679c\u6b63\u5728\u5199\u5165\u65b0\u884c\uff0c\u5219\u6b64\u6e90\u5c06\u91cd\u65b0\u5c1d\u8bd5\u8bfb\u53d6\u5b83\u4eec\uff0c\u7b49\u5f85\u5199\u5165\u5b8c\u6210\u3002<br \/>\n\u8fd9\u4e2a\u6e90\u662f\u53ef\u9760\u7684\uff0c\u5373\u4f7f\u5728\u62d6\u5c3e\u6587\u4ef6\u65cb\u8f6c\u65f6\u4e5f\u4e0d\u4f1a\u4e22\u5931\u6570\u636e\u3002\u5b83\u5b9a\u671f\u5730\u4ee5JSON\u683c\u5f0f\u5728\u7ed9\u5b9a\u7684\u4f4d\u7f6e\u6587\u4ef6\u4e0a\u5199\u5165\u6bcf\u4e2a\u6587\u4ef6\u7684\u6700\u540e\u8bfb\u53d6\u4f4d\u7f6e\u3002\u5982\u679cFlume\u7531\u4e8e\u67d0\u79cd\u539f\u56e0\u88ab\u505c\u6b62\u6216\u5173\u95ed\uff0c\u5b83\u53ef\u4ee5\u4ece\u5199\u5165\u73b0\u6709\u4f4d\u7f6e\u6587\u4ef6\u7684\u4f4d\u7f6e\u91cd\u65b0\u5f00\u59cb\u5c3e\u968f\u3002\u76ee\u524d\u8fd9\u4e2a\u6e90\u4e0d\u652f\u6301\u8ddf\u8e2a\u4e8c\u8fdb\u5236\u6587\u4ef6\u3002\u5b83\u9010\u884c\u8bfb\u53d6\u6587\u672c\u6587\u4ef6\u3002<\/p>\n<pre><code class=\"language-bash line-numbers\">a1.sources.r1.type = TAILDIR\n#\u8bb0\u5f55\u4e86\u6bcf\u6761\u6587\u4ef6\u7684inode\uff0c\u4f5c\u7528\u5c31\u662fflume\u65ad\u5f00\u540e\uff0c\u4e0b\u6b21\u91c7\u96c6\u6570\u636e\u5c31\u4f1a\u4ece\u65b0\u7684\u4f4d\u7f6e\u5f00\u59cb\na1.sources.r1.positionFile = \/var\/log\/flume\/taildir_position.json\na1.sources.r1.filegroups = f1\n#\u5411\u4e0a\u9762\u7684json\u6587\u4ef6\u5199inode\u7684\u9891\u7387\na1.sources.r1.writePosInterval = 1000\n#\u8868\u793a\u53ea\u91c7\u96c6.txt\u7684\u6587\u4ef6\na1.sources.r1.filegroups.f1 = \/root\/cdhdata\/custom\/.*txt\n#Header value which is the set with header key. Multiple headers can be specified for one file group.\na1.sources.r1.headers.f1.headerKey1 = value1\n#Whether to add a header storing the absolute path filename.\na1.sources.r1.fileHeader = true\n#Header key to use when appending absolute path filename to event header.\na1.sources.r1.headers.f1.headerKey1 = value1\n#\u4ece\u540c\u4e00\u6587\u4ef6\u8fde\u7eed\u8bfb\u53d6\u7684\u6279\u6b21\u6570\uff0c\u5982\u679c\u6709\u591a\u4e2a\u6587\u4ef6\uff0c\u4e14\u5176\u4e2d\u4e00\u4e2a\u6587\u4ef6\u8bfb\u53d6\u901f\u5ea6\u5feb\uff0c\u90a3\u4e48\u8fd9\u4e2a\u6587\u4ef6\u5c31\u4f1a\u4e00\u76f4\u88ab\u8bfb\u53d6\uff0c\u5f71\u54cd\u5176\u4ed6\u6587\u4ef6\u7684\u4e13\u5c5e\na1.sources.r1.maxBatchCount = 1000\n<\/code><\/pre>\n<p><code>cat \/var\/log\/flume\/taildir_position.json<\/code><\/p>\n<pre><code class=\"language-json line-numbers\">[\n{\"inode\":69131227,\"pos\":14402,\"file\":\"\/root\/test\/w.text\"},\n{\"inode\":69131221,\"pos\":14450,\"file\":\"\/root\/test\/w1.text\"},\n{\"inode\":69131185,\"pos\":14306,\"file\":\"\/root\/test\/w.log\"}\n]\ninode\uff08\u6587\u4ef6\u552f\u4e00\u6807\u8bc6\u53f7\u7801\uff09\u3001pos\uff08\u88ab\u91c7\u96c6\u6587\u4ef6\u7684\u6700\u540e\u91c7\u96c6\u4f4d\u7f6e\uff0c\u4e5f\u5c31\u662f\u6587\u4ef6\u7684byte\u5b57\u8282\u6570\uff09\n<\/code><\/pre>\n<ul>\n<li><strong>kafka source<\/strong><\/li>\n<\/ul>\n<p>Kafka Source\u662f\u4e00\u4e2aApache Kafka\u6d88\u8d39\u8005\uff0c\u5b83\u8bfb\u53d6\u6765\u81eaKafka\u4e3b\u9898\u7684\u6d88\u606f\u3002\u5982\u679c\u6709\u591a\u4e2aKafka\u6e90\u5728\u8fd0\u884c\uff0c\u53ef\u4ee5\u5c06\u5b83\u4eec\u914d\u7f6e\u4e3a\u76f8\u540c\u7684\u4f7f\u7528\u8005\u7ec4\uff0c\u8fd9\u6837\u6bcf\u4e2a\u4f7f\u7528\u8005\u90fd\u5c06\u4e3a\u4e3b\u9898\u8bfb\u53d6\u4e00\u7ec4\u60df\u4e00\u7684\u5206\u533a\u3002<\/p>\n<pre><code class=\"language-bash line-numbers\">a3.sources.r1.type = org.apache.flume.source.kafka.KafkaSource\n#\u79ef\u65125000\u4e2aevent\u5230\u4e00\u4e2a\u5730\u65b9\na3.sources.r1.batchSize = 5000 \na3.sources.r1.batchDurationMillis = 2000\n#kafka\u96c6\u7fa4\na3.sources.r1.kafka.bootstrap.servers = wq1:9092,wq2:9092,wq3:9092\na3.sources.r1.kafka.consumer.auto.offset.reset = earliest\n#\u591a\u4e2atopic\u7528\uff0c\u9694\u5f00\na3.sources.r1.kafka.topics = hdfs-hive\n<\/code><\/pre>\n<ul>\n<li>Avro source<\/li>\n<\/ul>\n<p>\u901a\u5e38\u8de8\u673a\u5668\u91c7\u96c6\u6570\u636e\u65f6\u4f7f\u7528<\/p>\n<pre><code class=\"language-bash line-numbers\">a1.sources = r1\na1.channels = c1\na1.sources.r1.type = avro\na1.sources.r1.channels = c1\na1.sources.r1.bind = 0.0.0.0\na1.sources.r1.port = 4141\nipFilter=true\nipFilterRules=allow:ip:127.*,allow:name:localhost,deny:ip:*\n<\/code><\/pre>\n<hr \/>\n<h3><span class=\"ez-toc-section\" id=\"sink\"><\/span>sink<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<ul>\n<li><strong>HDFS Sink<\/strong><\/li>\n<\/ul>\n<p>This sink writes events into the Hadoop Distributed File System (HDFS). It currently supports creating text and sequence files. It supports compression in both file types. The files can be rolled (close current file and create a new one) periodically based on the elapsed time or size of data or number of events.<\/p>\n<pre><code class=\"language-bash line-numbers\">a1.channels = c1\na1.sinks = k1\na1.sinks.k1.type = hdfs\na1.sinks.k1.channel = c1\n#\u751f\u6210\u7684\u6587\u4ef6\u5939\u540d\u4e3a\u5e74\u6708\u65e5\u65f6\u5206\u79d2\na1.sinks.k1.hdfs.path = \/flume\/events\/%y-%m-%d\/%H%M\/%S\na1.sinks.k1.hdfs.filePrefix = events- #\u5bfc\u5165\u6587\u4ef6\u7684\u524d\u7f00\na1.sinks.k1.hdfs.round = true #\u662f\u5426\u6309\u7167\u65f6\u95f4\u751f\u6210\u4e00\u4e2a\u65b0\u7684\u6587\u4ef6\u5939\uff0c\u57fa\u4e8e\u8def\u5f84path\u53bb\u6eda\u52a8\u751f\u6210\u6587\u4ef6\u5939\u7684\na1.sinks.k1.hdfs.roundValue = 10 #\u8fd9\u4e2a\u65f6\u95f4\u7684\u503c\u662f\u591a\u5c11\uff0c\u6b64\u5904\u662f10\u4e5f\u5c31\u6ca1\u5206\u949f\u751f\u6210\u4e00\u4e2a\u6587\u4ef6\u5939\na1.sinks.k1.hdfs.roundUnit = minute #second,minute,hour\u3002\n##\u8bbe\u7f6e\u7684\u6587\u4ef6\u7c7b\u578b\uff0c\u8fd9\u79cd\u7c7b\u578b\u652f\u6301\u538b\u7f29\uff0c\u5305\u62ec\uff1aSequenceFile, DataStream,CompressedStream\n#\u4f7f\u7528DataStream\u65f6\u5019\uff0c\u6587\u4ef6\u4e0d\u4f1a\u88ab\u538b\u7f29\uff0c\u4e0d\u9700\u8981\u8bbe\u7f6ehdfs.codeC;\n#\u5f53\u4f7f\u7528CompressedStream\u65f6\u5019\uff0c\u5fc5\u987b\u8bbe\u7f6e\u4e00\u4e2a\u6b63\u786e\u7684hdfs.codeC\u503c\uff1b\na3.sinks.k1.hdfs.fileType = DataStream \n#\u6587\u4ef6\u7684\u683c\u5f0f\na3.sinks.k1.hdfs.writeFormat = Text\n<\/code><\/pre>\n<ul>\n<li><strong>Hive sink<\/strong><\/li>\n<\/ul>\n<p>\u6b64\u63a5\u6536\u5668\u5c06\u5305\u542b\u5206\u9694\u6587\u672c\u6216JSON\u6570\u636e\u7684\u4e8b\u4ef6\u76f4\u63a5\u6d41\u5230Hive\u8868\u6216\u5206\u533a\u4e2d\u3002\u4e8b\u4ef6\u662f\u4f7f\u7528Hive\u4e8b\u52a1\u7f16\u5199\u7684\u3002\u53ea\u8981\u5c06\u4e00\u7ec4\u4e8b\u4ef6\u63d0\u4ea4\u5230Hive\uff0c\u5b83\u4eec\u5c31\u4f1a\u7acb\u5373\u5bf9Hive\u67e5\u8be2\u53ef\u89c1\u3002flume\u5c06\u6d41\u5230\u7684\u5206\u533a\u65e2\u53ef\u4ee5\u9884\u5148\u521b\u5efa\uff0c\u4e5f\u53ef\u4ee5\u9009\u62e9\u5728\u7f3a\u5c11\u5206\u533a\u65f6\u521b\u5efa\u5b83\u4eec\u3002\u6765\u81ea\u4f20\u5165\u4e8b\u4ef6\u6570\u636e\u7684\u5b57\u6bb5\u88ab\u6620\u5c04\u5230Hive\u8868\u4e2d\u7684\u76f8\u5e94\u5217\u3002<\/p>\n<pre><code class=\"language-bash line-numbers\">a1.channels = c1\na1.channels.c1.type = memory\na1.sinks = k1\na1.sinks.k1.type = hive\na1.sinks.k1.channel = c1\na1.sinks.k1.hive.metastore = thrift:\/\/127.0.0.1:9083\na1.sinks.k1.hive.database = logsdb\na1.sinks.k1.hive.table = weblogs\na1.sinks.k1.hive.partition = asia,%{country},%y-%m-%d-%H-%M\na1.sinks.k1.useLocalTimeStamp = false\na1.sinks.k1.round = true\na1.sinks.k1.roundValue = 10\na1.sinks.k1.roundUnit = minute\na1.sinks.k1.serializer = DELIMITED\na1.sinks.k1.serializer.delimiter = \"\\t\"\na1.sinks.k1.serializer.serdeSeparator = '\\t'\na1.sinks.k1.serializer.fieldnames =id,,msg\n<\/code><\/pre>\n<ul>\n<li><strong>Avro sink<\/strong><\/li>\n<\/ul>\n<p>This sink forms one half of Flume\u2019s tiered collection support. Flume events sent to this sink are turned into Avro events and sent to the configured hostname \/ port pair. The events are taken from the configured Channel in batches of the configured batch size.<\/p>\n<pre><code class=\"language-bash line-numbers\">a1.channels = c1\na1.sinks = k1\na1.sinks.k1.type = avro\na1.sinks.k1.channel = c1\na1.sinks.k1.hostname = 10.10.10.10\na1.sinks.k1.port = 4545\n<\/code><\/pre>\n<ul>\n<li>kafaka sink<\/li>\n<\/ul>\n<p>This is a Flume Sink implementation that can publish data to a Kafka topic. One of the objective is to integrate Flume with Kafka so that pull based processing systems can process the data coming through various Flume sources.<\/p>\n<p>kafka.producer.acks 1   How many replicas must acknowledge a message before its considered successfully written. Accepted values are 0 (Never wait for acknowledgement), 1 (wait for leader only), -1 (wait for all replicas) Set this to -1 to avoid data loss in some cases of leader failure.<\/p>\n<pre><code class=\"language-bash line-numbers\">a1.sinks.k1.channel = c1\na1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink\na1.sinks.k1.kafka.topic = mytopic\na1.sinks.k1.kafka.bootstrap.servers = localhost:9092\na1.sinks.k1.kafka.flumeBatchSize = 20\na1.sinks.k1.kafka.producer.acks = 1\na1.sinks.k1.kafka.producer.linger.ms = 1\na1.sinks.k1.kafka.producer.compression.type = snappy\n<\/code><\/pre>\n<hr \/>\n<h3><span class=\"ez-toc-section\" id=\"channel\"><\/span>channel<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<ul>\n<li><strong>Memory Channel<\/strong><\/li>\n<\/ul>\n<p>\u4e8b\u4ef6\u5b58\u50a8\u5728\u5185\u5b58\u4e2d\u7684\u961f\u5217\u4e2d\uff0c\u5176\u6700\u5927\u5927\u5c0f\u53ef\u914d\u7f6e\u3002\u5bf9\u4e8e\u9700\u8981\u66f4\u9ad8\u541e\u5410\u91cf\u5e76\u51c6\u5907\u5728\u4ee3\u7406\u5931\u8d25\u65f6\u4e22\u5931\u5206\u6bb5\u6570\u636e\u7684\u6d41\u6765\u8bf4\uff0c\u5b83\u975e\u5e38\u7406\u60f3<\/p>\n<pre><code class=\"language-bash line-numbers\">a1.channels = c1\na1.channels.c1.type = memory\na1.channels.c1.capacity = 10000\na1.channels.c1.transactionCapacity = 10000\n#  Defines the percent of buffer between byteCapacity and the estimated total size of all events in the channel, to account for data in headers. \na1.channels.c1.byteCapacityBufferPercentage = 20\na1.channels.c1.byteCapacity = 800000\n<\/code><\/pre>\n<ul>\n<li><strong>File Channel<\/strong><\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">a3.channels.c1.type = file\na3.channels.c1.capacity = 10000\na3.channels.c1.byteCapacityBufferPercentage = 20\na3.channels.c1.transactionCapacity = 10000\na3.channels.c1.byteCapacity = 20000000\n<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"<p>\u5b98\u7f51\u7ed9\u51fa\u7684\u4ecb\u7ecd Flume is a distributed, reliable, and available [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":860,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[7],"tags":[],"_links":{"self":[{"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/posts\/854"}],"collection":[{"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=854"}],"version-history":[{"count":27,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/posts\/854\/revisions"}],"predecessor-version":[{"id":2240,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/posts\/854\/revisions\/2240"}],"wp:featuredmedia":[{"embeddable":true,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/media\/860"}],"wp:attachment":[{"href":"http:\/\/www.specialwu.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=854"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=854"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=854"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}