{"id":1786,"date":"2021-02-20T16:27:46","date_gmt":"2021-02-20T08:27:46","guid":{"rendered":"https:\/\/www.specialwu.com\/?p=1786"},"modified":"2024-03-25T23:29:58","modified_gmt":"2024-03-25T15:29:58","slug":"%e5%a4%a7%e6%95%b0%e6%8d%ae%e9%a1%b9%e7%9b%ae%e6%b5%81%e7%a8%8b","status":"publish","type":"post","link":"http:\/\/www.specialwu.com\/?p=1786","title":{"rendered":"\u8bb0\u4e00\u6b21\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u6d41\u7a0b"},"content":{"rendered":"<div id=\"ez-toc-container\" class=\"ez-toc-v2_0_61 ez-toc-wrap-center counter-hierarchy ez-toc-counter ez-toc-custom ez-toc-container-direction\">\n<div class=\"ez-toc-title-container\">\n<p class=\"ez-toc-title \" >\u76ee\u5f55<\/p>\n<span class=\"ez-toc-title-toggle\"><a href=\"#\" class=\"ez-toc-pull-right ez-toc-btn ez-toc-btn-xs ez-toc-btn-default ez-toc-toggle\" aria-label=\"Toggle Table of Content\"><span class=\"ez-toc-js-icon-con\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #000000;color:#000000\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #000000;color:#000000\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/span><\/a><\/span><\/div>\n<nav><ul class='ez-toc-list ez-toc-list-level-1 ' ><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"http:\/\/www.specialwu.com\/?p=1786\/#%E5%89%8D%E8%A8%80\" title=\"\u524d\u8a00\">\u524d\u8a00<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"http:\/\/www.specialwu.com\/?p=1786\/#%E5%AE%9E%E6%96%BD%E8%BF%87%E7%A8%8B\" title=\"\u5b9e\u65bd\u8fc7\u7a0b\">\u5b9e\u65bd\u8fc7\u7a0b<\/a><ul class='ez-toc-list-level-2' ><li class='ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-3\" href=\"http:\/\/www.specialwu.com\/?p=1786\/#%E5%90%84%E4%B8%AA%E9%9B%86%E7%BE%A4%E5%90%AF%E5%8A%A8\" title=\"\u5404\u4e2a\u96c6\u7fa4\u542f\u52a8\">\u5404\u4e2a\u96c6\u7fa4\u542f\u52a8<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-4\" href=\"http:\/\/www.specialwu.com\/?p=1786\/#flume%E5%BC%80%E5%A7%8B%E6%8B%BF%E6%95%B0%E6%8D%AE\" title=\"flume\u5f00\u59cb\u62ff\u6570\u636e\">flume\u5f00\u59cb\u62ff\u6570\u636e<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-5\" href=\"http:\/\/www.specialwu.com\/?p=1786\/#Connection_to_node_-1_could_not_be_established_Broker_may_not_be_available\" title=\"Connection to node -1 could not be established. Broker may not be available.\">Connection to node -1 could not be established. Broker may not be available.<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-6\" href=\"http:\/\/www.specialwu.com\/?p=1786\/#Hive%E6%95%B0%E4%BB%93\" title=\"Hive\u6570\u4ed3\">Hive\u6570\u4ed3<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-2'><a class=\"ez-toc-link ez-toc-heading-7\" href=\"http:\/\/www.specialwu.com\/?p=1786\/#%E5%B0%86%E6%95%B0%E6%8D%AE%E5%AF%BC%E5%85%A5MySQL\" title=\"\u5c06\u6570\u636e\u5bfc\u5165MySQL\">\u5c06\u6570\u636e\u5bfc\u5165MySQL<\/a><\/li><\/ul><\/li><\/ul><\/nav><\/div>\n<h1><span class=\"ez-toc-section\" id=\"%E5%89%8D%E8%A8%80\"><\/span>\u524d\u8a00<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<p>\u672c\u6765\u524d\u8a00\u8fd9\u90e8\u5206\u51c6\u5907\u4e86\u4e00\u5927\u5806\u8bdd\u6765\u63cf\u8ff0\u4e0b\u672c\u7bc7\u8981\u4ecb\u7ecd\u7684\u5185\u5bb9\uff0c\u7b97\u4e86\uff0c\u770b\u56fe\u5427\uff0c\u672c\u7bc7\u4f1a\u7ed9\u4f60\u6253\u6765\u7684\u6700\u5927\u5e2e\u52a9\u5c31\u662f\uff0c\u5448\u5148\u5927\u6570\u636e\u4e00\u90e8\u5206\u9879\u76ee\u7684\u6837\u8c8c\uff0c\u8fd9\u7bc7\u5185\u5bb9\u4e5f\u4e0d\u662f\u4e00\u6b21\u5199\u5b8c\u4e86\uff0c\u9996\u5148\u662f\u5448\u5148\u51fa\u4e00\u4e2a\u5b8c\u6574\u8fc7\u7a0b\uff0c\u540e\u671f\u4f1a\u4e0d\u65ad\u586b\u5145\u7ec6\u8282\uff0c\u4e0d\u591a\u8bf4\u4e86\uff0c\u5168\u5728\u56fe\u91cc\u9762\uff0c\u5e72\u4e86\u3002<br \/>\n<img src=\"https:\/\/www.specialwu.com\/wp-content\/uploads\/2021\/02\/\u5927\u6570\u636e\u9879\u76ee\u6d41\u7a0b.png\" alt=\"\" \/><\/p>\n<h1><span class=\"ez-toc-section\" id=\"%E5%AE%9E%E6%96%BD%E8%BF%87%E7%A8%8B\"><\/span>\u5b9e\u65bd\u8fc7\u7a0b<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<h2><span class=\"ez-toc-section\" id=\"%E5%90%84%E4%B8%AA%E9%9B%86%E7%BE%A4%E5%90%AF%E5%8A%A8\"><\/span>\u5404\u4e2a\u96c6\u7fa4\u542f\u52a8<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<ol>\n<li>\u542f\u52a8hdfs\u96c6\u7fa4<\/li>\n<\/ol>\n<p>jps(Java Virtual Machine Process Status Tool)\u662fjava\u63d0\u4f9b\u7684\u4e00\u4e2a\u663e\u793a\u5f53\u524d\u6240\u6709java\u8fdb\u7a0bpid\u7684\u547d\u4ee4;\u5982\u679c\u7b2c\u4e00\u6b21\u542f\u52a8namenode\u6ca1\u542f\u52a8\u8d77\u6765\u53ef\u6267\u884c\u521d\u59cb\u5316\u547d\u4ee4<code>hdfs namenode -format<\/code><\/p>\n<pre><code class=\"language-bash line-numbers\">[root@wq1 ~]# start-dfs.sh\n[root@wq1 ~]# jps\n6162 DataNode\n8949 Jps\n6518 SecondaryNameNode\n6874 NameNode\n<\/code><\/pre>\n<p>\u6839\u636e\u81ea\u5df1\u7684IP\u7aef\u53e3\u8fdb\u5165namennode\u7684web\u754c\u9762http:\/\/192.168.3.233:50070\/explorer.html#\/<br \/>\n2. \u542f\u52a8zookeeper<\/p>\n<pre><code class=\"language-bash line-numbers\">[root@wq2 bin]# zkServer.sh start\nZooKeeper JMX enabled by default\nUsing config: \/opt\/zookeeper-3.4.14\/bin\/..\/conf\/zoo.cfg\nStarting zookeeper ... STARTED\n<\/code><\/pre>\n<ol start=\"3\">\n<li>\u542f\u52a8kafka<\/li>\n<\/ol>\n<pre><code class=\"language-bash line-numbers\">[root@wq1 ~]# kafka-server-start.sh -daemon \/opt\/kafka\/config\/server.properties\n<\/code><\/pre>\n<ol start=\"4\">\n<li>\u542f\u52a8hive<br \/>\n\u542f\u52a8hiveserver2\u662f\u4e3a\u4e86\u4f7f\u7528hive\u6570\u4ed3\u7684\u53ef\u89c6\u5316\u754c\u9762\u4f7f\u7528\uff08\u8981\u8fdc\u7a0b\u8fde\u63a5\uff09\uff0c\u542f\u52a8hive\u76ee\u7684\u662f\u5728\u670d\u52a1\u5668\u4e0a\u8fdb\u884c\u4e00\u4e9b\u5efa\u8868\u64cd\u4f5c<\/li>\n<\/ol>\n<pre><code class=\"language-bash line-numbers\">[root@wq1 ~]# hive\n[root@wq1 ~]# hiveserver2\n<\/code><\/pre>\n<hr \/>\n<h2><span class=\"ez-toc-section\" id=\"flume%E5%BC%80%E5%A7%8B%E6%8B%BF%E6%95%B0%E6%8D%AE\"><\/span>flume\u5f00\u59cb\u62ff\u6570\u636e<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u56e0\u4e3aflume-\u300bkafka->flume-\u300bhdfs<br \/>\n1. kafka\u63a5\u6570\u636e\u524d\u521b\u5efa\u4e00\u4e2atopic<\/p>\n<pre><code class=\"language-bash line-numbers\">[root@wq1 my_conf]# kafka-topics.sh --zookeeper wq1:2181 --create --topic mywebdata --partitions 3 --replication-factor 1       \nCreated topic \"mywebdata\".\n<\/code><\/pre>\n<h3><span class=\"ez-toc-section\" id=\"Connection_to_node_-1_could_not_be_established_Broker_may_not_be_available\"><\/span>Connection to node -1 could not be established. Broker may not be available.<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>\u5728wq2\u6267\u884c\u5982\u4e0b\u547d\u4ee4\u65f6\u51fa\u73b0\u4e86\u8be5\u62a5\u9519\uff0c\u867d\u7136\u5728\u6267\u884c\u540e\u53f0\u542f\u52a8\u547d\u4ee4\u540e\u67e5\u770bJava\u8fdb\u7a0b\u53ef\u4ee5\u770b\u5230\u8be5\u673a\u5668kafka\u5df2\u7ecf\u8d77\u4e86\u8d77\u6765\uff0c\u4f46\u662f\u4e00\u65e6\u8fdb\u884c\u64cd\u4f5c\u547d\u4ee4\uff0ckafka\u5c31\u4f1a\u6302\u6389\u3002<\/p>\n<pre><code class=\"language-bash line-numbers\">kafka-console-consumer.sh --bootstrap-server wq2:9092 --topic topic1\n<\/code><\/pre>\n<p>\u67e5\u770b\u8be5\u673a\u5668Java\u8fdb\u7a0b\u53d1\u73b0\u5c11\u673a\u5668\u5173\u4e86\uff0c\u90a3\u4e48\u5c31\u5b58\u5728\u4e00\u4e2a\u95ee\u9898\uff0c\u5982\u679ckafka\u96c6\u7fa4\u4e09\u53f0\u6302\u4e86\u4e00\u53f0\u600e\u4e48\u542f\u52a8\u53e6\u4e00\u53f0\u3002\u521d\u6b65\u5206\u6790\u95ee\u9898\u53ef\u80fd\u662f\u7f51\u7edc\u6ce2\u52a8\u5bfc\u81f4\u8fd9\u53f0\u673a\u5668\u6ca1\u6709\u88ab\u9009\u4e3e\u4e0a\uff0c\u91cd\u542fzookeeper\u8fd8\u662f\u6ca1\u6709\u89e3\u51b3\uff0c\u8fd9\u4e0b\u5c31\u5f97\u770b\u62a5\u9519\u4fe1\u606f\u4e86\u3002<\/p>\n<pre><code class=\"language-bash line-numbers\">#\u67e5\u770bkafka\u7684server.log\u5185\u5bb9\u5982\u4e0b\nkafka.common.InconsistentBrokerIdException: Configured broker.id 2 doesn't match stored broker.id 3 in meta.properties. \nIf you moved your data, make sure your configured broker.id matches. If you intend to create a new broker, you should remove all data in your data directories (log.dirs).\n<\/code><\/pre>\n<p>\u6700\u7ec8\u89e3\u51b3\u529e\u6cd5\uff0c\u9996\u5148\u6211\u4eec\u67e5\u770bkafka\u96c6\u7fa4\u7684\u914d\u7f6e\u8fc7\u7a0b\uff0c\u77e5\u9053zookeeper\u7684zoo.cfg\u4e2d\u6709\u4e00\u9879\u914d\u7f6e\u662f\u4e0b\u9762\u8fd9\uff0c\u800c\u5728kafka\u7684sever.properties\u4e2d\u6709\u4e00\u9879\u662f\u8bbe\u7f6ebroker.id\uff0c\u8fd9\u4e2aid\u6211\u5c31\u8bbe\u7f6e\u9519\u4e86\uff0c\u5728zoo.cfg\u4e2d\u8bbe\u7f6e\u7684dataDir\u4e0b\uff0c\u521b\u5efa\u4e00\u4e2a\u540d\u4e3amyid\u7684\u6587\u4ef6\uff0cmyid\u6587\u4ef6\u4e2d\u5199\u4e0a\u5f53\u524d\u673a\u5668\u5bf9\u5e94\u7684id\u53f7\uff0c\u90a3\u4e48\u5c31\u9700\u8981\u8fd9\u4e09\u4e2aid\u90fd\u5bf9\u5f97\u4e0a\u3002id\u8fd9\u5757\u8bbe\u7f6e\u5b8c\u4e86\uff0c\u4f46\u8fd8\u662f\u6709\u95ee\u9898\uff0c\u67e5\u770b\u5982\u4e0a\u82f1\u6587\u62a5\u9519\u4fe1\u606f\uff0c\u6539\u4e86id\u8fd8\u4e0d\u884c\u5fc5\u987b\u628a\u76f8\u5173\u7684\u65e5\u5fd7\u4fe1\u606f\u5220\u4e86\u624d\u884c\uff0c\u5220\u9664\u540e\u81f3\u6b64\u95ee\u9898\u89e3\u51b3<\/p>\n<pre><code class=\"language-bash line-numbers\"># server.id=\u4e3b\u673a\u540d:\u901a\u4fe1\u7aef\u53e3:\u9009\u4e3e\u7aef\u53e3\nserver.1=wq1:2888:3888\nserver.2=wq2:2888:3888\nserver.3=wq3:2888:3888\n<\/code><\/pre>\n<p>\u67e5\u770b\u8fd9\u4e2atopic\u60c5\u51b5(\u901a\u8fc7\u4e0b\u65b9\u60c5\u51b5\u8fd9\u4e2atopic\u6709\u4e09\u4e2a\u5206\u533a\u4e00\u4e2a\u526f\u672c\uff0c\u5206\u533a0\u548c2\u7684\u6570\u636e\u5b58\u5728\u7b2c\u4e8c\u53f0\u673a\u5668\u4e0a\uff0c\u5206\u533a1\u7684\u6570\u636e\u5728\u7b2c\u4e00\u53f0\u673a\u5668\u4e0a)<\/p>\n<pre><code class=\"language-bash line-numbers\">[root@wq2 bin]# kafka-topics.sh --zookeeper wq1:2181 --describe --topic mywebdata\nTopic:mywebdata PartitionCount:3        ReplicationFactor:1     Configs:\n        Topic: mywebdata        Partition: 0    Leader: 2       Replicas: 2     Isr: 2\n        Topic: mywebdata        Partition: 1    Leader: 1       Replicas: 1     Isr: 1\n        Topic: mywebdata        Partition: 2    Leader: 2       Replicas: 2     Isr: 2\n<\/code><\/pre>\n<ol start=\"2\">\n<li>flume\u7684\u6267\u884c\u811a\u672c<br \/>\n\u6267\u884c\u524d\u9996\u5148\u5bf9flume\u4f20\u6570\u636e\u60c5\u51b5\u8fdb\u884c\u4e86\u89e3<\/li>\n<\/ol>\n<pre><code class=\"language-bash line-numbers\">flume-ng agent \\\n-c \/opt\/flume-1.9.0\/conf \\\n-f \/opt\/flume-1.9.0\/conf\/my_conf\/flume-kafka.conf \\\n-n a1 \\\n-Dflume.monitoring.type=http \\\n-Dflume.monitoring.port=5653 \\\n-Dflume.root.logger=INFO,console\n<\/code><\/pre>\n<p>\u8bbf\u95ee\u5730\u5740<\/p>\n<pre><code class=\"language-java line-numbers\">http:\/\/\u4f60\u7684\u4e3b\u673a\u540d:5653\/metrics\n<\/code><\/pre>\n<p>http\u4e0a\u5c55\u793a\u7684\u5185\u5bb9<\/p>\n<pre><code class=\"language-json line-numbers\">{\"SINK.k1\":{\"ConnectionCreatedCount\":\"0\",\"BatchCompleteCount\":\"0\",\"BatchEmptyCount\":\"147\",\"EventDrainAttemptCount\":\"10\",\"StartTime\":\"1613963308675\",\"BatchUnderflowCount\":\"1\",\"ConnectionFailedCount\":\"0\",\"ConnectionClosedCount\":\"0\",\"Type\":\"SINK\",\"RollbackCount\":\"0\",\"EventDrainSuccessCount\":\"10\",\"KafkaEventSendTimer\":\"48502\",\"StopTime\":\"0\"},\n \"CHANNEL.c1\":{\"ChannelCapacity\":\"1000\",\"ChannelFillPercentage\":\"0.0\",\"Type\":\"CHANNEL\",\"ChannelSize\":\"0\",\"EventTakeSuccessCount\":\"10\",\"EventTakeAttemptCount\":\"158\",\"StartTime\":\"1613963279841\",\"EventPutAttemptCount\":\"10\",\"EventPutSuccessCount\":\"10\",\"StopTime\":\"0\"},\n \"SOURCE.r1\":{\"AppendBatchAcceptedCount\":\"0\",\"GenericProcessingFail\":\"0\",\"EventAcceptedCount\":\"10\",\"AppendReceivedCount\":\"0\",\"StartTime\":\"1613963279882\",\"AppendBatchReceivedCount\":\"0\",\"ChannelWriteFail\":\"0\",\"EventReceivedCount\":\"10\",\"EventReadFail\":\"0\",\"Type\":\"SOURCE\",\"AppendAcceptedCount\":\"0\",\"OpenConnectionCount\":\"0\",\"StopTime\":\"0\"}}\n<\/code><\/pre>\n<p>\u542b\u4e49\u5bf9\u7167<\/p>\n<pre><code class=\"language-json line-numbers\">{\n    \"SOURCE.src-1\":{\n        \"OpenConnectionCount\":\"0\",      \/\/\u76ee\u524d\u4e0e\u5ba2\u6237\u7aef\u6216sink\u4fdd\u6301\u8fde\u63a5\u7684\u603b\u6570\u91cf(\u76ee\u524d\u53ea\u6709avro source\u5c55\u73b0\u8be5\u5ea6\u91cf)\n        \"Type\":\"SOURCE\",                    \n        \"AppendBatchAcceptedCount\":\"1355\",  \/\/\u6210\u529f\u63d0\u4ea4\u5230channel\u7684\u6279\u6b21\u7684\u603b\u6570\u91cf\n        \"AppendBatchReceivedCount\":\"1355\",  \/\/\u63a5\u6536\u5230\u4e8b\u4ef6\u6279\u6b21\u7684\u603b\u6570\u91cf\n        \"EventAcceptedCount\":\"28286\",   \/\/\u6210\u529f\u5199\u51fa\u5230channel\u7684\u4e8b\u4ef6\u603b\u6570\u91cf\uff0c\u4e14source\u8fd4\u56desuccess\u7ed9\u521b\u5efa\u4e8b\u4ef6\u7684sink\u6216RPC\u5ba2\u6237\u7aef\u7cfb\u7edf\n        \"AppendReceivedCount\":\"0\",      \/\/\u6bcf\u6279\u53ea\u6709\u4e00\u4e2a\u4e8b\u4ef6\u7684\u4e8b\u4ef6\u603b\u6570\u91cf(\u4e0eRPC\u8c03\u7528\u4e2d\u7684\u4e00\u4e2aappend\u8c03\u7528\u76f8\u7b49)\n        \"StopTime\":\"0\",         \/\/source\u505c\u6b62\u65f6\u81eaEpoch\u4ee5\u6765\u7684\u6beb\u79d2\u503c\u65f6\u95f4\n        \"StartTime\":\"1442566410435\",    \/\/source\u542f\u52a8\u65f6\u81eaEpoch\u4ee5\u6765\u7684\u6beb\u79d2\u503c\u65f6\u95f4\n        \"EventReceivedCount\":\"28286\",   \/\/\u76ee\u524d\u4e3a\u6b62source\u5df2\u7ecf\u63a5\u6536\u5230\u7684\u4e8b\u4ef6\u603b\u6570\u91cf\n        \"AppendAcceptedCount\":\"0\"       \/\/\u5355\u72ec\u4f20\u5165\u7684\u4e8b\u4ef6\u5230Channel\u4e14\u6210\u529f\u8fd4\u56de\u7684\u4e8b\u4ef6\u603b\u6570\u91cf\n    },\n    \"CHANNEL.ch-1\":{\n        \"EventPutSuccessCount\":\"28286\", \/\/\u6210\u529f\u5199\u5165channel\u4e14\u63d0\u4ea4\u7684\u4e8b\u4ef6\u603b\u6570\u91cf\n        \"ChannelFillPercentage\":\"0.0\",  \/\/channel\u6ee1\u65f6\u7684\u767e\u5206\u6bd4\n        \"Type\":\"CHANNEL\",\n        \"StopTime\":\"0\",         \/\/channel\u505c\u6b62\u65f6\u81eaEpoch\u4ee5\u6765\u7684\u6beb\u79d2\u503c\u65f6\u95f4\n        \"EventPutAttemptCount\":\"28286\", \/\/Source\u5c1d\u8bd5\u5199\u5165Channe\u7684\u4e8b\u4ef6\u603b\u6570\u91cf\n        \"ChannelSize\":\"0\",          \/\/\u76ee\u524dchannel\u4e2d\u4e8b\u4ef6\u7684\u603b\u6570\u91cf\n        \"StartTime\":\"1442566410326\",    \/\/channel\u542f\u52a8\u65f6\u81eaEpoch\u4ee5\u6765\u7684\u6beb\u79d2\u503c\u65f6\u95f4\n        \"EventTakeSuccessCount\":\"28286\",    \/\/sink\u6210\u529f\u8bfb\u53d6\u7684\u4e8b\u4ef6\u7684\u603b\u6570\u91cf\n        \"ChannelCapacity\":\"1000000\",       \/\/channel\u7684\u5bb9\u91cf\n        \"EventTakeAttemptCount\":\"313734329512\" \/\/sink\u5c1d\u8bd5\u4ecechannel\u62c9\u53d6\u4e8b\u4ef6\u7684\u603b\u6570\u91cf\u3002\u8fd9\u4e0d\u610f\u5473\u7740\u6bcf\u6b21\u4e8b\u4ef6\u90fd\u88ab\u8fd4\u56de\uff0c\u56e0\u4e3asink\u62c9\u53d6\u7684\u65f6\u5019channel\u53ef\u80fd\u6ca1\u6709\u4efb\u4f55\u6570\u636e\n    },\n    \"SINK.sink-1\":{\n        \"Type\":\"SINK\",\n        \"ConnectionClosedCount\":\"0\",    \/\/\u4e0b\u4e00\u9636\u6bb5\u6216\u5b58\u50a8\u7cfb\u7edf\u5173\u95ed\u7684\u8fde\u63a5\u6570\u91cf(\u5982\u5728HDFS\u4e2d\u5173\u95ed\u4e00\u4e2a\u6587\u4ef6)\n        \"EventDrainSuccessCount\":\"28286\",   \/\/sink\u6210\u529f\u5199\u51fa\u5230\u5b58\u50a8\u7684\u4e8b\u4ef6\u603b\u6570\u91cf\n        \"KafkaEventSendTimer\":\"482493\",    \n        \"BatchCompleteCount\":\"0\",       \/\/\u4e0e\u6700\u5927\u6279\u91cf\u5c3a\u5bf8\u76f8\u7b49\u7684\u6279\u91cf\u7684\u6570\u91cf\n        \"ConnectionFailedCount\":\"0\",    \/\/\u4e0b\u4e00\u9636\u6bb5\u6216\u5b58\u50a8\u7cfb\u7edf\u7531\u4e8e\u9519\u8bef\u5173\u95ed\u7684\u8fde\u63a5\u6570\u91cf\uff08\u5982HDFS\u4e0a\u4e00\u4e2a\u65b0\u521b\u5efa\u7684\u6587\u4ef6\u56e0\u4e3a\u8d85\u65f6\u800c\u5173\u95ed\uff09\n        \"EventDrainAttemptCount\":\"0\",   \/\/sink\u5c1d\u8bd5\u5199\u51fa\u5230\u5b58\u50a8\u7684\u4e8b\u4ef6\u603b\u6570\u91cf\n        \"ConnectionCreatedCount\":\"0\",   \/\/\u4e0b\u4e00\u4e2a\u9636\u6bb5\u6216\u5b58\u50a8\u7cfb\u7edf\u521b\u5efa\u7684\u8fde\u63a5\u6570\u91cf\uff08\u5982HDFS\u521b\u5efa\u4e00\u4e2a\u65b0\u6587\u4ef6\uff09\n        \"BatchEmptyCount\":\"0\",      \/\/\u7a7a\u7684\u6279\u91cf\u7684\u6570\u91cf\uff0c\u5982\u679c\u6570\u91cf\u5f88\u5927\u8868\u793asouce\u5199\u6570\u636e\u6bd4sink\u6e05\u7406\u6570\u636e\u6162\u901f\u5ea6\u6162\u5f88\u591a\n        \"StopTime\":\"0\",         \n        \"RollbackCount\":\"9\",            \/\/\n        \"StartTime\":\"1442566411897\",\n        \"BatchUnderflowCount\":\"0\"       \/\/\u6bd4sink\u914d\u7f6e\u4f7f\u7528\u7684\u6700\u5927\u6279\u91cf\u5c3a\u5bf8\u66f4\u5c0f\u7684\u6279\u91cf\u7684\u6570\u91cf\uff0c\u5982\u679c\u8be5\u503c\u5f88\u9ad8\u4e5f\u8868\u793asink\u6bd4souce\u66f4\u5feb\n    }\n    }\n<\/code><\/pre>\n<p>flume\u5230kafka\u7684\u811a\u672c\u3002<\/p>\n<pre><code class=\"language-bash line-numbers\"># Name the components on this agent\n# a1\u662f\u6211\u4eec\u81ea\u5b9a\u4e49\u7684agent\u7684\u540d\u5b57\n# a1.sources  a1\u8fd9\u4e2aagent\u5305\u542b\u7684\u591a\u4e2asource\u7684\u540d\u5b57\n# \u5305\u542b\u51e0\u4e2a\u7ec4\u4ef6\u5c31\u5199\u4e00\u4e2a\u540d\u5b57\u7a7a\u683c\u9694\u5f00\na1.sources = r1\na1.sinks = k1\na1.channels = c1\n# Describe\/configure the source\n# \u914d\u7f6esource\n# \u914d\u7f6esource\na1.sources.r1.type = exec\na1.sources.r1.command = tail -F \/root\/mywebdata\/access.log\n# Describe the sink\n# \u914d\u7f6esink\na1.sinks.k1.channel = c1\na1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink\na1.sinks.k1.kafka.topic = mywebdata\na1.sinks.k1.kafka.bootstrap.servers = wq1:9092,wq2:9092,wq3:9092\na1.sinks.k1.kafka.flumeBatchSize = 1000\na1.sinks.k1.kafka.producer.acks = 1\na1.sinks.k1.kafka.producer.linger.ms = 1\na1.sinks.k1.kafka.producer.compression.type = snappy\n# Use a channel which buffers events in memory\n# \u914d\u7f6echannel\na1.channels.c1.type = memory\na1.channels.c1.capacity = 1000\na1.channels.c1.transactionCapacity = 100\n# Bind the source and sink to the channel\n# \u5c06\u7ec4\u4ef6\u8fde\u63a5\u8d77\u6765\na1.sources.r1.channels = c1\na1.sinks.k1.channel = c1\n<\/code><\/pre>\n<p>flume\u4ecekafka\u91c7\u96c6\u5230hdfs\u7684\u811a\u672c<\/p>\n<pre><code class=\"language-bash line-numbers\"># Name the components on this agent\na3.sources = r1\na3.sinks = k1\na3.channels = c1\n# Describe\/configure the source\na3.sources.r1.type = org.apache.flume.source.kafka.KafkaSource\na3.sources.r1.batchSize = 5000\na3.sources.r1.batchDurationMillis = 2000\na3.sources.r1.kafka.bootstrap.servers = wq1:9092,wq2:9092,wq3:9092\na3.sources.r1.kafka.consumer.auto.offset.reset = earliest\na3.sources.r1.kafka.topics = mywebdata    \n# Describe the sink\na3.sinks.k1.type = hdfs\na3.sinks.k1.hdfs.path = \/flume-hdfs\/\na3.sinks.k1.hdfs.filePrefix = webdata    \na3.sinks.k1.hdfs.rollInterval = 0\na3.sinks.k1.hdfs.rollSize = 67108864\na3.sinks.k1.hdfs.rollCount = 0\na3.sinks.k1.hdfs.fileType = DataStream\na3.sinks.k1.hdfs.writeFormat = Text\n# Use a channel which buffers events in memory\na3.channels.c1.type = file\na3.channels.c1.capacity = 10000\na3.channels.c1.byteCapacityBufferPercentage = 20\na3.channels.c1.transactionCapacity = 10000\na3.channels.c1.byteCapacity = 20000000\n# Bind the source and sink to the channel\na3.sources.r1.channels = c1\na3.sinks.k1.channel = c1\n<\/code><\/pre>\n<p>\u62a5\u9519<\/p>\n<pre><code class=\"language-java line-numbers\">ava.io.IOException: Callable timed out after 30000 ms on file: \/flume-hdfs\/\/webdata.1613873488282.tmp\n        at org.apache.flume.sink.hdfs.BucketWriter.callWithTimeout(BucketWriter.java:741)\n        at org.apache.flume.sink.hdfs.BucketWriter.doFlush(BucketWriter.java:517)\n        at org.apache.flume.sink.hdfs.BucketWriter.flush(BucketWriter.java:479)\n        at org.apache.flume.sink.hdfs.HDFSEventSink.process(HDFSEventSink.java:441)\n        at org.apache.flume.sink.DefaultSinkProcessor.process(DefaultSinkProcessor.java:67)\n        at org.apache.flume.SinkRunner$PollingRunner.run(SinkRunner.java:145)\n        at java.lang.Thread.run(Thread.java:748)\nCaused by: java.util.concurrent.TimeoutException\n        at java.util.concurrent.FutureTask.get(FutureTask.java:205)\n        at org.apache.flume.sink.hdfs.BucketWriter.callWithTimeout(BucketWriter.java:734)\n        ... 6 more\n<\/code><\/pre>\n<p>\u76ee\u524d\u53ef\u80fd\u60f3\u5230\u7684\u64cd\u4f5c\u5c31\u662f\u8be5\u673a\u5668\u4f5c\u4e3a\u6d88\u8d39\u8005\uff0c\u6700\u540e\u6ca1\u5173\u5bfc\u81f4\u95ee\u9898\u51fa\u73b0\uff08Soft lockup\u540d\u79f0\u89e3\u91ca\uff1a\u6240\u8c13\uff0csoft lockup\u5c31\u662f\u8bf4\uff0c\u8fd9\u4e2abug\u6ca1\u6709\u8ba9\u7cfb\u7edf\u5f7b\u5e95\u6b7b\u673a\uff0c\u4f46\u662f\u82e5\u5e72\u4e2a\u8fdb\u7a0b\uff08\u6216\u8005kernel thread\uff09\u88ab\u9501\u6b7b\u5728\u4e86\u67d0\u4e2a\u72b6\u6001\uff08\u4e00\u822c\u5728\u5185\u6838\u533a\u57df\uff09\uff0c\u5f88\u591a\u60c5\u51b5\u4e0b\u8fd9\u4e2a\u662f\u7531\u4e8e\u5185\u6838\u9501\u7684\u4f7f\u7528\u7684\u95ee\u9898\u3002\uff09\u5185\u6838\u53c2\u6570kernel.watchdog_thresh\uff08\/proc\/sys\/kernel\/watchdog_thresh\uff09\u7cfb\u7edf\u9ed8\u8ba4\u503c\u4e3a10\u3002\u5982\u679c\u8d85\u8fc72*10\u79d2\u4f1a\u6253\u5370\u4fe1\u606f\uff0c\u6ce8\u610f\uff1a\u8c03\u6574\u503c\u65f6\u53c2\u6570\u4e0d\u80fd\u5927\u4e8e60\u3002<\/p>\n<pre><code class=\"language-java line-numbers\">kernel:NMI watchdog: BUG: soft lockup - CPU#0 stuck for 24s! [kworker\/u256:1:21875]\n<\/code><\/pre>\n<p>\u89e3\u51b3\u529e\u6cd5<\/p>\n<pre><code class=\"language-bash line-numbers\">echo 30 &gt; \/proc\/sys\/kernel\/watchdog_thresh \n<\/code><\/pre>\n<h2><span class=\"ez-toc-section\" id=\"Hive%E6%95%B0%E4%BB%93\"><\/span>Hive\u6570\u4ed3<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u4e0b\u9762\u4e09\u4e2a\u6b65\u9aa4\u5c31\u662f\u5229\u7528\u6570\u4ed3\u5bf9\u6570\u636e\u5904\u7406\u6570\u636e\u65f6\u8d70\u8fc7\u7684\u51e0\u6b65<br \/>\n\u7b2c\u4e00\u6b65\uff1a\u67e5\u770b\u6570\u636e\u7684\u683c\u5f0f<\/p>\n<pre><code class=\"language-bash line-numbers\">[root@wq1 dt=26]# tail -f categories-.1603704061065 \n82      IPDDpvnbs\n94      QDTVDbatyc\n11      Ubk\n23      YCdjqay\n[root@wq1 dt=26]# tail -f customers-.1603704300625 \n999431  Dow Jones &amp; Co. 'Nnypet Ekmcidvwrnomfk' 'Gqzvqt Susckrpkfog'    26, Great North Circle  Montrose        2005-04-17      South   73159   Nigeria      (839) 0574180   (185) 54897386\n999491  Harrah's Entertainment Inc.     'Jpt Japhmiwb'  'Vxiwek Miquz'  5, Sanya Circle Mandalay        2004-05-09      East    64044   Bulgaria    (055) 6860602    (370) 10903682\n\n[root@wq1 dt=26]# tail -f employees-.1603706521271\n4739    Ellerkamp       Liza    'Sales Hpatgpvbn'       'Fla.'  1973-06-01      2019-01-05      163, West 5th Avenue    Zagreb  West    55828   Ghan'(03) 555-7341'  '3356'  WNGCMYQSE       WVDEA   66      75455.65\n2103    Mcleskey        Contessa        'Sales Ojwcaexw'        'Ipw.'  1973-03-30      2016-04-29      14, Alamosa Drive       Bakersfield     Nort52540    Finland '(14) 555-5542' '6383'  MUGIBADJF       VHUSQRVXHX      44      96812.09\n2948    Bigusiak        Markus  'Sales Jfnxtiypx'       'Jnv.'  1972-04-22      2018-11-23      225, Montrose Circle    Wrexham South   16134   Algeria      '(620) 555-8951'        '1129'  QEQAOYNJD       XGGZSBTX        13      78077.37\n9306    Lavgle  Blondell        'Sales Vbcsraacanf'     'Ogf.'  1988-06-27      2016-12-20      61, Melody Lane Crystal Cove    North   89766   Tuvalu       '(694) 555-0165'        '3189'  DJNOS   KW      49      77783.98\n\n[root@wq1 dt=26]# tail -f  orders-.1603708021871\n2107109 395777  7721    2015-01-20      2018-12-23      2016-05-17      3       69606.79        Laila Mcglockton        7, Cumberland Circle    Jatiluhur    North   15685   Tanzania\n2107121 460594  9092    2018-10-26      2016-09-20      2017-04-07      1       12460.05        Delana Chiotti  83, Bear Mountain View Circle   Oklahoma City        East    47512   Eritrea\n2107133 582440  2649    2017-06-28      2015-06-01      2019-05-28      2       397072.97       Jennefer Freudiger      79, Viewpark Road       Okinawa      West    58620   Israel\n2107145 151015  6753    2017-01-26      2019-01-28      2015-08-12      2       650344.50       Alana Kawasaki  23, H And H Avenue      Colon   West57409    Switzerland\n\n[root@wq1 dt=26]# tail -f orderdetails-.1603699455177\n2441191 9014    259.92  91      0.42\n8467405 4039    83.91   21      0.30\n7858542 1714    189.81  70      0.46\n7872718 2661    207.48  173     0.00\n\n[root@wq1 dt=26]# tail -f products-.1603703609450 \n9409    YVMSauyq        6       1       '1 jng qo 44 vctsfkw'   158.71\n9469    NJkxkwa 7       2       '77 omzj'       35.75\n9529    JOOMRpptrr      14      9       '265 ym'        38.65\n9589    Dbhyba  15      9       '65 - 112 b uwkfn'      96.62\n\n<\/code><\/pre>\n<p>\u7b2c\u4e8c\u6b65:\u6570\u4ed3\u5206\u5c42\u8bbe\u8ba1\u8fc7\u7a0b<br \/>\n<img src=\"https:\/\/www.specialwu.com\/wp-content\/uploads\/2021\/04\/20210401\u5927\u6570\u636e\u9879\u76ee\u8fc7\u7a0b.png\" alt=\"\" \/><\/p>\n<p>\u5728\u67e5\u770b\u8fc7\u6570\u636e\u7684\u7ed3\u6784\u540e\uff0c\u7ed8\u5236\u4e00\u5f20\u8868\u683c\u5bf9\u6bcf\u5c42\u6240\u9700\u5b57\u6bb5\u8fdb\u884c\u68b3\u7406\uff0c\u540e\u671f\u5b57\u6bb5\u53d8\u52a8\u4e5f\u65b9\u4fbf\u67e5\u627e\u63d0\u9ad8\u6548\u7387\uff1b\u6570\u4ed3\u5728\u8bbe\u8ba1\u8fc7\u7a0b\u4e2d\u6839\u636e\u81ea\u5df1\u7684\u4e1a\u52a1\u8fdb\u884c\u5904\u7406\u3002\u4e3a\u4ec0\u4e48\u8981\u5206\u5c42\uff0c\u9996\u5148\uff0c\u6570\u636e\u91cf\u5927\uff0c\u6e90\u6570\u636e\u6709\u65f6\u5b57\u6bb5\u591a\u5927\u4e0a\u5343\u4e2a\uff0c\u5982\u679c\u4e0d\u8fdb\u884c\u5206\u5c42\u68b3\u7406\uff0c\u90a3\u4e48\u5904\u7406\u8d77\u6765\u6548\u7387\u6781\u4f4e\uff0c\u5176\u6b21\uff0c\u6bcf\u5c42\u90fd\u53ef\u4ee5\u5bf9\u5e94\u4e00\u5b9a\u7684\u4e1a\u52a1\u9700\u6c42\uff0c\u5176\u6b21\uff0c\u6bcf\u5c42\u4e0b\u5206\u62c5\u4e00\u5b9a\u6570\u636e\u91cf\uff0c\u51cf\u8f7b\u96c6\u7fa4\u538b\u529b\uff0c\u51cf\u5c11\u6570\u636e\u5904\u7406\u65f6\u95f4\uff0c\u6700\u540e\uff0c\u6bcf\u4e00\u5c42\uff0c\u53ef\u5bf9\u63a5\u4e00\u90e8\u5206\u4e1a\u52a1\uff1b<\/p>\n<p><code>ODS(Operational Data Store)<\/code>\u64cd\u4f5c\u6570\u636e\u5c42,\u7ec6\u8282\u6570\u636e\u67e5\u8be2\u7684\u529f\u80fd\u5728ODS\u6765\u5b8c\u6210,\u4e0e\u6e90\u7cfb\u7edf\u7684\u589e\u91cf\u6216\u8005\u5168\u91cf\u6570\u636e\u57fa\u672c\u4fdd\u6301\u4e00\u81f4\u3002\u603b\u7684\u6765\u8bf4\u8fd9\u4e00\u5c42\u5305\u542b\u6e90\u6570\u636e\u6240\u6709\u5b57\u6bb5<\/p>\n<table>\n<thead>\n<tr>\n<th>\u8868\u540d<\/th>\n<th>employees_dwd<\/th>\n<th>\u5458\u5de5\u4fe1\u606f\u660e\u7ec6\u8868<\/th>\n<th><\/th>\n<th><\/th>\n<\/tr>\n<\/thead>\n<tbody>\n<tr>\n<td>\u5b57\u6bb5<\/td>\n<td>employeeId<\/td>\n<td>emloyeeIdname<\/td>\n<td>title<\/td>\n<td>reportTo<\/td>\n<\/tr>\n<tr>\n<td>\u5b57\u6bb5\u7c7b\u578b<\/td>\n<td>int<\/td>\n<td>string<\/td>\n<td>string<\/td>\n<td>string<\/td>\n<\/tr>\n<tr>\n<td>\u5b57\u6bb5\u6620\u5c04<\/td>\n<td>\u5458\u5de5id<\/td>\n<td>\u5458\u5de5\u59d3\u540d<\/td>\n<td>\u804c\u4f4d<\/td>\n<td>\u4e0a\u7ea7<\/td>\n<\/tr>\n<tr>\n<td>\u8868\u95f4\u5173\u7cfb<\/td>\n<td>LOAD DATA INPATH &#8216;hdfs:\/\/wq1:9000\/source-six\/employee&#8217; overwrite into table employees_ods partition  (yt=&#8221;2021&#8243;,mt=&#8221;03&#8243;,dt=&#8221;29&#8243;);<\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p><code>CDM(Common Data Model)<\/code>:\u516c\u5171\u7ef4\u5ea6\u6a21\u578b\u5c42\uff1b\u7ec6\u5206\u4e3aDWD\u548cDWS\uff1b\u4e3b\u8981\u4f5c\u7528\u662f\u5b8c\u6210\u6570\u636e\u52a0\u5de5\u4e0e\u6574\u5408\u3001\u5efa\u7acb\u4e00\u81f4\u6027\u7684\u7ef4\u5ea6\u3001\u6784\u5efa\u53ef\u590d\u7528\u7684\u9762\u5411\u5206\u6790\u548c\u7edf\u8ba1\u7684\u660e\u7ec6\u4e8b\u5b9e\u8868\u4ee5\u53ca\u6c47\u603b\u516c\u5171\u7c92\u5ea6\u7684\u6307\u6807\u3002<br \/>\n<code>DWD(Data Warehouse Detail)\uff1a\u660e\u7ec6\u6570\u636e\u5c42<\/code><\/p>\n<table>\n<thead>\n<tr>\n<th>\u8868\u540d<\/th>\n<th>employees_dwd<\/th>\n<th>\u5458\u5de5\u4fe1\u606f\u660e\u7ec6\u8868<\/th>\n<th><\/th>\n<th><\/th>\n<\/tr>\n<\/thead>\n<tbody>\n<tr>\n<td>\u5b57\u6bb5<\/td>\n<td>employeeId2dwd<\/td>\n<td>emloyeeIdname2dwd<\/td>\n<td>title2dwd<\/td>\n<td>reportTo2dwd<\/td>\n<\/tr>\n<tr>\n<td>\u5b57\u6bb5\u7c7b\u578b<\/td>\n<td>int<\/td>\n<td>string<\/td>\n<td>string<\/td>\n<td>string<\/td>\n<\/tr>\n<tr>\n<td>\u5b57\u6bb5\u6765\u6e90<\/td>\n<td>a.employeeId<\/td>\n<td>concat(a.firstName,a.lastname)<\/td>\n<td>a.title<\/td>\n<td>b.emloyeeIdname2dwd<\/td>\n<\/tr>\n<tr>\n<td>\u5b57\u6bb5\u6620\u5c04<\/td>\n<td>\u5458\u5de5id<\/td>\n<td>\u5458\u5de5\u59d3\u540d<\/td>\n<td>\u804c\u4f4d<\/td>\n<td>\u4e0a\u7ea7<\/td>\n<\/tr>\n<tr>\n<td>\u5efa\u8868\u7c7b\u578b<\/td>\n<td>\u4e8b\u5b9e\u660e\u7ec6\u5c42<\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<\/tr>\n<tr>\n<td>\u8868\u95f4\u5173\u7cfb<\/td>\n<td>FROM shoppingmall.employees_ods a INNER JOIN wushi_cdm.duty_dwd b ON a.reportTo=b.employeeId<\/td>\n<td><\/td>\n<td><\/td>\n<td><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p><code>DWS(Data Warehouse Summary):\u6c47\u603b\u6570\u636e\u5c42<\/code><\/p>\n<table>\n<thead>\n<tr>\n<th>\u8868\u540d<\/th>\n<th>employeehm_dws<\/th>\n<th>\u5458\u5de5\u5165\u804c\u6708\u4efd\u7ef4\u5ea6\u8868<\/th>\n<\/tr>\n<\/thead>\n<tbody>\n<tr>\n<td>\u5b57\u6bb5<\/td>\n<td>employeeId2dws<\/td>\n<td>month2dws<\/td>\n<\/tr>\n<tr>\n<td>\u5b57\u6bb5\u7c7b\u578b<\/td>\n<td>int<\/td>\n<td>string<\/td>\n<\/tr>\n<tr>\n<td>\u5b57\u6bb5\u6765\u6e90<\/td>\n<td>employeeId<\/td>\n<td>month(birthdate)<\/td>\n<\/tr>\n<tr>\n<td>\u5b57\u6bb5\u6620\u5c04<\/td>\n<td>\u5458\u5de5id<\/td>\n<td>\u5165\u804c\u6708\u4efd<\/td>\n<\/tr>\n<tr>\n<td>\u5efa\u8868\u7c7b\u578b<\/td>\n<td>\u6c47\u603b\u7ef4\u5ea6\u5c42<\/td>\n<td><\/td>\n<\/tr>\n<tr>\n<td>\u8868\u95f4\u5173\u7cfb<\/td>\n<td>FROM category_ods<\/td>\n<td><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p><code>ADS(Application Data Service):\u5e94\u7528\u6570\u636e\u5c42<\/code><\/p>\n<table>\n<thead>\n<tr>\n<th>\u8868\u540d<\/th>\n<th>saletop10_ads<\/th>\n<th>\u9500\u552e\u7247\u533aTop10\u8be6\u60c5\u8868<\/th>\n<\/tr>\n<\/thead>\n<tbody>\n<tr>\n<td>\u5b57\u6bb5<\/td>\n<td>producename2ads<\/td>\n<td>sale2ads<\/td>\n<\/tr>\n<tr>\n<td>\u5b57\u6bb5\u7c7b\u578b<\/td>\n<td>int<\/td>\n<td>string<\/td>\n<\/tr>\n<tr>\n<td>\u5b57\u6bb5\u6765\u6e90<\/td>\n<td>b.producename<\/td>\n<td>sum(a.unitprice<em>quantity<\/em>discount)<\/td>\n<\/tr>\n<tr>\n<td>\u5b57\u6bb5\u6620\u5c04<\/td>\n<td>\u5546\u54c1\u540d\u79f0<\/td>\n<td>\u9500\u552e\u989d<\/td>\n<\/tr>\n<tr>\n<td>\u5efa\u8868\u7c7b\u578b<\/td>\n<td>\u6c47\u603b\u7ef4\u5ea6\u5c42<\/td>\n<td><\/td>\n<\/tr>\n<tr>\n<td>\u8868\u95f4\u5173\u7cfb<\/td>\n<td>FROM (SELECT b.producename,sum(a.unitprice<em>quantity<\/em>discount) c FROM shoppingmall.orderdetails_ods a INNER JOIN shoppingmall.products_ods  b ON a.productid=b.productid GROUP BY b.producename ORDER BY c limit 10\uff09tmp;<\/td>\n<td><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<p>\u7b2c\u4e09\u6b65\uff1a\u5404\u5c42\u5efa\u8868\u8bed\u53e5\uff0c\u540c\u6b65\u8fc7\u7a0b<\/p>\n<ol>\n<li>ods\u5c42<\/li>\n<\/ol>\n<ul>\n<li>\u5efa\u5e93\u5efa\u8868<\/li>\n<\/ul>\n<pre><code class=\"language-sql line-numbers\">CREATE EXTERNAL TABLE IF NOT EXISTS employees_ods_t\n(\nemployeeId INT COMMENT \"\u5458\u5de5id\",\nlastname STRING COMMENT \"\u5458\u5de5\u540d\",\nfirstName STRING COMMENT \"\u5458\u5de5\u59d3\",\ntitle STRING COMMENT \"\u5934\u8854\",\ntitleofCountry STRING COMMENT \"\u6240\u5c5e\u56fd\u5bb6\",\nbirthdate DATE COMMENT \"\u51fa\u751f\u65e5\u671f\",\nhiredate DATE COMMENT \"\u5165\u804c\u65f6\u95f4\",\naddress STRING COMMENT \"\u5730\u5740\",\ncity STRING COMMENT \"\u57ce\u5e02\",\nregion STRING COMMENT \"\u7247\u533a\",\npostalCode STRING COMMENT \"\u90ae\u7f16\",\ncountry STRING COMMENT \"\u56fd\u7c4d\",\nhomePhone STRING COMMENT \"\u7535\u8bdd\",\nextension STRING COMMENT \"\u9644\u52a0\",\nphoto STRING COMMENT \"\u7167\u7247\",\nnote STRING COMMENT \"\u6ce8\u610f\",\nreportTo INT COMMENT \"\u4e0a\u7ea7\u9886\u5bfc\",\nsalary DOUBLE COMMENT \"\u5de5\u8d44\"\n)\nCOMMENT \"\u5458\u5de5\u8868\"\npartitioned by (yt string,mt string,dt string)\nrow format delimited fields terminated by '\\t'\nLOCATION '\/user\/hive\/warehouse\/ods_database.db\/employees_ods_t';\n<\/code><\/pre>\n<ul>\n<li>\u6620\u5c04hdfs\u7684\u6570\u636e\u5230hive\u4e2d\uff08hdfs\u7684\u6587\u4ef6\u53ea\u662f\u6362\u4e24\u4e2a\u4e0ehive\u8868location\u76f8\u540c\u7684\u4f4d\u7f6e\u800c\u5df2\uff09<\/li>\n<\/ul>\n<pre><code class=\"language-sql line-numbers\">LOAD DATA INPATH 'hdfs:\/\/wq1:9000\/source-six\/employee' overwrite into table employees_ods partition (yt=\"2021\",mt=\"03\",dt=\"29\");\n<\/code><\/pre>\n<ul>\n<li>\u786e\u8ba4\u6570\u636e<\/li>\n<\/ul>\n<pre><code class=\"language-sql line-numbers\">select * from employees_ods limit 10;\n#\u5982\u679c\u67e5\u770bhdfs\u96c6\u7fa4\u4e0a\u7684\u90a3\u4e2aLOCATION\u6709\u6570\u636e\uff0c\u4f46\u662f\u5728hive\u4e2d\u67e5\u4e0d\u51fa\u6570\u636e\uff0c\u6267\u884c\u4e0b\u9762\u8fd9\u4e2a\uff0c\u4e0d\u5c0f\u5fc3\u5220\u9664\u5916\u90e8\u8868\u540e\uff0c\u53ef\u518d\u6267\u884c\u8be5\u547d\u4ee4\nMSCK REPAIR TABLE employees_ods\uff1b\n<\/code><\/pre>\n<ol start=\"2\">\n<li>DWD\u5c42<\/li>\n<\/ol>\n<ul>\n<li>\u5efa\u5e93\u5efa\u8868<\/li>\n<\/ul>\n<pre><code class=\"language-sql line-numbers\">CREATE EXTERNAL TABLE IF NOT EXISTS employees_dwd\n(\nemployeeId INT COMMENT \"\u5458\u5de5id\",\nemloyeeIdname2dwd STRING COMMENT \"\u5458\u5de5\u59d3\u540d\",\ntitle STRING COMMENT \"\u5934\u8854\",\nreportTo INT COMMENT \"\u4e0a\u7ea7\u9886\u5bfc\",\n)\nCOMMENT \"\u5458\u5de5\u4fe1\u606f\u660e\u7ec6\u8868\"\npartitioned by (yt string,mt string,dt string)\nLOCATION '\/user\/hive\/warehouse\/wushi_cdm.db\/employees_dwd';\n<\/code><\/pre>\n<ul>\n<li>\u6570\u636e\u540c\u6b65\u3001\u5904\u7406<\/li>\n<\/ul>\n<pre><code class=\"language-sql line-numbers\">insert \ninto  employees_dwd partition(yt=\"2021\",mt=\"04\",dt=\"01\")\nselect\na.employeeId          ,\nconcat(a.firstName,a.lastname),\na.title       ,\nb.emloyeeIdname2dwd          ,\nFROM shoppingmall.employees_ods a\nINNER JOIN\nwushi_cdm.duty_dwd b\nON\na.reportTo=b.employeeId2dwd;\n<\/code><\/pre>\n<ol start=\"3\">\n<li>ADS<\/li>\n<\/ol>\n<pre><code class=\"language-sql line-numbers\">create table sourth \nas  \nselect\nb.producename ,\nsum(a.unitprice*quantity*discount) \nas c  \nfrom \npdw_database.orderdetails_pdw_t \nas a  \ninner join \npdw_database.products_pdw_t \nas b  \non a.productid=b.productid  \ninner join\npdw_database.orders_pdw_t \nas e \non a.orderid=e.orderid \nwhere e.shipregion='South' \ngroup by b.producename \norder by c\nlimit 10\n<\/code><\/pre>\n<blockquote><p>\n  \u81ea\u5b9a\u4e49hive\u51fd\u6570\n<\/p><\/blockquote>\n<h2><span class=\"ez-toc-section\" id=\"%E5%B0%86%E6%95%B0%E6%8D%AE%E5%AF%BC%E5%85%A5MySQL\"><\/span>\u5c06\u6570\u636e\u5bfc\u5165MySQL<span class=\"ez-toc-section-end\"><\/span><\/h2>\n<p>\u5bfc\u5165MySQL\u524d\uff0c\u5728\u6570\u636e\u5e93\u4e0a\u5efa\u540c\u6837\u7ed3\u6784\u7684\u8868<\/p>\n<pre><code class=\"language-sql line-numbers\">CREATE  TABLE  lake \n( \n    table_name  VARCHAR(20) COMMENT \"\u8868\u540d\", \n    FieldName VARCHAR(20) COMMENT \"\u5217\u540d\" \n);\n<\/code><\/pre>\n<p>sqoop\u5bfc\u6570\u636e\u7684\u811a\u672c\uff08\u6309\u7167\u5236\u8868\u7b26\u5206\u9694\uff0cmap\u6570\u91cf\u4e3a1\uff09<\/p>\n<pre><code class=\"language-bash line-numbers\">sqoop export \n--connect jdbc:mysql:\/\/localhost:3306\/system_db\n--username root \n--password 123456 \n--table lake \n--export-dir \/user\/hive\/warehouse\/system_db.db\/lake\/year=2021\/month=04\/day=01\/lake.txt\n--fields-terminated-by '\\t' \n--m 1\n<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"<p>\u524d\u8a00 \u672c\u6765\u524d\u8a00\u8fd9\u90e8\u5206\u51c6\u5907\u4e86\u4e00\u5927\u5806\u8bdd\u6765\u63cf\u8ff0\u4e0b\u672c\u7bc7\u8981\u4ecb\u7ecd\u7684\u5185\u5bb9\uff0c\u7b97\u4e86\uff0c\u770b\u56fe\u5427\uff0c\u672c\u7bc7\u4f1a\u7ed9\u4f60\u6253\u6765\u7684\u6700\u5927\u5e2e\u52a9\u5c31\u662f\uff0c\u5448\u5148\u5927 [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[7],"tags":[],"_links":{"self":[{"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/posts\/1786"}],"collection":[{"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=1786"}],"version-history":[{"count":17,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/posts\/1786\/revisions"}],"predecessor-version":[{"id":2282,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/posts\/1786\/revisions\/2282"}],"wp:attachment":[{"href":"http:\/\/www.specialwu.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=1786"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=1786"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=1786"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}