{"id":410,"date":"2020-10-21T22:11:46","date_gmt":"2020-10-21T14:11:46","guid":{"rendered":"https:\/\/www.specialwu.com\/?p=410"},"modified":"2021-03-30T17:46:13","modified_gmt":"2021-03-30T09:46:13","slug":"cdh005","status":"publish","type":"post","link":"http:\/\/www.specialwu.com\/?p=410","title":{"rendered":"CDH005"},"content":{"rendered":"<div id=\"ez-toc-container\" class=\"ez-toc-v2_0_61 ez-toc-wrap-center counter-hierarchy ez-toc-counter ez-toc-custom ez-toc-container-direction\">\n<div class=\"ez-toc-title-container\">\n<p class=\"ez-toc-title \" >\u76ee\u5f55<\/p>\n<span class=\"ez-toc-title-toggle\"><a href=\"#\" class=\"ez-toc-pull-right ez-toc-btn ez-toc-btn-xs ez-toc-btn-default ez-toc-toggle\" aria-label=\"Toggle Table of Content\"><span class=\"ez-toc-js-icon-con\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #000000;color:#000000\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #000000;color:#000000\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/span><\/a><\/span><\/div>\n<nav><ul class='ez-toc-list ez-toc-list-level-1 ' ><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"http:\/\/www.specialwu.com\/?p=410\/#%E5%89%8D%E8%A8%80\" title=\"\u524d\u8a00\">\u524d\u8a00<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"http:\/\/www.specialwu.com\/?p=410\/#CDH%E6%90%AD%E5%BB%BA%E6%AD%A5%E9%AA%A4\" title=\"CDH\u642d\u5efa\u6b65\u9aa4\">CDH\u642d\u5efa\u6b65\u9aa4<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-3\" href=\"http:\/\/www.specialwu.com\/?p=410\/#%E5%8E%9F%E5%A7%8B%E6%95%B0%E6%8D%AE%E4%BF%A1%E6%81%AF%E8%A1%A8\" title=\"\u539f\u59cb\u6570\u636e\u4fe1\u606f\u8868\">\u539f\u59cb\u6570\u636e\u4fe1\u606f\u8868<\/a><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-4\" href=\"http:\/\/www.specialwu.com\/?p=410\/#%E9%9B%86%E7%BE%A4%E9%85%8D%E7%BD%AE%E6%83%85%E5%86%B5\" title=\"\u96c6\u7fa4\u914d\u7f6e\u60c5\u51b5\">\u96c6\u7fa4\u914d\u7f6e\u60c5\u51b5<\/a><\/li><\/ul><\/li><\/ul><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-5\" href=\"http:\/\/www.specialwu.com\/?p=410\/#1%E3%80%81%E9%9C%80%E6%B1%82%E5%88%86%E6%9E%90%EF%BC%88%E6%A2%B3%E7%90%86%E8%A1%A8%E7%BB%93%E6%9E%84%EF%BC%8C%E6%A2%B3%E7%90%86%E5%85%B6%E5%BA%A6%E9%87%8F%EF%BC%89\" title=\"1\u3001\u9700\u6c42\u5206\u6790\uff08\u68b3\u7406\u8868\u7ed3\u6784\uff0c\u68b3\u7406\u5176\u5ea6\u91cf\uff09\">1\u3001\u9700\u6c42\u5206\u6790\uff08\u68b3\u7406\u8868\u7ed3\u6784\uff0c\u68b3\u7406\u5176\u5ea6\u91cf\uff09<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-6\" href=\"http:\/\/www.specialwu.com\/?p=410\/#2%E3%80%81%E9%9B%86%E7%BE%A4%E6%9E%B6%E6%9E%84%E8%AE%BE%E8%AE%A1%EF%BC%88%E4%BD%BF%E7%94%A8%E9%82%A3%E4%BA%9B%E7%BB%84%E4%BB%B6%E3%80%81%E6%89%80%E6%9C%89%E7%BB%84%E4%BB%B6%E5%92%8C%E6%9C%BA%E5%99%A8%E7%9A%84%E5%AF%B9%E5%BA%94%E5%85%B3%E7%B3%BB%EF%BC%89\" title=\"2\u3001\u96c6\u7fa4\u67b6\u6784\u8bbe\u8ba1\uff08\u4f7f\u7528\u90a3\u4e9b\u7ec4\u4ef6\u3001\u6240\u6709\u7ec4\u4ef6\u548c\u673a\u5668\u7684\u5bf9\u5e94\u5173\u7cfb\uff09\">2\u3001\u96c6\u7fa4\u67b6\u6784\u8bbe\u8ba1\uff08\u4f7f\u7528\u90a3\u4e9b\u7ec4\u4ef6\u3001\u6240\u6709\u7ec4\u4ef6\u548c\u673a\u5668\u7684\u5bf9\u5e94\u5173\u7cfb\uff09<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-7\" href=\"http:\/\/www.specialwu.com\/?p=410\/#3%E3%80%81%E6%95%B0%E6%8D%AE%E5%A4%84%E7%90%86%E6%B5%81%E7%A8%8B%E8%AE%BE%E8%AE%A1%EF%BC%88%E6%95%B0%E6%8D%AE%E6%B5%81%E7%A8%8B%E5%9B%BE%EF%BC%89\" title=\"3\u3001\u6570\u636e\u5904\u7406\u6d41\u7a0b\u8bbe\u8ba1\uff08\u6570\u636e\u6d41\u7a0b\u56fe\uff09\">3\u3001\u6570\u636e\u5904\u7406\u6d41\u7a0b\u8bbe\u8ba1\uff08\u6570\u636e\u6d41\u7a0b\u56fe\uff09<\/a><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-8\" href=\"http:\/\/www.specialwu.com\/?p=410\/#%E5%85%AD%E5%BC%A0%E8%A1%A8%E4%B9%8B%E9%97%B4%E7%9A%84%E5%85%B3%E7%B3%BB\" title=\"\u516d\u5f20\u8868\u4e4b\u95f4\u7684\u5173\u7cfb\">\u516d\u5f20\u8868\u4e4b\u95f4\u7684\u5173\u7cfb<\/a><\/li><\/ul><\/li><\/ul><\/li><\/ul><\/li><\/ul><\/nav><\/div>\n<h1><span class=\"ez-toc-section\" id=\"%E5%89%8D%E8%A8%80\"><\/span>\u524d\u8a00<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<p>\u867d\u7136\u8fd9\u662f\u4e00\u4e2a\u7528CDH\u642d\u5efa\u8d77\u6765\u7684\u96c6\u7fa4\uff0c\u672c\u7bc7\u524d\u534a\u90e8\u5206\u5927\u6982\u5c55\u793a\u4e0bCDH\u642d\u5efa\u8fc7\u7a0b\uff0c\u4ee5\u53ca\u642d\u5efa\u597d\u540e\u5b83\u7684\u6837\u5b50\u662f\u4ec0\u4e48\uff0c\u4f46\u662f\u4e3b\u8981\u4fa7\u91cd\u8bb0\u5f55\u5355\u72ec\u67d0\u4e00\u7ec4\u4ef6\u53d1\u6325\u7684\u4f5c\u7528<\/p>\n<h1><span class=\"ez-toc-section\" id=\"CDH%E6%90%AD%E5%BB%BA%E6%AD%A5%E9%AA%A4\"><\/span>CDH\u642d\u5efa\u6b65\u9aa4<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<ol>\n<li>\u4e0b\u8f7dClouderaManager<br \/>\n<a class=\"wp-editor-md-post-content-link\" href=\"https:\/\/archive.cloudera.com\/cm6\/6.3.1\/redhat7\/yum\/RPMS\/x86_64\/\" title=\"\u70b9\u51fb\u8fdb\u5165\u4e0b\u8f7d\u754c\u9762\">\u70b9\u51fb\u8fdb\u5165\u4e0b\u8f7d\u754c\u9762<\/a><\/p>\n<\/li>\n<li>\n<p>\u73af\u5883\u51c6\u5907<\/p>\n<\/li>\n<\/ol>\n<ul>\n<li>\u5404\u4e2a\u4e3b\u673a\u540d\u4e4b\u95f4\u8bbe\u7f6e\u6620\u5c04<\/li>\n<li>\u5173\u95ed\u5e76\u7981\u7528\u9632\u706b\u5899<\/li>\n<li>\u5173\u95edSELinux<\/li>\n<li>\u8bbe\u7f6eSSH\u514d\u5bc6\u767b\u9646(\u6240\u6709\u8282\u70b9)<\/li>\n<li>\u7981\u7528\u900f\u660e\u5927\u9875(\u6240\u6709\u8282\u70b9)<\/li>\n<li>\u4fee\u6539Linux swappiness\u53c2\u6570(\u6240\u6709\u8282\u70b9)<\/li>\n<li>\u5b89\u88c5JDK(\u6240\u6709\u8282\u70b9)\u8981\u6c42\u4f7f\u7528\u6307\u5b9a\u7248\u672c\u7684oracle-j2sdk1.8<\/li>\n<li>\u4e0a\u4f20JDBC\u4f9d\u8d56\u5305(\u6240\u6709\u8282\u70b9)<\/li>\n<\/ul>\n<ol start=\"3\">\n<li>\u5b89\u88c5<\/li>\n<\/ol>\n<ul>\n<li>\u5b89\u88c5MySQL(master\u8282\u70b9\uff09<\/li>\n<li>\u5b89\u88c5Apache Httpd\u670d\u52a1(master\u8282\u70b9\uff09<\/li>\n<li>\u914d\u7f6eCloudera Manager\u5b89\u88c5\u5305yum\u6e90(master\u8282\u70b9)<\/li>\n<li>\u5b89\u88c5 Cloudera Manager(master\u8282\u70b9)<\/li>\n<\/ul>\n<h1><span class=\"ez-toc-section\" id=\"%E5%8E%9F%E5%A7%8B%E6%95%B0%E6%8D%AE%E4%BF%A1%E6%81%AF%E8%A1%A8\"><\/span>\u539f\u59cb\u6570\u636e\u4fe1\u606f\u8868<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<blockquote>\n<p>\u540e\u671f\u7684\u64cd\u4f5c\u4e3b\u8981\u56f4\u7ed5\u4e0b\u9762\u8fd9\u516d\u5f20\u8868\u7684\u6570\u636e\u5c55\u5f00\n<\/p><\/blockquote>\n\n<table id=\"tablepress-3\" class=\"tablepress tablepress-id-3\">\n<tbody class=\"row-hover\">\n<tr class=\"row-1 odd\">\n\t<td class=\"column-1\">Products(\u4ea7\u54c1\u8868\uff09<\/td><td class=\"column-2\"><\/td><td class=\"column-3\"><\/td><td class=\"column-4\"><\/td>\n<\/tr>\n<tr class=\"row-2 even\">\n\t<td class=\"column-1\">\u5217\u540d\u79f0<\/td><td class=\"column-2\">\u7c7b\u578b<\/td><td class=\"column-3\">\u6570\u91cf<\/td><td class=\"column-4\">\u63cf\u8ff0<\/td>\n<\/tr>\n<tr class=\"row-3 odd\">\n\t<td class=\"column-1\">productid<\/td><td class=\"column-2\">int<\/td><td class=\"column-3\">10000<\/td><td class=\"column-4\">\u4ea7\u54c1id<\/td>\n<\/tr>\n<tr class=\"row-4 even\">\n\t<td class=\"column-1\">produceName<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">10000<\/td><td class=\"column-4\">\u4ea7\u54c1\u540d<\/td>\n<\/tr>\n<tr class=\"row-5 odd\">\n\t<td class=\"column-1\">supplier<\/td><td class=\"column-2\">int<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u4f9b\u5e94\u5546<\/td>\n<\/tr>\n<tr class=\"row-6 even\">\n\t<td class=\"column-1\">categoryID<\/td><td class=\"column-2\">int<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u4ea7\u54c1\u7c7b\u578bID<\/td>\n<\/tr>\n<tr class=\"row-7 odd\">\n\t<td class=\"column-1\">quantityPerUint<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u6bcf\u79cd\u4ea7\u54c1\u7684\u6570\u91cf<\/td>\n<\/tr>\n<tr class=\"row-8 even\">\n\t<td class=\"column-1\">unitPrice<\/td><td class=\"column-2\">double<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u5355\u4ef7<\/td>\n<\/tr>\n<tr class=\"row-9 odd\">\n\t<td class=\"column-1\">Employee\uff08\u5458\u5de5\u8868\uff09<\/td><td class=\"column-2\"><\/td><td class=\"column-3\"><\/td><td class=\"column-4\"><\/td>\n<\/tr>\n<tr class=\"row-10 even\">\n\t<td class=\"column-1\">\u5217\u540d\u79f0<\/td><td class=\"column-2\">\u7c7b\u578b<\/td><td class=\"column-3\">\u6570\u91cf<\/td><td class=\"column-4\">\u63cf\u8ff0<\/td>\n<\/tr>\n<tr class=\"row-11 odd\">\n\t<td class=\"column-1\">employeeId<\/td><td class=\"column-2\">int<\/td><td class=\"column-3\">10000<\/td><td class=\"column-4\">\u5458\u5de5id<\/td>\n<\/tr>\n<tr class=\"row-12 even\">\n\t<td class=\"column-1\">lastname<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">10000<\/td><td class=\"column-4\">\u59d3\u540d<\/td>\n<\/tr>\n<tr class=\"row-13 odd\">\n\t<td class=\"column-1\">firstName<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">2000<\/td><td class=\"column-4\">\u540d\u5b57<\/td>\n<\/tr>\n<tr class=\"row-14 even\">\n\t<td class=\"column-1\">title<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">200<\/td><td class=\"column-4\">\u804c\u4f4d<\/td>\n<\/tr>\n<tr class=\"row-15 odd\">\n\t<td class=\"column-1\">titleofCountry<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">100<\/td><td class=\"column-4\"><\/td>\n<\/tr>\n<tr class=\"row-16 even\">\n\t<td class=\"column-1\">birthdate<\/td><td class=\"column-2\">date<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u51fa\u751f\u65e5\u671f<\/td>\n<\/tr>\n<tr class=\"row-17 odd\">\n\t<td class=\"column-1\">hiredate<\/td><td class=\"column-2\">date<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u5165\u804c\u65f6\u95f4<\/td>\n<\/tr>\n<tr class=\"row-18 even\">\n\t<td class=\"column-1\">address<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u4f4f\u5740<\/td>\n<\/tr>\n<tr class=\"row-19 odd\">\n\t<td class=\"column-1\">city<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">100<\/td><td class=\"column-4\">\u57ce\u5e02<\/td>\n<\/tr>\n<tr class=\"row-20 even\">\n\t<td class=\"column-1\">region<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">10<\/td><td class=\"column-4\">\u7247\u533a<\/td>\n<\/tr>\n<tr class=\"row-21 odd\">\n\t<td class=\"column-1\">postalCode<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">100<\/td><td class=\"column-4\">\u90ae\u7f16<\/td>\n<\/tr>\n<tr class=\"row-22 even\">\n\t<td class=\"column-1\">country<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">3<\/td><td class=\"column-4\">\u56fd\u5bb6<\/td>\n<\/tr>\n<tr class=\"row-23 odd\">\n\t<td class=\"column-1\">homePhone<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">10000<\/td><td class=\"column-4\">\u5bb6\u5ead\u7535\u8bdd<\/td>\n<\/tr>\n<tr class=\"row-24 even\">\n\t<td class=\"column-1\">extension<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u5907\u6ce8<\/td>\n<\/tr>\n<tr class=\"row-25 odd\">\n\t<td class=\"column-1\">photo<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u5934\u50cf<\/td>\n<\/tr>\n<tr class=\"row-26 even\">\n\t<td class=\"column-1\">note<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\"><\/td><td class=\"column-4\"><\/td>\n<\/tr>\n<tr class=\"row-27 odd\">\n\t<td class=\"column-1\">reportTo<\/td><td class=\"column-2\">int<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u4e0a\u7ea7\u9886\u5bfc<\/td>\n<\/tr>\n<tr class=\"row-28 even\">\n\t<td class=\"column-1\">salary<\/td><td class=\"column-2\">double<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u85aa\u6c34<\/td>\n<\/tr>\n<tr class=\"row-29 odd\">\n\t<td class=\"column-1\">custom\uff08\u987e\u5ba2\u8868\uff09<\/td><td class=\"column-2\"><\/td><td class=\"column-3\"><\/td><td class=\"column-4\"><\/td>\n<\/tr>\n<tr class=\"row-30 even\">\n\t<td class=\"column-1\">customerID<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">1000000<\/td><td class=\"column-4\">\u5ba2\u6237id<\/td>\n<\/tr>\n<tr class=\"row-31 odd\">\n\t<td class=\"column-1\">companyName<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">10000<\/td><td class=\"column-4\">\u516c\u53f8\u540d\u79f0<\/td>\n<\/tr>\n<tr class=\"row-32 even\">\n\t<td class=\"column-1\">contactName<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">1000000<\/td><td class=\"column-4\">\u8054\u7cfb\u4eba\u59d3\u540d<\/td>\n<\/tr>\n<tr class=\"row-33 odd\">\n\t<td class=\"column-1\">contactTile<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">10000<\/td><td class=\"column-4\">\u8054\u7edc\u4eba\u804c\u4f4d<\/td>\n<\/tr>\n<tr class=\"row-34 even\">\n\t<td class=\"column-1\">address<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">10000<\/td><td class=\"column-4\">\u5730\u5740<\/td>\n<\/tr>\n<tr class=\"row-35 odd\">\n\t<td class=\"column-1\">city<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">100<\/td><td class=\"column-4\">\u57ce\u5e02<\/td>\n<\/tr>\n<tr class=\"row-36 even\">\n\t<td class=\"column-1\">regdate<\/td><td class=\"column-2\">Date<\/td><td class=\"column-3\">\u5ba2\u6237\u6ce8\u518c\u65f6\u95f4<\/td><td class=\"column-4\">\u5ba2\u6237\u6ce8\u518c\u65f6\u95f4<\/td>\n<\/tr>\n<tr class=\"row-37 odd\">\n\t<td class=\"column-1\">region<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">10<\/td><td class=\"column-4\">\u5ba2\u6237\u533a\u57df<\/td>\n<\/tr>\n<tr class=\"row-38 even\">\n\t<td class=\"column-1\">postalcode<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">100<\/td><td class=\"column-4\">\u90ae\u653f\u7f16\u7801<\/td>\n<\/tr>\n<tr class=\"row-39 odd\">\n\t<td class=\"column-1\">country<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">3<\/td><td class=\"column-4\">\u56fd\u5bb6<\/td>\n<\/tr>\n<tr class=\"row-40 even\">\n\t<td class=\"column-1\">phone<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">1000000<\/td><td class=\"column-4\">\u7535\u8bdd<\/td>\n<\/tr>\n<tr class=\"row-41 odd\">\n\t<td class=\"column-1\">fax<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">1000000<\/td><td class=\"column-4\">\u4f20\u771f<\/td>\n<\/tr>\n<tr class=\"row-42 even\">\n\t<td class=\"column-1\">order\uff08\u8ba2\u5355\uff09<\/td><td class=\"column-2\"><\/td><td class=\"column-3\"><\/td><td class=\"column-4\"><\/td>\n<\/tr>\n<tr class=\"row-43 odd\">\n\t<td class=\"column-1\">orderid<\/td><td class=\"column-2\">Int<\/td><td class=\"column-3\">10000000<\/td><td class=\"column-4\">\u8ba2\u5355id<\/td>\n<\/tr>\n<tr class=\"row-44 even\">\n\t<td class=\"column-1\">customerID<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">1000000<\/td><td class=\"column-4\">\u5ba2\u6237id<\/td>\n<\/tr>\n<tr class=\"row-45 odd\">\n\t<td class=\"column-1\">employID<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">1000<\/td><td class=\"column-4\">\u96c7\u5458id<\/td>\n<\/tr>\n<tr class=\"row-46 even\">\n\t<td class=\"column-1\">orderdate<\/td><td class=\"column-2\">date<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u8ba2\u5355\u751f\u6210\u65f6\u95f4<\/td>\n<\/tr>\n<tr class=\"row-47 odd\">\n\t<td class=\"column-1\">requiredDate<\/td><td class=\"column-2\">date<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u9700\u7528\u65e5\u671f<\/td>\n<\/tr>\n<tr class=\"row-48 even\">\n\t<td class=\"column-1\">shippedDate<\/td><td class=\"column-2\">date<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u53d1\u8d27\u65e5\u671f<\/td>\n<\/tr>\n<tr class=\"row-49 odd\">\n\t<td class=\"column-1\">shipvia<\/td><td class=\"column-2\">int<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u53d1\u8d27\u65b9\u5f0f<\/td>\n<\/tr>\n<tr class=\"row-50 even\">\n\t<td class=\"column-1\">freight<\/td><td class=\"column-2\">double<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u8fd0\u8d39<\/td>\n<\/tr>\n<tr class=\"row-51 odd\">\n\t<td class=\"column-1\">shipname<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">100<\/td><td class=\"column-4\">\u8fd0\u8f93\u65b9\u5f0f<\/td>\n<\/tr>\n<tr class=\"row-52 even\">\n\t<td class=\"column-1\">shipadress<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">100000<\/td><td class=\"column-4\">\u53d1\u8d27\u5730\u5740<\/td>\n<\/tr>\n<tr class=\"row-53 odd\">\n\t<td class=\"column-1\">shipcity<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">100<\/td><td class=\"column-4\">\u53d1\u8d27\u57ce\u5e02<\/td>\n<\/tr>\n<tr class=\"row-54 even\">\n\t<td class=\"column-1\">shipregion<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">10<\/td><td class=\"column-4\">\u53d1\u8d27\u533a\u57df<\/td>\n<\/tr>\n<tr class=\"row-55 odd\">\n\t<td class=\"column-1\">shippostalcode<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">100000<\/td><td class=\"column-4\">\u53d1\u8d27\u90ae\u653f\u7f16\u7801<\/td>\n<\/tr>\n<tr class=\"row-56 even\">\n\t<td class=\"column-1\">shipcountry<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">11<\/td><td class=\"column-4\">\u53d1\u8d27\u56fd\u5bb6<\/td>\n<\/tr>\n<tr class=\"row-57 odd\">\n\t<td class=\"column-1\">orderdatail\uff08\u8ba2\u5355\u8be6\u60c5\uff09<\/td><td class=\"column-2\"><\/td><td class=\"column-3\"><\/td><td class=\"column-4\"><\/td>\n<\/tr>\n<tr class=\"row-58 even\">\n\t<td class=\"column-1\">\u5217\u540d\u79f0<\/td><td class=\"column-2\">\u7c7b\u578b<\/td><td class=\"column-3\">\u6570\u91cf<\/td><td class=\"column-4\">\u63cf\u8ff0<\/td>\n<\/tr>\n<tr class=\"row-59 odd\">\n\t<td class=\"column-1\">orderid<\/td><td class=\"column-2\">int<\/td><td class=\"column-3\">10000000<\/td><td class=\"column-4\">\u8ba2\u5355id<\/td>\n<\/tr>\n<tr class=\"row-60 even\">\n\t<td class=\"column-1\">productId<\/td><td class=\"column-2\">int<\/td><td class=\"column-3\">10000<\/td><td class=\"column-4\">\u4ea7\u54c1id<\/td>\n<\/tr>\n<tr class=\"row-61 odd\">\n\t<td class=\"column-1\">unitPrice<\/td><td class=\"column-2\">double<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u5355\u4ef7<\/td>\n<\/tr>\n<tr class=\"row-62 even\">\n\t<td class=\"column-1\">quantity<\/td><td class=\"column-2\">int<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u6570\u91cf<\/td>\n<\/tr>\n<tr class=\"row-63 odd\">\n\t<td class=\"column-1\">discount<\/td><td class=\"column-2\">double<\/td><td class=\"column-3\"><\/td><td class=\"column-4\">\u6298\u6263<\/td>\n<\/tr>\n<tr class=\"row-64 even\">\n\t<td class=\"column-1\">category\uff08\u4ea7\u54c1\u76ee\u5f55\uff09<\/td><td class=\"column-2\"><\/td><td class=\"column-3\"><\/td><td class=\"column-4\"><\/td>\n<\/tr>\n<tr class=\"row-65 odd\">\n\t<td class=\"column-1\">\u5217\u540d\u79f0<\/td><td class=\"column-2\">\u7c7b\u578b<\/td><td class=\"column-3\">\u6570\u91cf<\/td><td class=\"column-4\">\u63cf\u8ff0<\/td>\n<\/tr>\n<tr class=\"row-66 even\">\n\t<td class=\"column-1\">categoryID<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">100<\/td><td class=\"column-4\">\u4ea7\u54c1\u7c7b\u522bID<\/td>\n<\/tr>\n<tr class=\"row-67 odd\">\n\t<td class=\"column-1\">categoryName<\/td><td class=\"column-2\">string<\/td><td class=\"column-3\">100<\/td><td class=\"column-4\">\u4ea7\u54c1\u7c7b\u522b\u540d\u79f0<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<!-- #tablepress-3 from cache -->\n<h4><span class=\"ez-toc-section\" id=\"%E9%9B%86%E7%BE%A4%E9%85%8D%E7%BD%AE%E6%83%85%E5%86%B5\"><\/span>\u96c6\u7fa4\u914d\u7f6e\u60c5\u51b5<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<p><img src=\"https:\/\/img-blog.csdnimg.cn\/20201109172831294.jpg\" alt=\"\u5404\u7ec4\u4ef6\u5206\u5e03\" title=\"\u5404\u7ec4\u4ef6\u5206\u5e03\" \/><br \/>\n<img src=\"https:\/\/img-blog.csdnimg.cn\/20201109172747369.jpg\" alt=\"\u5185\u5b58\u5360\u7528\u60c5\u51b5\" title=\"\u5185\u5b58\u5360\u7528\u60c5\u51b5\" \/><br \/>\n&#8211; cdh0051<\/p>\n<pre><code class=\"language-bash line-numbers\">[root@cdh0051 ~]# jps\n3152 HeadlampServer\n3153 Main\n1651 Main\n108244 Jps\n3386 EventCatcherService\n3150 Main\n3151 AlertPublisher\n<\/code><\/pre>\n<ul>\n<li>cdh0052<\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">[root@cdh0052 ~]# jps\n3009 ResourceManager\n2641 EmbeddedOozieServer\n2642 Application\n3011 DataNode\n2645 QuorumPeerMain\n15366 Jps\n3035 NodeManager\n2639 NameNode\n<\/code><\/pre>\n<ul>\n<li>cdh0053<\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">[root@cdh0053 ~]# jps\n2721 SecondaryNameNode\n45733 Jps\n2744 DataNode\n2713 NodeManager\n2633 Kafka\n2636 QuorumPeerMain\n<\/code><\/pre>\n<ul>\n<li>cdh0054<\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">[root@cdh0054 ~]# jps\n69521 Jps\n2625 Kafka\n2915 NodeManager\n2627 HistoryServer\n2708 DataNode\n2710 QuorumPeerMain\n2631 RunJar\n2632 RunJar\n<\/code><\/pre>\n<h1><span class=\"ez-toc-section\" id=\"1%E3%80%81%E9%9C%80%E6%B1%82%E5%88%86%E6%9E%90%EF%BC%88%E6%A2%B3%E7%90%86%E8%A1%A8%E7%BB%93%E6%9E%84%EF%BC%8C%E6%A2%B3%E7%90%86%E5%85%B6%E5%BA%A6%E9%87%8F%EF%BC%89\"><\/span>1\u3001\u9700\u6c42\u5206\u6790\uff08\u68b3\u7406\u8868\u7ed3\u6784\uff0c\u68b3\u7406\u5176\u5ea6\u91cf\uff09<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<blockquote><p>\n  \u5ea6\u91cf Metric\n<\/p><\/blockquote>\n<p>\u5ba2\u6237\u6570\uff08Customer \u6570\u91cf\uff09<br \/>\n\u6d3b\u8dc3\u7387\uff08\u6d3b\u8dc3\u5ba2\u6237\u6570\/\u603b\u5ba2\u6237\u6570\uff1b\u6d3b\u8dc3\u5b9a\u4e49\uff1a\u4e00\u5468\u5185\u4e0b\u8fc7\u8ba2\u5355\uff1b\uff09<br \/>\n\u9500\u552e\u989d\uff08orderdetails \u8868\u5185\u7684unit price * quantity &#8211; discount\uff09<br \/>\n\u8ba2\u5355\u91cf\uff08order \u7684\u6570\u91cf\uff09<br \/>\n\u9500\u552e\u91cf\uff08\u4ea7\u54c1\u6240\u6709order \u7684quantity \u7684\u603b\u548c\uff09<br \/>\n\u9500\u552e\u989dTop100\uff08\u7247\u533a\uff09<br \/>\n\u9500\u552e\u989dTopn\uff08\u5458\u5de5\uff09<br \/>\n\u9500\u552e\u989dTopn\uff08\u4ea7\u54c1\uff09<\/p>\n<blockquote><p>\n  \u7ef4\u5ea6 Dimension\n<\/p><\/blockquote>\n<p>\u65f6\u95f4\uff08\u5982\u6309\u65f6\u95f4\u5207\u9500\u552e\u989d\u7b49\u6307\u6807\uff09<br \/>\n\u5458\u5de5\uff08\u5982\u5206\u6790\u4e0d\u540c\u5458\u5de5\u4fc3\u6210\u7684\u9500\u552e\u989d\u7b49\u6307\u6807\uff09<br \/>\n\u9500\u552e\u7247\u533a\uff08\u5206\u6790\u4e0d\u540c\u9500\u552e\u7247\u533a\u4fc3\u6210\u7684\u9500\u552e\u989d\u7b49\u6307\u6807\uff09<br \/>\n\u4ea7\u54c1\uff08\u5206\u6790\u4e0d\u540c\u4ea7\u54c1\u7684\u9500\u552e\u989d\u7b49\u6307\u6807\uff09<br \/>\n\u4ea7\u54c1\u7c7b\u522b\uff08\u5206\u6790\u4e0d\u540c\u7c7b\u522b\u4ea7\u54c1\u7684\u9500\u552e\u989d\u7b49\u6307\u6807\uff09<\/p>\n<ul>\n<li>\u4ee5\u4e0b\u9700\u6c42\u662f\u4e00\u4e2a\u5efa\u8bae\uff0c\u53ef\u4ee5\u4ece\u5f88\u591a\u89d2\u5ea6\u5bf9\u6570\u636e\u8fdb\u884c\u5206\u6790<\/li>\n<\/ul>\n<ol>\n<li>\u6628\u65e5\u9500\u552e\u989d\u591a\u5c11\uff1f<\/li>\n<li>\u6628\u65e5\u65b0\u589e\u5ba2\u6237\u591a\u5c11\uff1f<\/li>\n<li>\u672c\u5468\u603b\u5171\u65b0\u589e\u5ba2\u6237\u591a\u5c11\uff1f<\/li>\n<li>\u76ee\u524d\u5e73\u53f0\u6d3b\u8dc3\u5ba2\u6237\u7387\u5982\u4f55\uff1f<\/li>\n<li>\u5e73\u53f0\u4ef6\u5747\u4ef7\u591a\u5c11\uff1f<\/li>\n<li>\u672c\u5468\u5e73\u53f0\u6700\u7545\u9500\u7684\u4ea7\u54c1\u662f\u4ec0\u4e48\uff1f\u610f\u56fe\uff08\u5ea6\u91cf\uff09\uff1a\u6700\u7545\u9500\u4ea7\u54c1Top-1\u7ef4\u5ea6\uff1a\u65f6\u95f4\u3001\uff08\u9500\u91cf\uff09<\/li>\n<li>\u6700\u7545\u9500\u4ea7\u54c1\u6392\u540d\u524d\u5341\u662f\uff1f<\/li>\n<\/ol>\n<h1><span class=\"ez-toc-section\" id=\"2%E3%80%81%E9%9B%86%E7%BE%A4%E6%9E%B6%E6%9E%84%E8%AE%BE%E8%AE%A1%EF%BC%88%E4%BD%BF%E7%94%A8%E9%82%A3%E4%BA%9B%E7%BB%84%E4%BB%B6%E3%80%81%E6%89%80%E6%9C%89%E7%BB%84%E4%BB%B6%E5%92%8C%E6%9C%BA%E5%99%A8%E7%9A%84%E5%AF%B9%E5%BA%94%E5%85%B3%E7%B3%BB%EF%BC%89\"><\/span>2\u3001\u96c6\u7fa4\u67b6\u6784\u8bbe\u8ba1\uff08\u4f7f\u7528\u90a3\u4e9b\u7ec4\u4ef6\u3001\u6240\u6709\u7ec4\u4ef6\u548c\u673a\u5668\u7684\u5bf9\u5e94\u5173\u7cfb\uff09<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<pre><code class=\"language-bash line-numbers\">Cloudera Manager:http:\/\/192.168.3.51:7180\/\nNamenode http:\/\/192.168.3.52:9870\/\nHue http:\/\/192.168.3.53:8889\/\n<\/code><\/pre>\n<blockquote><p>\n  cdh0051\n<\/p><\/blockquote>\n\n<table id=\"tablepress-4\" class=\"tablepress tablepress-id-4\">\n<tbody class=\"row-hover\">\n<tr class=\"row-1 odd\">\n\t<td class=\"column-1\">\u7ec4\u4ef6\/\u89d2\u8272<\/td><td class=\"column-2\">\u4f5c\u7528<\/td>\n<\/tr>\n<tr class=\"row-2 even\">\n\t<td class=\"column-1\">Alert Publisher<\/td><td class=\"column-2\">\u4e3a\u7279\u5b9a\u7c7b\u578b\u7684\u4e8b\u4ef6\u751f\u6210\u548c\u63d0\u4f9b\u8b66\u62a5<\/td>\n<\/tr>\n<tr class=\"row-3 odd\">\n\t<td class=\"column-1\">Event Server<\/td><td class=\"column-2\">\u805a\u5408 relevant Hadoop \u4e8b\u4ef6\u5e76\u5c06\u5176\u7528\u4e8e\u8b66\u62a5\u548c\u641c\u7d22<\/td>\n<\/tr>\n<tr class=\"row-4 even\">\n\t<td class=\"column-1\">Host Monitor<\/td><td class=\"column-2\"> \u6536\u96c6\u6709\u5173\u4e3b\u673a\u7684\u8fd0\u884c\u72b6\u51b5\u548c\u6307\u6807\u4fe1\u606f<\/td>\n<\/tr>\n<tr class=\"row-5 odd\">\n\t<td class=\"column-1\">Reports Manager <\/td><td class=\"column-2\">\u751f\u6210\u62a5\u544a\uff0c\u5b83\u63d0\u4f9b\u7528\u6237\u3001\u7528\u6237\u7ec4\u548c\u76ee\u5f55\u7684\u78c1\u76d8\u4f7f\u7528\u7387\u7684\u5386\u53f2\u89c6\u56fe\uff0c\u7528\u6237\u548c YARN \u6c60\u7684\u5904\u7406\u6d3b\u52a8\uff0c\u4ee5\u53ca HBase \u8868\u548c\u547d\u540d\u7a7a\u95f4\u3002<\/td>\n<\/tr>\n<tr class=\"row-6 even\">\n\t<td class=\"column-1\">Service Monitor<\/td><td class=\"column-2\">\u6536\u96c6\u6709\u5173\u670d\u52a1\u7684\u8fd0\u884c\u72b6\u51b5\u548c\u6307\u6807\u4fe1\u606f\u4ee5\u53ca YARN \u548c Impala \u670d\u52a1\u4e2d\u7684\u6d3b\u52a8\u4fe1\u606f<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<!-- #tablepress-4 from cache -->\n<blockquote><p>\n  cdh0052\n<\/p><\/blockquote>\n\n<table id=\"tablepress-6\" class=\"tablepress tablepress-id-6\">\n<tbody class=\"row-hover\">\n<tr class=\"row-1 odd\">\n\t<td class=\"column-1\">\u7ec4\u4ef6\/\u89d2\u8272<\/td><td class=\"column-2\">\u4f5c\u7528<\/td>\n<\/tr>\n<tr class=\"row-2 even\">\n\t<td class=\"column-1\">Flume<\/td><td class=\"column-2\">1)\u63d0\u4f9b\u4e00\u4e2a\u5206\u5e03\u5f0f\u7684\uff0c\u53ef\u9760\u7684\uff0c\u5bf9\u5927\u6570\u636e\u91cf\u7684\u65e5\u5fd7\u8fdb\u884c\u9ad8\u6548\u6536\u96c6\u3001\u805a\u96c6\u3001\u79fb\u52a8\u7684\u670d\u52a1\uff0c<\/td>\n<\/tr>\n<tr class=\"row-3 odd\">\n\t<td class=\"column-1\">DataNode<\/td><td class=\"column-2\">\u6587\u4ef6\u7cfb\u7edf\u7684\u5de5\u4f5c\u8282\u70b9\uff0c\u6839\u636e\u9700\u8981\u5b58\u50a8\u548c\u68c0\u7d22\u6570\u636e\u5757\uff0c\u5e76\u4e14\u5b9a\u671f\u5411namenode\u53d1\u9001\u4ed6\u4eec\u6240\u5b58\u50a8\u7684\u5757\u7684\u5217\u8868\u3002<\/td>\n<\/tr>\n<tr class=\"row-4 even\">\n\t<td class=\"column-1\">NameNode<\/td><td class=\"column-2\">Namenode\u5b58\u653e\u6587\u4ef6\u7cfb\u7edf\u6811\u53ca\u6240\u6709\u6587\u4ef6\u3001\u76ee\u5f55\u7684\u5143\u6570\u636e<\/td>\n<\/tr>\n<tr class=\"row-5 odd\">\n\t<td class=\"column-1\">hive\/Gateway<\/td><td class=\"column-2\">\u5b83\u4eec\u5145\u5f53\u4e86\u544a\u8bc9\u5ba2\u6237\u7aef\u914d\u7f6e\u5e94\u8be5\u653e\u7f6e\u5728\u54ea\u91cc\u3002<\/td>\n<\/tr>\n<tr class=\"row-6 even\">\n\t<td class=\"column-1\">Oozie Server<\/td><td class=\"column-2\">Oozie\u662f\u4e00\u4e2a\u5de5\u4f5c\u6d41\u8c03\u5ea6\u5668\u7cfb\u7edf\uff0c\u7528\u4e8e\u7ba1\u7406Apache Hadoop\u4f5c\u4e1a\u3002<\/td>\n<\/tr>\n<tr class=\"row-7 odd\">\n\t<td class=\"column-1\">spark\/Gateway<\/td><td class=\"column-2\">spark gateway\u662f\u7528\u4e8e\u63a5\u6536cloudera\u7ba1\u7406\u7684\u5e94\u7528<\/td>\n<\/tr>\n<tr class=\"row-8 even\">\n\t<td class=\"column-1\">YARN\/JobHistoryServer<\/td><td class=\"column-2\">\u5386\u53f2\u670d\u52a1\u5668\uff0c\u7ba1\u7406\u8005\u53ef\u4ee5\u901a\u8fc7\u5386\u53f2\u670d\u52a1\u5668\u67e5\u770b\u5df2\u7ecf\u8fd0\u884c\u5b8c\u6210\u7684Mapreduce\u4f5c\u4e1a\u8bb0\u5f55\uff0c\u6bd4\u5982\u7528\u4e86\u591a\u5c11\u4e2aMap\u3001\u591a\u5c11\u4e2aReduce\u3001\u4f5c\u4e1a\u63d0\u4ea4\u65f6\u95f4\u3001\u4f5c\u4e1a\u542f\u52a8\u65f6\u95f4\u3001\u4f5c\u4e1a\u5b8c\u6210\u65f6\u95f4\u7b49\u4fe1\u606f\u3002<\/td>\n<\/tr>\n<tr class=\"row-9 odd\">\n\t<td class=\"column-1\">YARN\/NodeManager<\/td><td class=\"column-2\">NodeManager\u7ba1\u7406\u4e00\u4e2aYARN\u96c6\u7fa4\u4e2d\u7684\u6bcf\u4e00\u4e2a\u8282\u70b9\u3002\u6bd4\u5982\u76d1\u89c6\u8d44\u6e90\u4f7f\u7528\u60c5\u51b5\uff08 CPU\uff0c\u5185\u5b58\uff0c\u786c\u76d8\uff0c\u7f51\u7edc\uff09\uff0c\u8ddf\u8e2a\u8282\u70b9\u5065\u5eb7\u7b49\u3002<\/td>\n<\/tr>\n<tr class=\"row-10 even\">\n\t<td class=\"column-1\">YARN\/ResourceManager<\/td><td class=\"column-2\">ResourceManager\u662fYarn\u96c6\u7fa4\u4e3b\u63a7\u8282\u70b9\uff0c\u8d1f\u8d23\u534f\u8c03\u548c\u7ba1\u7406\u6574\u4e2a\u96c6\u7fa4\uff08\u6240\u6709NodeManager\uff09\u7684\u8d44\u6e90\u3002<\/td>\n<\/tr>\n<tr class=\"row-11 odd\">\n\t<td class=\"column-1\">ZooKeeper Server<\/td><td class=\"column-2\">\u914d\u7f6e\u7ba1\u7406\uff0c\u540d\u5b57\u670d\u52a1\uff0c\u63d0\u4f9b\u5206\u5e03\u5f0f\u540c\u6b65\u4ee5\u53ca\u96c6\u7fa4\u7ba1\u7406<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<!-- #tablepress-6 from cache -->\n<blockquote><p>\n  cdh0053\n<\/p><\/blockquote>\n\n<table id=\"tablepress-8\" class=\"tablepress tablepress-id-8\">\n<tbody class=\"row-hover\">\n<tr class=\"row-1 odd\">\n\t<td class=\"column-1\">\u7ec4\u4ef6<\/td><td class=\"column-2\">\u4f5c\u7528<\/td>\n<\/tr>\n<tr class=\"row-2 even\">\n\t<td class=\"column-1\">DataNode<\/td><td class=\"column-2\"><\/td>\n<\/tr>\n<tr class=\"row-3 odd\">\n\t<td class=\"column-1\">hive\/Gateway<\/td><td class=\"column-2\"><\/td>\n<\/tr>\n<tr class=\"row-4 even\">\n\t<td class=\"column-1\">Metastore Server<\/td><td class=\"column-2\"><\/td>\n<\/tr>\n<tr class=\"row-5 odd\">\n\t<td class=\"column-1\">HiveServer2<\/td><td class=\"column-2\"><\/td>\n<\/tr>\n<tr class=\"row-6 even\">\n\t<td class=\"column-1\">Kafka Broker<\/td><td class=\"column-2\"><\/td>\n<\/tr>\n<tr class=\"row-7 odd\">\n\t<td class=\"column-1\">spark\/Gateway<\/td><td class=\"column-2\"><\/td>\n<\/tr>\n<tr class=\"row-8 even\">\n\t<td class=\"column-1\">History Server<\/td><td class=\"column-2\"><\/td>\n<\/tr>\n<tr class=\"row-9 odd\">\n\t<td class=\"column-1\">NodeManager<\/td><td class=\"column-2\"><\/td>\n<\/tr>\n<tr class=\"row-10 even\">\n\t<td class=\"column-1\">ZooKeeper Server<\/td><td class=\"column-2\"><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<!-- #tablepress-8 from cache -->\n<blockquote><p>\n  cdh0054\n<\/p><\/blockquote>\n\n<table id=\"tablepress-7\" class=\"tablepress tablepress-id-7\">\n<tbody class=\"row-hover\">\n<tr class=\"row-1 odd\">\n\t<td class=\"column-1\">\u7ec4\u4ef6<\/td><td class=\"column-2\">\u4f5c\u7528<\/td>\n<\/tr>\n<tr class=\"row-2 even\">\n\t<td class=\"column-1\">DataNode<\/td><td class=\"column-2\"><\/td>\n<\/tr>\n<tr class=\"row-3 odd\">\n\t<td class=\"column-1\">SecondaryNameNode<\/td><td class=\"column-2\">\u9996\u5148\uff0c\u5b83\u5b9a\u65f6\u5230NameNode\u53bb\u83b7\u53d6edit logs\uff0c\u5e76\u66f4\u65b0\u5230fsimage\u4e0a\u3002<br \/>\n\u4e00\u65e6\u5b83\u6709\u4e86\u65b0\u7684fsimage\u6587\u4ef6\uff0c\u5b83\u5c06\u5176\u62f7\u8d1d\u56deNameNode\u4e2d\u3002<br \/>\nNameNode\u5728\u4e0b\u6b21\u91cd\u542f\u65f6\u4f1a\u4f7f\u7528\u8fd9\u4e2a\u65b0\u7684fsimage\u6587\u4ef6\uff0c\u4ece\u800c\u51cf\u5c11\u91cd\u542f\u7684\u65f6\u95f4\u3002<\/td>\n<\/tr>\n<tr class=\"row-4 even\">\n\t<td class=\"column-1\">hive\/Gateway<\/td><td class=\"column-2\"><\/td>\n<\/tr>\n<tr class=\"row-5 odd\">\n\t<td class=\"column-1\">Load Balancer<\/td><td class=\"column-2\">LoadBalancer \u53ef\u4ee5\u5c06\u6765\u81ea\u5ba2\u6237\u7aef\u7684\u8bf7\u6c42\u5206\u53d1\u5230\u4e0d\u540c\u7684\u670d\u52a1\u5668<\/td>\n<\/tr>\n<tr class=\"row-6 even\">\n\t<td class=\"column-1\">History Server<\/td><td class=\"column-2\">Spark\u5e94\u7528\u7a0b\u5e8f\u5728\u8fd0\u884c\u5b8c\u5e94\u7528\u7a0b\u5e8f\u4e4b\u540e\uff0c\u5c06\u5e94\u7528\u7a0b\u5e8f\u7684\u8fd0\u884c\u4fe1\u606f\u5199\u5165\u6307\u5b9a\u76ee\u5f55\uff08\u6307\u5b9a\u65e5\u5fd7\u8bb0\u5f55\u7684\u76ee\u5f55\uff09\uff0c\u800cSpark history server\u53ef\u4ee5\u5c06\u8fd9\u4e9b\u8fd0\u884c\u4fe1\u606f\u88c5\u8f7d\u5e76\u4ee5web\u7684\u65b9\u5f0f\u4f9b\u7528\u6237\u6d4f\u89c8\uff08\u5c55\u793a\uff09\u3002<\/td>\n<\/tr>\n<tr class=\"row-7 odd\">\n\t<td class=\"column-1\">Kafka Broker<\/td><td class=\"column-2\">\u7f13\u5b58\u4ee3\u7406\uff0cKafka\u96c6\u7fa4\u4e2d\u7684\u4e00\u53f0\u6216\u591a\u53f0\u670d\u52a1\u5668\u7edf\u79f0broker.<\/td>\n<\/tr>\n<tr class=\"row-8 even\">\n\t<td class=\"column-1\">spark\/Gateway<\/td><td class=\"column-2\"><\/td>\n<\/tr>\n<tr class=\"row-9 odd\">\n\t<td class=\"column-1\">NodeManager<\/td><td class=\"column-2\"><\/td>\n<\/tr>\n<tr class=\"row-10 even\">\n\t<td class=\"column-1\">ZooKeeper Server<\/td><td class=\"column-2\"><\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<!-- #tablepress-7 from cache -->\n<h1><span class=\"ez-toc-section\" id=\"3%E3%80%81%E6%95%B0%E6%8D%AE%E5%A4%84%E7%90%86%E6%B5%81%E7%A8%8B%E8%AE%BE%E8%AE%A1%EF%BC%88%E6%95%B0%E6%8D%AE%E6%B5%81%E7%A8%8B%E5%9B%BE%EF%BC%89\"><\/span>3\u3001\u6570\u636e\u5904\u7406\u6d41\u7a0b\u8bbe\u8ba1\uff08\u6570\u636e\u6d41\u7a0b\u56fe\uff09<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<blockquote><p>\n  \u5904\u7406\u524d\u5148\u4e86\u89e3\u5404\u4e2a\u7ec4\u4ef6\u7684\u529f\u80fd\n<\/p><\/blockquote>\n<p>\n<table id=\"tablepress-9\" class=\"tablepress tablepress-id-9\">\n<tbody class=\"row-hover\">\n<tr class=\"row-1 odd\">\n\t<td class=\"column-1\">\u7ec4\u4ef6<\/td><td class=\"column-2\">\u4f5c\u7528<\/td><td class=\"column-3\">\u539f\u7406<\/td>\n<\/tr>\n<tr class=\"row-2 even\">\n\t<td class=\"column-1\">Flume<\/td><td class=\"column-2\">\u5b9e\u65f6\u8fdb\u884c\u6570\u636e\u6536\u96c6<\/td><td class=\"column-3\">    \u4e8b\u4ef6\u4f5c\u4e3aFlume\u5185\u90e8\u6570\u636e\u4f20\u8f93\u7684\u6700\u57fa\u672c\u5355\u5143.\u5b83\u662f\u7531\u4e00\u4e2a\u8f6c\u8f7d\u6570\u636e\u7684\u5b57\u8282\u6570\u7ec4(\u4f20\u8f93\u4e2d\u7684\u6570\u636e)\u548c\u4e00\u4e2a\u53ef\u9009\u5934\u90e8\u6784\u6210.\u5bf9\u4e8e\u4e8e\u6bcf\u4e00\u4e2aAgent\u6765\u8bf4,\u5b83\u5c31\u662f\u4e00\u5171\u72ec\u7acb\u7684\u5b88\u62a4\u8fdb\u7a0b\u3002<br \/>\nAgent\u4e3b\u8981\u7531:source,channel,sink\u4e09\u4e2a\u7ec4\u4ef6\u7ec4\u6210.<br \/>\nSource:\u4ece\u6570\u636e\u53d1\u751f\u5668\u63a5\u6536\u6570\u636e,\u5e76\u5c06\u63a5\u6536\u7684\u6570\u636e\u4ee5Flume\u7684event\u683c\u5f0f\u4f20\u9012\u7ed9\u4e00\u4e2a\u6216\u8005\u591a\u4e2a\u901a\u9053channal\u3002<br \/>\nchannal\u662f\u4e00\u79cd\u77ed\u6682\u7684\u5b58\u50a8\u5bb9\u5668,\u5b83\u5c06\u4ecesource\u5904\u63a5\u6536\u5230\u7684event\u683c\u5f0f\u7684\u6570\u636e\u7f13\u5b58\u8d77\u6765,\u76f4\u5230\u5b83\u4eec\u88absinks\u6d88\u8d39\u6389,\u5b83\u5728source\u548csink\u95f4\u8d77\u7740\u4e00\u5171\u6865\u6881\u7684\u4f5c\u7528,channal\u662f\u4e00\u4e2a\u5b8c\u6574\u7684\u4e8b\u52a1,\u8fd9\u4e00\u70b9\u4fdd\u8bc1\u4e86\u6570\u636e\u5728\u6536\u53d1\u7684\u65f6\u5019\u7684\u4e00\u81f4\u6027. <br \/>\n sink\u5c06\u6570\u636e\u5b58\u50a8\u5230\u96c6\u4e2d\u5b58\u50a8\u5668\u6bd4\u5982Hbase\u548cHDFS,\u5b83\u4ecechannals\u6d88\u8d39\u6570\u636e(events)\u5e76\u5c06\u5176\u4f20\u9012\u7ed9\u76ee\u6807\u5730. \u76ee\u6807\u5730\u53ef\u80fd\u662f\u53e6\u4e00\u4e2asink,\u4e5f\u53ef\u80fdHDFS,HBase.<\/td>\n<\/tr>\n<tr class=\"row-3 odd\">\n\t<td class=\"column-1\">Kafka<\/td><td class=\"column-2\">\u662f\u4e00\u4e2a\u53ef\u6301\u4e45\u5316\u7684\u5206\u5e03\u5f0f\u7684\u6d88\u606f\u961f\u5217<\/td><td class=\"column-3\">kafka\u548cflume\u90fd\u662f\u65e5\u5fd7\u7cfb\u7edf\uff0ckafka\u662f\u5206\u5e03\u5f0f\u6d88\u606f\u4e2d\u95f4\u4ef6\uff0c\u81ea\u5e26\u5b58\u50a8\uff0c\u63d0\u4f9bpush\u548cpull\u5b58\u53d6\u6570\u636e\u529f\u80fd\u3002flume\u5206\u4e3aagent\uff08\u6570\u636e\u91c7\u96c6\u5668\uff09,collector\uff08\u6570\u636e\u7b80\u5355\u5904\u7406\u548c\u5199\u5165\uff09,storage\uff08\u5b58\u50a8\u5668\uff09\u4e09\u90e8\u5206\uff0c\u6bcf\u4e00\u90e8\u5206\u90fd\u662f\u53ef\u4ee5\u5b9a\u5236\u7684\u3002<br \/>\nkafka\u505a\u65e5\u5fd7\u7f13\u5b58\u5e94\u8be5\u662f\u66f4\u4e3a\u5408\u9002\u7684\uff0c\u4f46\u662f flume\u7684\u6570\u636e\u91c7\u96c6\u90e8\u5206\u505a\u7684\u5f88\u597d\uff0c\u53ef\u4ee5\u5b9a\u5236\u5f88\u591a\u6570\u636e\u6e90\uff0c\u51cf\u5c11\u5f00\u53d1\u91cf\u3002<br \/>\nCloudera \u5efa\u8bae\u5982\u679c\u6570\u636e\u88ab\u591a\u4e2a\u7cfb\u7edf\u6d88\u8d39\u7684\u8bdd\uff0c\u4f7f\u7528kafka\uff1b\u5982\u679c\u6570\u636e\u88ab\u8bbe\u8ba1\u7ed9Hadoop\u4f7f\u7528\uff0c\u4f7f\u7528Flume\u3002<br \/>\n<br \/>\nbroker \u7ecf\u7eaa\u4eba(\u4ee3\u7406\u4eba)\uff1akafka\u670d\u52a1\u7684java\u8fdb\u7a0b\u5b9e\u4f8b\u3001broker\u4e4b\u95f4\u6ca1\u6709\u4e3b\u4ece\u5173\u7cfb\uff0c\u4ed6\u4eec\u662f\u5e73\u7b49\u7684\u8282\u70b9\u5173\u7cfb\u3001broker\u9700\u8981\u4f9d\u9760zookeeper\u96c6\u7fa4\u8fdb\u884c\u901a\u4fe1<br \/>\n<br \/>\ntopic \u4e3b\u9898\uff1a\u4e8ekafka\u6765\u8bf4 topic\u7528\u6765\u5bf9\u6d88\u606f\u8fdb\u884c\u5f52\u7c7b\u3001\u751f\u4ea7\u8005\u5199\u5165\u6570\u636e\u548c\u6d88\u8d39\u8005\u8bfb\u53d6\u6570\u636e\u65f6\uff0c\u90fd\u9700\u8981\u6307\u5b9atopic<br \/>\n<br \/>\npartition \u5206\u533a\uff1atopic\u662f\u4e00\u4e2a\u903b\u8f91\u6982\u5ff5\uff0c\u4ed6\u5bf9\u4e8ekafka\u96c6\u7fa4\u6765\u8bf4\u662f\u5168\u5c40\u7684\u3001topic\u771f\u6b63\u5b58\u6d88\u606f\u65f6\uff0c\u662f\u5c06\u6d88\u606f\u5199\u5230\u7279\u5b9a\u76ee\u5f55\u7684\u7279\u5b9a\u6587\u4ef6\u4e2d\u3001\u7279\u5b9a\u76ee\u5f55\u5c31\u662f \u5206\u533a\u3001\u7279\u5b9a\u6587\u4ef6\u5c31\u662f \u5206\u533a\u8fd9\u4e2a\u76ee\u5f55\u4e0b\u7684\u6587\u4ef6\u3001\u5206\u533a\u7684\u547d\u540d\u65b9\u5f0f(\u6587\u4ef6\u5939)\u7684\u547d\u540d\u65b9\u5f0f \u4e3b\u9898\u540d-\u5206\u533a\u7f16\u53f7 topic-1<br \/>\n<br \/>\noffset\u504f\u79fb\u91cf\uff1a\u5728kafka\u7684\u6bcf\u4e2a\u5206\u533a\u4e2d\uff0c\u4f7f\u7528offset\u6765\u5bf9\u6d88\u606f\u8fdb\u884c\u7f16\u53f7\uff0c\u6bcf\u4e2a\u5206\u533a\u5185\u90e8\uff0coffset\u662f\u5355\u8c03\u9012\u589e\u4e14\u552f\u4e00\u7684\u3001\u56e0\u4e3aoffset\u7531\u5206\u533a\u6765\u7ef4\u62a4\uff0c\u6240\u4ee5\u4e0d\u540c\u5206\u533a\u53ef\u80fd\u4f1a\u6709\u76f8\u540c\u7684offset<br \/>\n<br \/>\nreplication \u526f\u672c\uff1a\u67d0\u4e2a\u5206\u533a\u7684\u6570\u636e\u5728\u96c6\u7fa4\u4e2d\u5b58\u591a\u5c11\u4efd\u3001\u6bcf\u4e2a\u5206\u533a\u90fd\u53ef\u4ee5\u8bbe\u7f6e\u591a\u4e2a\u526f\u672c\u3001\u6ce8\u610f\uff01 \u526f\u672c\u7684\u6570\u91cf\u4e0d\u80fd\u8d85\u8fc7broker\u6570\u91cf\u3001\u540c\u4e00\u4e2a\u4e3b\u9898\u7684\u540c\u4e00\u4e2a\u5206\u533a\u7684\u4e0d\u540c\u526f\u672c\u4e4b\u95f4\uff0c\u4f1a\u9009\u4e3e\u51fa\u6765\u4e00\u4e2aleader\uff0c\u5176\u4ed6\u7684\u526f\u672c\u4f5c\u4e3afollower\u3001\u5ba2\u6237\u7aef\u8fdb\u884ctopic\u7684\u8bfb\u5199\u65f6\u662f\u4e0eleader\u8fdb\u884c\u901a\u4fe1\u3001follower\u8d1f\u8d23\u540c\u6b65leader\u7684\u6570\u636e<br \/>\n<br \/>\nproducer \u751f\u4ea7\u8005\uff1a\u8d1f\u8d23\u5f80kafka\u7684topic\u4e2d\u5199\u5165\u6570\u636e\u3001\u540c\u4e00\u4e2atopic\u53ef\u4ee5\u6709\u591a\u4e2a\u751f\u4ea7\u8005<br \/>\n\u540c\u4e00\u4e2a\u751f\u4ea7\u8005\u4e5f\u53ef\u4ee5\u540c\u65f6\u5199\u5165\u591a\u4e2atopic<br \/>\n<br \/>\nconsumer \u6d88\u8d39\u8005\uff1a\u8d1f\u8d23\u4ecekafka\u7684topic\u4e2d\u8bfb\u53d6\u6570\u636econsumer\u4f1a\u8bb0\u5f55\u81ea\u5df1\u8bfb\u53d6\u7684offset\uff0c\u4e0b\u6b21\u542f\u52a8\u65f6\u53ef\u4ee5\u4ece\u4e0a\u6b21\u7684\u4f4d\u7f6e\u7ee7\u7eed\u8bfb\u53d6\u6570\u636e\u3001\u4e5f\u53ef\u4ee5\u81ea\u5b9a\u4e49offset\u4ece\u7279\u5b9a\u4f4d\u7f6e\u5f00\u59cb\u6d88\u8d39\u6570\u636e<br \/>\n<br \/>\ncousumer group \u6d88\u8d39\u8005\u7ec4\u3001\u540c\u7ec4\u7684\u6d88\u8d39\u8005\uff0c\u4f1a\u5171\u4eaboffset\u8bb0\u5f55<\/td>\n<\/tr>\n<tr class=\"row-4 even\">\n\t<td class=\"column-1\">hive<\/td><td class=\"column-2\">\u7528\u4e8e\u89e3\u51b3\u6d77\u91cf\u7ed3\u6784\u5316\u65e5\u5fd7\u7684\u6570\u636e\u7edf\u8ba1\u3002<br \/>\n<\/td><td class=\"column-3\">Hive\u662f\u57fa\u4e8eHadoop\u7684\u4e00\u4e2a\u6570\u636e\u4ed3\u5e93\u5de5\u5177\uff0c\u53ef\u4ee5\u5c06\u7ed3\u6784\u5316\u7684\u6570\u636e\u6587\u4ef6\u6620\u5c04\u4e3a\u4e00\u5f20\u8868\uff0c\u5e76\u63d0\u4f9b\u7c7bSQL\u67e5\u8be2\u529f\u80fd\u3002<br \/>\n\u672c\u8d28\u662f\uff1a\u5c06HQL\u8f6c\u5316\u6210MapReduce\u7a0b\u5e8f<br \/>\nHive\u5904\u7406\u7684\u6570\u636e\u5b58\u50a8\u5728HDFS<br \/>\nHive\u5206\u6790\u6570\u636e\u5e95\u5c42\u7684\u5b9e\u73b0\u662fMapReduce<br \/>\n\u6267\u884c\u7a0b\u5e8f\u8fd0\u884c\u5728YARN\u4e0a<br \/>\nHive\u4f18\u52bf\u5728\u4e8e\u5904\u7406\u5927\u6570\u636e\uff0c\u5bf9\u4e8e\u5904\u7406\u5c0f\u6570\u636e\u6ca1\u6709\u4f18\u52bf\uff0c\u56e0\u4e3aHive\u7684\u6267\u884c\u5ef6\u8fdf\u6bd4\u8f83\u9ad8\u3002<br \/>\n\u5185\u90e8\u8868(\u7ba1\u7406\u8868):ive\u4f1a\uff08\u6216\u591a\u6216\u5c11\u5730\uff09\u63a7\u5236\u7740\u6570\u636e\u7684\u751f\u547d\u5468\u671f<br \/>\n\u5916\u90e8\u8868\uff1a\u56e0\u4e3a\u8868\u662f\u5916\u90e8\u8868\uff0c\u6240\u6709Hive\u5e76\u975e\u8ba4\u4e3a\u5176\u5b8c\u5168\u62e5\u6709\u8fd9\u4efd\u6570\u636e\u3002\u5220\u9664\u8be5\u8868\u5e76\u4e0d\u4f1a\u5220\u9664\u6389\u8fd9\u4efd\u6570\u636e\uff0c\u4e0d\u8fc7\u63cf\u8ff0\u8868\u7684\u5143\u6570\u636e\u4fe1\u606f\u4f1a\u88ab\u5220\u9664\u6389<br \/>\n\u5206\u533a\u8868\uff1a\u5206\u533a\u8868\u5b9e\u9645\u4e0a\u5c31\u662f\u5bf9\u5e94\u4e00\u4e2aHDFS\u6587\u4ef6\u7cfb\u7edf\u4e0a\u7684\u72ec\u7acb\u7684\u6587\u4ef6\u5939\uff0c\u8be5\u6587\u4ef6\u5939\u4e0b\u662f\u8be5\u5206\u533a\u6240\u6709\u7684\u6570\u636e\u6587\u4ef6\u3002<br \/>\n<\/td>\n<\/tr>\n<tr class=\"row-5 odd\">\n\t<td class=\"column-1\">spark<\/td><td class=\"column-2\">\u662f\u4e00\u79cd\u5feb\u901f\u3001\u901a\u7528\u3001\u53ef\u6269\u5c55\u7684\u5927\u6570\u636e\u5206\u6790\u5f15\u64ce<\/td><td class=\"column-3\">Spark Core\uff1a\u5b9e\u73b0\u4e86 Spark \u7684\u57fa\u672c\u529f\u80fd\uff0c\u5305\u542b\u4efb\u52a1\u8c03\u5ea6\u3001\u5185\u5b58\u7ba1\u7406\u3001\u9519\u8bef\u6062\u590d\u3001\u4e0e\u5b58\u50a8\u7cfb\u7edf \u4ea4\u4e92\u7b49\u6a21\u5757\u3002Spark Core \u4e2d\u8fd8\u5305\u542b\u4e86\u5bf9\u5f39\u6027\u5206\u5e03\u5f0f\u6570\u636e\u96c6(resilient distributed dataset\uff0c\u7b80\u79f0RDD)\u7684 API \u5b9a\u4e49\u3002 <br \/>\nSpark SQL\uff1a\u662f Spark \u7528\u6765\u64cd\u4f5c\u7ed3\u6784\u5316\u6570\u636e\u7684\u7a0b\u5e8f\u5305\u3002\u901a\u8fc7 Spark SQL\uff0c\u6211\u4eec\u53ef\u4ee5\u4f7f\u7528 SQL \u6216\u8005 Apache Hive \u7248\u672c\u7684 SQL \u65b9\u8a00(HQL)\u6765\u67e5\u8be2\u6570\u636e\u3002Spark SQL \u652f\u6301\u591a\u79cd\u6570\u636e\u6e90\uff0c\u6bd4 \u5982 Hive \u8868\u3001Parquet \u4ee5\u53ca JSON \u7b49\u3002 <br \/>\nSpark Streaming\uff1a\u662f Spark \u63d0\u4f9b\u7684\u5bf9\u5b9e\u65f6\u6570\u636e\u8fdb\u884c\u6d41\u5f0f\u8ba1\u7b97\u7684\u7ec4\u4ef6\u3002\u63d0\u4f9b\u4e86\u7528\u6765\u64cd\u4f5c\u6570\u636e\u6d41\u7684 API\uff0c\u5e76\u4e14\u4e0e Spark Core \u4e2d\u7684 RDD API \u9ad8\u5ea6\u5bf9\u5e94\u3002 <br \/>\nSpark MLlib\uff1a\u63d0\u4f9b\u5e38\u89c1\u7684\u673a\u5668\u5b66\u4e60(ML)\u529f\u80fd\u7684\u7a0b\u5e8f\u5e93\u3002\u5305\u62ec\u5206\u7c7b\u3001\u56de\u5f52\u3001\u805a\u7c7b\u3001\u534f\u540c\u8fc7\u6ee4\u7b49\uff0c\u8fd8\u63d0\u4f9b\u4e86\u6a21\u578b\u8bc4\u4f30\u3001\u6570\u636e \u5bfc\u5165\u7b49\u989d\u5916\u7684\u652f\u6301\u529f\u80fd\u3002 <br \/>\n\u96c6\u7fa4\u7ba1\u7406\u5668\uff1aSpark \u8bbe\u8ba1\u4e3a\u53ef\u4ee5\u9ad8\u6548\u5730\u5728\u4e00\u4e2a\u8ba1\u7b97\u8282\u70b9\u5230\u6570\u5343\u4e2a\u8ba1\u7b97\u8282\u70b9\u4e4b\u95f4\u4f38\u7f29\u8ba1 \u7b97\u3002\u4e3a\u4e86\u5b9e\u73b0\u8fd9\u6837\u7684\u8981\u6c42\uff0c\u540c\u65f6\u83b7\u5f97\u6700\u5927\u7075\u6d3b\u6027\uff0cSpark \u652f\u6301\u5728\u5404\u79cd\u96c6\u7fa4\u7ba1\u7406\u5668(cluster manager)\u4e0a\u8fd0\u884c\uff0c\u5305\u62ec Hadoop YARN\u3001Apache Mesos\uff0c\u4ee5\u53ca Spark \u81ea\u5e26\u7684\u4e00\u4e2a\u7b80\u6613\u8c03\u5ea6 \u5668\uff0c\u53eb\u4f5c\u72ec\u7acb\u8c03\u5ea6\u5668\u3002 <\/td>\n<\/tr>\n<tr class=\"row-6 even\">\n\t<td class=\"column-1\">HDFS<\/td><td class=\"column-2\">\u5206\u5e03\u5f0f\u6587\u4ef6\u7ba1\u7406\u7cfb\u7edf<\/td><td class=\"column-3\"> HDFS\u57fa\u4e8e\u8fd9\u6837\u7684\u4e00\u4e2a\u5047\u8bbe\uff1a\u6700\u6709\u6548\u7684\u6570\u636e\u5904\u7406\u6a21\u5f0f\u662f\u4e00\u6b21\u5199\u5165\u3001\u591a\u6b21\u8bfb\u53d6\u6570\u636e\u96c6\u7ecf\u5e38\u4ece\u6570\u636e\u6e90\u751f\u6210\u6216\u8005\u62f7\u8d1d\u4e00\u6b21\uff0c\u7136\u540e\u5728\u5176\u4e0a\u505a\u5f88\u591a\u5206\u6790\u5de5\u4f5c <\/td>\n<\/tr>\n<tr class=\"row-7 odd\">\n\t<td class=\"column-1\">Yarn<\/td><td class=\"column-2\">\u8d44\u6e90\u534f\u8c03\u8005<\/td><td class=\"column-3\">YARN\u7684\u57fa\u672c\u601d\u60f3\u662f\u5c06JobTracker\u7684\u4e24\u4e2a\u4e3b\u8981\u529f\u80fd(\u8d44\u6e90\u7ba1\u7406\u548c\u4f5c\u4e1a\u8c03\u5ea6\/\u76d1\u63a7)\u5206\u79bb\uff0c\u4e3b\u8981\u65b9\u6cd5\u662f\u521b\u5efa\u4e00\u4e2a\u5168\u5c40\u7684ResourceManager(RM)\u548c\u82e5\u5e72\u4e2a\u9488\u5bf9\u5e94\u7528\u7a0b\u5e8f\u7684ApplicationMaster(AM)\u3002\u8fd9\u91cc\u7684\u5e94\u7528\u7a0b\u5e8f\u662f\u6307\u4f20\u7edf\u7684MapReduce\u4f5c\u4e1a\u6216\u4f5c\u4e1a\u7684DAG(\u6709\u5411\u65e0\u73af\u56fe)\u3002<\/td>\n<\/tr>\n<tr class=\"row-8 even\">\n\t<td class=\"column-1\">Zookeeper<\/td><td class=\"column-2\">\u5206\u5e03\u5f0f\u670d\u52a1\u6846\u67b6<\/td><td class=\"column-3\">\u5b83\u4e3b\u8981\u662f\u7528\u6765\u89e3\u51b3\u5206\u5e03\u5f0f\u5e94\u7528\u4e2d\u7ecf\u5e38\u9047\u5230\u7684\u4e00\u4e9b\u6570\u636e\u7ba1\u7406\u95ee\u9898\uff0c\u5982\uff1a\u7edf\u4e00\u547d\u540d\u670d\u52a1\u3001\u72b6\u6001\u540c\u6b65\u670d\u52a1\u3001\u96c6\u7fa4\u7ba1\u7406\u3001\u5206\u5e03\u5f0f\u5e94\u7528\u914d\u7f6e\u9879\u7684\u7ba1\u7406\u7b49\u3002<br \/>\n\u73b0\u5728\u628a\u8fd9\u4e9b\u914d\u7f6e\u5168\u90e8\u653e\u5230zookeeper\u4e0a\u53bb\uff0c\u4fdd\u5b58\u5728 zookeeper \u7684\u67d0\u4e2a\u76ee\u5f55\u8282\u70b9\u4e2d\uff0c\u7136\u540e\u6240\u6709\u76f8\u5173\u5e94\u7528\u7a0b\u5e8f\u5bf9\u8fd9\u4e2a\u76ee\u5f55\u8282\u70b9\u8fdb\u884c\u76d1\u542c\uff0c\u4e00\u65e6\u914d\u7f6e\u4fe1\u606f\u53d1\u751f\u53d8\u5316\uff0c\u6bcf\u4e2a\u5e94\u7528\u7a0b\u5e8f\u5c31\u4f1a\u6536\u5230 zookeeper \u7684\u901a\u77e5\uff0c\u7136\u540e\u4ece zookeeper \u83b7\u53d6\u65b0\u7684\u914d\u7f6e\u4fe1\u606f\u5e94\u7528\u5230\u7cfb\u7edf\u4e2d\u3002<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<!-- #tablepress-9 from cache --><br \/>\n<img src=\"https:\/\/img-blog.csdnimg.cn\/20201024100030411.gif#pic_center\" alt=\"\u6570\u636e\u6d41\u7a0b\u56fe\" \/><\/p>\n<ol>\n<li>\u901a\u8fc7flume\u628a\u4e00\u4e2akafka\u96c6\u7fa4\u7684\u6570\u636e\u91c7\u96c6\u5230\u81ea\u5df1\u7684kafka\u96c6\u7fa4<\/li>\n<\/ol>\n<p>\u70b9\u51fb\u67e5\u770bflume\u5b98\u65b9\u914d\u7f6e\u6587\u4ef6<br \/>\nhttp:\/\/flume.apache.org\/releases\/content\/1.9.0\/FlumeUserGuide.html#file-channel<\/p>\n<pre><code class=\"language-bash line-numbers\">a1.sources=r1 \u00a0\na1.sinks=k1 \u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\u00a0\na1.channels=c1\n\u00a0\n# configure source \n# source \u4e3akafka\u7c7b\u578b\u00a0\u00a0\na1.sources.r1.type = org.apache.flume.source.kafka.KafkaSource\n\/\/\u6279\u91cf\u5927\u5c0f-\u4e00\u6b21\na1.sources.r1.batchSize = 5000\n\/\/\u6279\u5904\u7406\u65f6\u95f4-\u6bcf\u9694\u591a\u5c11\u6beb\u79d2\u5904\u7406\u7684\u65f6\u95f4\na1.sources.r1.batchDurationMillis = 2000\na1.sources.r1.kafka.bootstrap.servers = cdh0203:9092,cdh0204:9092,cdh0205:9092\n\/topic\na1.sources.r1.kafka.topics = topic01-categories\n#1earliest \u5f53\u5404\u5206\u533a\u4e0b\u6709\u5df2\u63d0\u4ea4\u7684offset\u65f6\uff0c\u4ece\u63d0\u4ea4\u7684offset\u5f00\u59cb\u6d88\u8d39\uff1b\u65e0\u63d0\u4ea4\u7684offset\u65f6\uff0c\u4ece\u5934\u5f00\u59cb\u6d88\u8d39\n#latest \u5f53\u5404\u5206\u533a\u4e0b\u6709\u5df2\u63d0\u4ea4\u7684offset\u65f6\uff0c\u4ece\u63d0\u4ea4\u7684offset\u5f00\u59cb\u6d88\u8d39\uff1b\u65e0\u63d0\u4ea4\u7684offset\u65f6\uff0c\u6d88\u8d39\u65b0\u4ea7\u751f\u7684\u8be5\u5206\u533a\u4e0b\u7684\u6570\u636e\na1.sources.r1.kafka.consumer.auto.offset.reset = earliest\n\/\/\u7ec4id,\u540d\u5b57\u968f\u610f\uff0c\u540c\u7ec4\u53ef\u6d88\u8d39\u4e00\u4e2atopic\na1.sources.r1.kafka.consumer.group.id = ds001\n# configure sink\na1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink\n\/\/\u81ea\u5df1\u96c6\u7fa4\u7684topic\uff0c\u4f46\u5c3d\u91cf\u4e0e\u6570\u636e\u6e90topic\u4e00\u81f4\na1.sinks.k1.kafka.topic = topic01-categories\na1.sinks.k1.kafka.bootstrap.servers = cdh0053:9092,cdh0054:9092\na1.sinks.k1.kafka.flumeBatchSize = 2000\na1.sinks.k1.kafka.producer.acks = 1\na1.sinks.k1.kafka.producer.linger.ms = 1\na1.sinks.k1.kafka.producer.compression.type = snappy\na1.sinks.kafka-sink.kafka.producer.batch.size = 1048576\n# configure channels \u00a0\na1.channels.c1.type = memory\na1.channels.c1.capacity = 10000\na1.channels.c1.byteCapacityBufferPercentage = 20\na1.channels.c1.transactionCapacity = 10000\na1.channels.c1.byteCapacity = 800000\na1.sources.r1.channels = c1\na1.sinks.k1.channel = c1\n<\/code><\/pre>\n<ol start=\"2\">\n<li>\u628a\u672c\u5730kafka\u96c6\u7fa4\u7684\u6570\u636e\u4f20\u5230hdfs\u4e0a<\/p>\n<\/li>\n<li>\n<p>\u8bbe\u8ba1hive\u4ed3\u5e93<\/p>\n<\/li>\n<\/ol>\n<ul>\n<li>\u6570\u636e\u4ed3\u5e93\u4e0e\u6570\u636e\u5e93\u7684\u533a\u522b<\/li>\n<\/ul>\n<p>\u6570\u636e\u5e93\u662f\u9762\u5411\u4e8b\u7269\u7684\uff0c\u5b83\u662f\u9488\u5bf9\u5177\u4f53\u4e1a\u52a1\u5728\u6570\u636e\u5e93\u8054\u673a\u7684\u65e5\u5e38\u64cd\u4f5c\uff0c\u6570\u636e\u91cf\u5c0f<br \/>\n\u6570\u636e\u4ed3\u5e93\u662f\u9762\u5411\u5904\u7406\u7684\uff0c\u5b83\u662f\u9488\u5bf9\u5177\u4f53\u4e1a\u52a1\u5728\u6570\u636e\u5e93\u8054\u673a\u7684\u65e5\u5e38\u64cd\u4f5c\uff0c\u6570\u636e\u91cf\u5927<br \/>\n\u6570\u636e\u4ed3\u5e93\u6807\u51c6\u4e0a\u53ef\u4ee5\u5206\u4e3a\u56db\u5c42\uff1aODS\uff08\u4e34\u65f6\u5b58\u50a8\u5c42\uff09\u3001PDW\uff08\u6570\u636e\u4ed3\u5e93\u5c42\uff09\u3001DM\uff08\u6570\u636e\u96c6\u5e02\u5c42\uff09\u3001APP\uff08\u5e94\u7528\u5c42\uff09\u3002<br \/>\nODS:\u4e00\u4e2a\u7528\u4e8e\u5b58\u50a8\u5f53\u524d\u9700\u8981\u52a0\u8f7d\u7684\u6570\u636e\uff0c\u4e00\u4e2a\u7528\u4e8e\u5b58\u50a8\u5904\u7406\u5b8c\u540e\u7684\u5386\u53f2\u6570\u636e\u3002<br \/>\nPDW:\u8fd9\u4e00\u5c42\u7684\u6570\u636e\u4e00\u822c\u662f\u9075\u5faa\u6570\u636e\u5e93\u7b2c\u4e09\u8303\u5f0f\u7684<br \/>\nDW:\u4ece\u6570\u636e\u7684\u65f6\u95f4\u8de8\u5ea6\u6765\u8bf4\uff0c\u901a\u5e38\u662fPDW\u5c42\u7684\u4e00\u90e8\u5206\uff0c\u4e3b\u8981\u7684\u76ee\u7684\u662f\u4e3a\u4e86\u6ee1\u8db3\u7528\u6237\u5206\u6790\u7684\u9700\u6c42<br \/>\nAPP:\u8fd9\u5c42\u6570\u636e\u662f\u5b8c\u5168\u4e3a\u4e86\u6ee1\u8db3\u5177\u4f53\u7684\u5206\u6790\u9700\u6c42\u800c\u6784\u5efa\u7684\u6570\u636e<\/p>\n<ul>\n<li>\u4e3a\u4ec0\u4e48\u8981\u5bf9\u6570\u636e\u4ed3\u5e93\u5206\u5c42\uff1a<\/li>\n<\/ul>\n<p>\u7528\u7a7a\u95f4\u6362\u65f6\u95f4\uff0c\u901a\u8fc7\u5927\u91cf\u7684\u9884\u5904\u7406\u6765\u63d0\u5347\u5e94\u7528\u7cfb\u7edf\u7684\u7528\u6237\u4f53\u9a8c\uff08\u6548\u7387\uff09\uff0c\u56e0\u6b64\u6570\u636e\u4ed3\u5e93\u4f1a\u5b58\u5728\u5927\u91cf\u5197\u4f59\u7684\u6570\u636e\uff1b<br \/>\n2\u5982\u679c\u4e0d\u5206\u5c42\u7684\u8bdd\uff0c\u5982\u679c\u6e90\u4e1a\u52a1\u7cfb\u7edf\u7684\u4e1a\u52a1\u89c4\u5219\u53d1\u751f\u53d8\u5316\u5c06\u4f1a\u5f71\u54cd\u6574\u4e2a\u6570\u636e\u6e05\u6d17\u8fc7\u7a0b\uff0c\u5de5\u4f5c\u91cf\u5de8\u5927<br \/>\n3\u901a\u8fc7\u6570\u636e\u5206\u5c42\u7ba1\u7406\u53ef\u4ee5\u7b80\u5316\u6570\u636e\u6e05\u6d17\u7684\u8fc7\u7a0b\uff0c\u56e0\u4e3a\u628a\u539f\u6765\u4e00\u6b65\u7684\u5de5\u4f5c\u5206\u5230\u4e86\u591a\u4e2a\u6b65\u9aa4\u53bb\u5b8c\u6210\uff0c\u76f8\u5f53\u4e8e\u628a\u4e00\u4e2a\u590d\u6742\u7684\u5de5\u4f5c\u62c6\u6210\u4e86\u591a\u4e2a\u7b80\u5355\u7684\u5de5\u4f5c\uff0c\u628a\u4e00\u4e2a\u5927\u7684\u9ed1\u76d2\u53d8\u6210\u4e86\u4e00\u4e2a\u767d\u76d2\uff0c\u6bcf\u4e00\u5c42\u7684\u5904\u7406\u903b\u8f91\u90fd\u76f8\u5bf9\u7b80\u5355\u548c\u5bb9\u6613\u7406\u89e3\uff0c\u8fd9\u6837\u6211\u4eec\u6bd4\u8f83\u5bb9\u6613\u4fdd\u8bc1\u6bcf\u4e00\u4e2a\u6b65\u9aa4\u7684\u6b63\u786e\u6027\uff0c\u5f53\u6570\u636e\u53d1\u751f\u9519\u8bef\u7684\u65f6\u5019\uff0c\u5f80\u5f80\u6211\u4eec\u53ea\u9700\u8981\u5c40\u90e8\u8c03\u6574\u67d0\u4e2a\u6b65\u9aa4\u5373\u53ef\u3002<\/p>\n<h4><span class=\"ez-toc-section\" id=\"%E5%85%AD%E5%BC%A0%E8%A1%A8%E4%B9%8B%E9%97%B4%E7%9A%84%E5%85%B3%E7%B3%BB\"><\/span>\u516d\u5f20\u8868\u4e4b\u95f4\u7684\u5173\u7cfb<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<p><img src=\"https:\/\/img-blog.csdnimg.cn\/20201110121847747.jpg\" alt=\"\u8868\u5173\u7cfb\" title=\"\u8868\u5173\u7cfb\" \/><\/p>\n<blockquote><p>\n  \u8ba2\u5355\u8868<br \/>\n  <code>MSCK REPAIR TABLE\u547d\u4ee4\u4e3b\u8981\u662f\u7528\u6765\u89e3\u51b3\u901a\u8fc7hdfs dfs -put\u6216\u8005hdfs api\u5199\u5165hive\u5206\u533a\u8868\u7684\u6570\u636e\u5728hive\u4e2d\u65e0\u6cd5\u88ab\u67e5\u8be2\u5230\u7684\u95ee\u9898\u3002<\/code>\n<\/p><\/blockquote>\n<pre><code class=\"language-sql line-numbers\">CREATE EXTERNAL TABLE IF NOT EXISTS orders_ods_t\n(\norderid INT COMMENT \"\u8ba2\u5355id\",    \ncustomerID INT COMMENT \"\u5ba2\u6237id\",\nemployID INT COMMENT \"\u5458\u5de5id\",\norderdate DATE COMMENT \"\u8ba2\u5355\u751f\u6210\u65f6\u95f4\",\nrequiredDate DATE COMMENT \"\u9700\u7528\u65e5\u671f\",\nshippedDate DATE COMMENT \"\u53d1\u8d27\u65e5\u671f\",\nshipvia INT COMMENT \"\u53d1\u8d27\u65b9\u5f0f\",\nfreight DOUBLE COMMENT \"\u8fd0\u8d39\",\nshipname STRING COMMENT \"\u8fd0\u8f93\u65b9\u5f0f\",\nshipadress STRING COMMENT \"\u53d1\u8d27\u5730\u5740\",\nshipcity STRING COMMENT \"\u53d1\u8d27\u57ce\u5e02\",\nshipregion STRING COMMENT \"\u53d1\u8d27\u533a\u57df\",\nshippostalcode STRING COMMENT \"\u53d1\u8d27\u90ae\u653f\u7f16\u7801\",\nshipcountry STRING COMMENT \"\u53d1\u8d27\u56fd\u5bb6\" \n)\nCOMMENT \"\u8ba2\u5355\u8868\"\npartitioned by (yt string,mt string,dt string)\nrow format delimited fields terminated by '\\t'\nLOCATION '\/user\/hive\/warehouse\/ods_database.db\/orders_ods_t';\nMSCK REPAIR TABLE orders_ods_t;\n<\/code><\/pre>\n<blockquote><p>\n  \u8ba2\u5355\u8be6\u60c5\u8868\n<\/p><\/blockquote>\n<pre><code class=\"language-sql line-numbers\">CREATE EXTERNAL TABLE IF NOT EXISTS orderdetails_ods_t\n(\norderid INT COMMENT \"\u8ba2\u5355id\",\nproductId INT COMMENT \"\u4ea7\u54c1id\",\nunitPrice DOUBLE COMMENT \"\u5355\u4ef7\",\nquantity INT COMMENT \"\u6570\u91cf\",\ndiscount DOUBLE COMMENT \"\u6298\u6263\" \n)\nCOMMENT \"\u8ba2\u5355\u8be6\u60c5\"\npartitioned by (yt string,mt string,dt string)\nrow format delimited fields terminated by '\\t'\nLOCATION '\/user\/hive\/warehouse\/ods_database.db\/orderdetails_ods_t';\n\n\nMSCK REPAIR TABLE orderdetails_ods_t;\n<\/code><\/pre>\n<blockquote><p>\n  \u987e\u5ba2\u8868\n<\/p><\/blockquote>\n<pre><code class=\"language-sql line-numbers\">CREATE EXTERNAL TABLE IF NOT EXISTS customers_ods_t\n(\ncustomerID INT COMMENT \"\u5ba2\u6237id\",    \ncompanyName STRING COMMENT \"\u516c\u53f8\u540d\u79f0\",\ncontactName STRING COMMENT \"\u8054\u7cfb\u4eba\u59d3\u540d\",\ncontactTile STRING COMMENT \"\u8054\u7edc\u4eba\u804c\u4f4d\",\naddress STRING COMMENT \"\u5730\u5740\",\ncity STRING COMMENT \"\u57ce\u5e02\",\nregdate DATE COMMENT \"\u5ba2\u6237\u6ce8\u518c\u65f6\u95f4\",\nregion STRING COMMENT \"\u5ba2\u6237\u533a\u57df\",\npostalcode STRING COMMENT \"\u90ae\u653f\u7f16\u7801\",\ncountry STRING COMMENT \"\u56fd\u5bb6\",\nphone STRING COMMENT \"\u7535\u8bdd\",\nfax STRING COMMENT \"\u4f20\u771f\" \n)\nCOMMENT \"\u987e\u5ba2\u8868\"\npartitioned by (yt string,mt string,dt string)\nrow format delimited fields terminated by '\\t'\nLOCATION '\/user\/hive\/warehouse\/ods_database.db\/customers_ods_t';\n\nMSCK REPAIR TABLE customers_ods_t;\n<\/code><\/pre>\n<blockquote><p>\n  \u5458\u5de5\u8868\n<\/p><\/blockquote>\n<pre><code class=\"language-sql line-numbers\">CREATE EXTERNAL TABLE IF NOT EXISTS employees_ods_t\n(\nemployeeId INT COMMENT \"\u5458\u5de5id\",\nlastname STRING COMMENT \"\u5458\u5de5\u540d\",\nfirstName STRING COMMENT \"\u5458\u5de5\u59d3\",\ntitle STRING COMMENT \"\u5934\u8854\",\ntitleofCountry STRING COMMENT \"\u6240\u5c5e\u56fd\u5bb6\",\nbirthdate DATE COMMENT \"\u51fa\u751f\u65e5\u671f\",\nhiredate DATE COMMENT \"\u5165\u804c\u65f6\u95f4\",\naddress STRING COMMENT \"\u5730\u5740\",\ncity STRING COMMENT \"\u57ce\u5e02\",\nregion STRING COMMENT \"\u7247\u533a\",\npostalCode STRING COMMENT \"\u90ae\u7f16\",\ncountry STRING COMMENT \"\u56fd\u7c4d\",\nhomePhone STRING COMMENT \"\u7535\u8bdd\",\nextension STRING COMMENT \"\u9644\u52a0\",\nphoto STRING COMMENT \"\u7167\u7247\",\nnote STRING COMMENT \"\u6ce8\u610f\",\nreportTo INT COMMENT \"\u4e0a\u7ea7\u9886\u5bfc\",\nsalary DOUBLE COMMENT \"\u5de5\u8d44\"\n)\nCOMMENT \"\u5458\u5de5\u8868\"\npartitioned by (yt string,mt string,dt string)\nrow format delimited fields terminated by '\\t'\nLOCATION '\/user\/hive\/warehouse\/ods_database.db\/employees_ods_t';\n\nMSCK REPAIR TABLE employees_ods_t;\n<\/code><\/pre>\n<blockquote><p>\n  \u4ea7\u54c1\u8868\n<\/p><\/blockquote>\n<pre><code class=\"language-sql line-numbers\">CREATE EXTERNAL TABLE IF NOT EXISTS products_ods_t\n(\nproductid INT COMMENT \"\u4ea7\u54c1id\",\nproduceName STRING COMMENT \"\u4ea7\u54c1\u540d\",\nsupplier INT COMMENT \"\u4f9b\u5e94\u5546\",\ncategoryID INT COMMENT \"\u4ea7\u54c1\u7c7b\u578bID\",\nquantityPerUint STRING COMMENT \"\u6bcf\u79cd\u4ea7\u54c1\u7684\u6570\u91cf\",\nunitPrice DOUBLE COMMENT \"\u5355\u4ef7\" \n)\nCOMMENT \"\u4ea7\u54c1\u8868\"\npartitioned by (yt string,mt string,dt string)\nrow format delimited fields terminated by '\\t'\nLOCATION '\/user\/hive\/warehouse\/ods_database.db\/products_ods_t';\n\nMSCK REPAIR TABLE products_ods_t;\nSELECT * FROM products_ods_t LIMIT 5;\n\n\npartitioned by (dt string,type string)                 \/\/\u5236\u5b9a\u5206\u533a\nrow format delimited fields terminated by '\\t'         \/\/\u6307\u5b9a\u5b57\u6bb5\u5206\u9694\u7b26\u4e3atab\ncollection items terminated by ','                     \/\/\u6307\u5b9a\u6570\u7ec4\u4e2d\u5b57\u6bb5\u5206\u9694\u7b26\u4e3a\u9017\u53f7\nmap keys terminated by ':'                             \/\/\u6307\u5b9a\u5b57\u5178\u4e2dKV\u5206\u9694\u7b26\u4e3a\u5192\u53f7\nlines terminated by '\\n'                               \/\/\u6307\u5b9a\u884c\u5206\u9694\u7b26\u4e3a\u56de\u8f66\u6362\u884c\nstored as textfile                                     \/\/\u6307\u5b9a\u5b58\u50a8\u7c7b\u578b\u4e3a\u6587\u4ef6\n<\/code><\/pre>\n<blockquote><p>\n  \u4ea7\u54c1\u76ee\u5f55\u8868\n<\/p><\/blockquote>\n<pre><code class=\"language-sql line-numbers\">CREATE EXTERNAL TABLE IF NOT EXISTS category_ods_t\n(\ncategoryID INT COMMENT \"\u4ea7\u54c1\u7c7b\u522bID\",\ncategoryName STRING COMMENT \"\u4ea7\u54c1\u7c7b\u522b\u540d\u79f0\"\n)\nCOMMENT \"\u4ea7\u54c1\u76ee\u5f55\u8868\"\npartitioned by (year string,month string,day string)\nrow format delimited fields terminated by '\\t'\nLOCATION '\/user\/hive\/warehouse\/ods_database.db\/category_ods_t';\n<\/code><\/pre>\n<p>\u52a0\u8f7dhdfs\u6570\u636e\u5230hive<\/p>\n<pre><code class=\"language-sql line-numbers\">LOAD DATA INPATH 'hdfs:\/\/wq1:9000\/source-six\/order' overwrite into table orders_ods_t partition (yt=\"2021\",mt=\"03\",dt=\"29\");\n\nLOAD DATA INPATH 'hdfs:\/\/wq1:9000\/source-six\/orderdetail' overwrite into table orderdetails_ods_t partition (yt=\"2021\",mt=\"03\",dt=\"29\");\n\nLOAD DATA INPATH 'hdfs:\/\/wq1:9000\/source-six\/custom' overwrite into table customers_ods_t partition (yt=\"2021\",mt=\"03\",dt=\"29\");\n\nLOAD DATA INPATH 'hdfs:\/\/wq1:9000\/source-six\/employee' overwrite into table employees_ods_t partition (yt=\"2021\",mt=\"03\",dt=\"29\");\n\nLOAD DATA INPATH 'hdfs:\/\/wq1:9000\/source-six\/products' overwrite into table products_ods_t partition (yt=\"2021\",mt=\"03\",dt=\"29\");\n\nLOAD DATA INPATH 'hdfs:\/\/wq1:9000\/source-six\/categories' overwrite into table category_ods_t partition(year=\"2021\",month=\"03\",day=\"29\");\n\n<\/code><\/pre>\n<ol start=\"4\">\n<li>\u5199HiveSQL\u8bed\u53e5\u5904\u7406\u6570\u636e<br \/>\n\u6700\u7545\u9500\u5546\u54c1\u6392\u540d\u524d100\u7684\u662f\uff1f(\u9ed8\u8ba4\u662f\u5347\u5e8f)<\/li>\n<\/ol>\n<pre><code class=\"language-sql line-numbers\">create table p100 \nas\nselect \nb.producename ,\nsum(a.unitprice*quantity*discount) \nas c \nfrom shoppingmall.orderdetails_ods_t\nas a \ninner join\nshoppingmall.products_ods_t \nas b \non a.productid=b.productid \ngroup by b.producename \norder by c limit 100;\n<\/code><\/pre>\n<ol start=\"5\">\n<li>sqoop\u5bfc\u6570\u636e\u5230MySQL<\/li>\n<\/ol>\n<blockquote><p>\n  sqoop\u5b89\u88c5\u4f4d\u7f6e<code>\/opt\/cloudera\/parcels\/CDH-6.3.2-\\1.cdh6.3.2.p0.1605554\/lib\/sqoop\/<\/code>\n<\/p><\/blockquote>\n<p>\u914d\u7f6e\u6587\u4ef6<\/p>\n<pre><code class=\"language-bash line-numbers\">sqoop export \\\n--connect jdbc:mysql:\/\/192.168.3.51:3306\/hive \\\n--username root \\\n--password 123456 \\\n--table north \\\n--export-dir \/user\/hive\/warehouse\/app_database.db\/north \\\n--fields-terminated-by '\\001' \\\n--m 1 \n<\/code><\/pre>\n<blockquote><p>\n  \u5982\u679c\u51fa\u73b0\u8fd9\u79cd\u9519\u8bef\uff0c\u5219\u662f\u6570\u636e\u5e93\u76f8\u5173\u7684\u9519\u8bef\uff0c\u6bd4\u5982\u6570\u636e\u5e93\u5b57\u6bb5\u4e0d\u5bf9\uff0c\u7f16\u7801\u683c\u5f0f\u7b49\uff1b\n<\/p><\/blockquote>\n<pre><code class=\"language-bash line-numbers\">ERROR tool.ExportTool: Error during export: \nExport job failed!\n        at org.apache.sqoop.mapreduce.ExportJobBase.runExport(ExportJobBase.java:445)\n        at org.apache.sqoop.manager.SqlManager.exportTable(SqlManager.java:931)\n        at org.apache.sqoop.tool.ExportTool.exportTable(ExportTool.java:80)\n        at org.apache.sqoop.tool.ExportTool.run(ExportTool.java:99)\n        at org.apache.sqoop.Sqoop.run(Sqoop.java:147)\n        at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)\n        at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:183)\n        at org.apache.sqoop.Sqoop.runTool(Sqoop.java:234)\n        at org.apache.sqoop.Sqoop.runTool(Sqoop.java:243)\n        at org.apache.sqoop.Sqoop.main(Sqoop.java:252)\n<\/code><\/pre>\n<ol start=\"6\">\n<li>QuickBI\u4e0a\u5c55\u793a\u7684\u6570\u636e<br \/>\n\u6bcf\u4e2a\u7247\u533a\u9500\u552e\u989d\u6392\u524d10\u7684\u5546\u54c1<\/li>\n<\/ol>\n<pre><code class=\"language-sql line-numbers\">create table sourth \nas  \nselect\nb.producename ,\nsum(a.unitprice*quantity*discount) \nas c  \nfrom \npdw_database.orderdetails_pdw_t \nas a  \ninner join \npdw_database.products_pdw_t \nas b  \non a.productid=b.productid  \ninner join\npdw_database.orders_pdw_t \nas e \non a.orderid=e.orderid \nwhere e.shipregion='South' \ngroup by b.producename \norder by c\nlimit 10\n<\/code><\/pre>\n<p><img src=\"https:\/\/img-blog.csdnimg.cn\/20201031175926813.jpg\" alt=\"\" \/><\/p>\n","protected":false},"excerpt":{"rendered":"<p>\u524d\u8a00 \u867d\u7136\u8fd9\u662f\u4e00\u4e2a\u7528CDH\u642d\u5efa\u8d77\u6765\u7684\u96c6\u7fa4\uff0c\u672c\u7bc7\u524d\u534a\u90e8\u5206\u5927\u6982\u5c55\u793a\u4e0bCDH\u642d\u5efa\u8fc7\u7a0b\uff0c\u4ee5\u53ca\u642d\u5efa\u597d\u540e\u5b83\u7684\u6837\u5b50\u662f\u4ec0\u4e48\uff0c\u4f46 [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":0,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[7],"tags":[],"_links":{"self":[{"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/posts\/410"}],"collection":[{"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=410"}],"version-history":[{"count":45,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/posts\/410\/revisions"}],"predecessor-version":[{"id":1877,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/posts\/410\/revisions\/1877"}],"wp:attachment":[{"href":"http:\/\/www.specialwu.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=410"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=410"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=410"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}