{"id":487,"date":"2020-10-26T18:38:32","date_gmt":"2020-10-26T10:38:32","guid":{"rendered":"https:\/\/www.specialwu.com\/?p=487"},"modified":"2021-04-13T17:28:44","modified_gmt":"2021-04-13T09:28:44","slug":"%e5%a4%a7%e6%95%b0%e6%8d%ae%e7%bb%84%e4%bb%b6%e7%bb%83%e4%b9%a0%e5%bc%82%e5%b8%b8%e8%ae%b0%e5%bd%95","status":"publish","type":"post","link":"http:\/\/www.specialwu.com\/?p=487","title":{"rendered":"\u5927\u6570\u636e\u7ec4\u4ef6\u7ec3\u4e60\u53ca\u51fa\u73b0\u7684\u5f02\u5e38"},"content":{"rendered":"<div id=\"ez-toc-container\" class=\"ez-toc-v2_0_61 ez-toc-wrap-center counter-hierarchy ez-toc-counter ez-toc-custom ez-toc-container-direction\">\n<div class=\"ez-toc-title-container\">\n<p class=\"ez-toc-title \" >\u76ee\u5f55<\/p>\n<span class=\"ez-toc-title-toggle\"><a href=\"#\" class=\"ez-toc-pull-right ez-toc-btn ez-toc-btn-xs ez-toc-btn-default ez-toc-toggle\" aria-label=\"Toggle Table of Content\"><span class=\"ez-toc-js-icon-con\"><span class=\"\"><span class=\"eztoc-hide\" style=\"display:none;\">Toggle<\/span><span class=\"ez-toc-icon-toggle-span\"><svg style=\"fill: #000000;color:#000000\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" class=\"list-377408\" width=\"20px\" height=\"20px\" viewBox=\"0 0 24 24\" fill=\"none\"><path d=\"M6 6H4v2h2V6zm14 0H8v2h12V6zM4 11h2v2H4v-2zm16 0H8v2h12v-2zM4 16h2v2H4v-2zm16 0H8v2h12v-2z\" fill=\"currentColor\"><\/path><\/svg><svg style=\"fill: #000000;color:#000000\" class=\"arrow-unsorted-368013\" xmlns=\"http:\/\/www.w3.org\/2000\/svg\" width=\"10px\" height=\"10px\" viewBox=\"0 0 24 24\" version=\"1.2\" baseProfile=\"tiny\"><path d=\"M18.2 9.3l-6.2-6.3-6.2 6.3c-.2.2-.3.4-.3.7s.1.5.3.7c.2.2.4.3.7.3h11c.3 0 .5-.1.7-.3.2-.2.3-.5.3-.7s-.1-.5-.3-.7zM5.8 14.7l6.2 6.3 6.2-6.3c.2-.2.3-.5.3-.7s-.1-.5-.3-.7c-.2-.2-.4-.3-.7-.3h-11c-.3 0-.5.1-.7.3-.2.2-.3.5-.3.7s.1.5.3.7z\"\/><\/svg><\/span><\/span><\/span><\/a><\/span><\/div>\n<nav><ul class='ez-toc-list ez-toc-list-level-1 ' ><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-1\" href=\"http:\/\/www.specialwu.com\/?p=487\/#Linux%E5%B8%B8%E7%94%A8%E5%91%BD%E4%BB%A4\" title=\"Linux\u5e38\u7528\u547d\u4ee4\">Linux\u5e38\u7528\u547d\u4ee4<\/a><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-2\" href=\"http:\/\/www.specialwu.com\/?p=487\/#vim%E5%91%BD%E4%BB%A4%E7%9B%B8%E5%85%B3\" title=\"vim\u547d\u4ee4\u76f8\u5173\">vim\u547d\u4ee4\u76f8\u5173<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-3\" href=\"http:\/\/www.specialwu.com\/?p=487\/#%E4%BB%A5%E4%B8%8A%E6%96%87%E4%BB%B6%E4%B8%BA%E4%B8%B4%E6%97%B6%E6%89%80%E8%A7%81%EF%BC%8CLinux%E7%9A%84%E5%91%BD%E4%BB%A4%E5%8F%AF%E5%88%86%E4%B8%BA%E4%BB%A5%E4%B8%8B%E5%8D%81%E7%A7%8D%E7%B1%BB%E5%9E%8B\" title=\"\u4ee5\u4e0a\u6587\u4ef6\u4e3a\u4e34\u65f6\u6240\u89c1\uff0cLinux\u7684\u547d\u4ee4\u53ef\u5206\u4e3a\u4ee5\u4e0b\u5341\u79cd\u7c7b\u578b\">\u4ee5\u4e0a\u6587\u4ef6\u4e3a\u4e34\u65f6\u6240\u89c1\uff0cLinux\u7684\u547d\u4ee4\u53ef\u5206\u4e3a\u4ee5\u4e0b\u5341\u79cd\u7c7b\u578b<\/a><\/li><\/ul><\/li><\/ul><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-4\" href=\"http:\/\/www.specialwu.com\/?p=487\/#%E5%A4%A7%E6%95%B0%E6%8D%AE%E7%BB%84%E4%BB%B6%E7%9A%84%E5%B8%B8%E7%94%A8%E5%91%BD%E4%BB%A4\" title=\"\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u5e38\u7528\u547d\u4ee4\">\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u5e38\u7528\u547d\u4ee4<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-5\" href=\"http:\/\/www.specialwu.com\/?p=487\/#hdfs%E7%BB%83%E4%B9%A0\" title=\"hdfs\u7ec3\u4e60\">hdfs\u7ec3\u4e60<\/a><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><ul class='ez-toc-list-level-3' ><li class='ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-6\" href=\"http:\/\/www.specialwu.com\/?p=487\/#File_profile1txt_could_only_be_replicated_to_0_nodes_instead_of_minReplication_1\" title=\"File \/profile\/1.txt could only be replicated to 0 nodes instead of minReplication (=1).\">File \/profile\/1.txt could only be replicated to 0 nodes instead of minReplication (=1).<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-7\" href=\"http:\/\/www.specialwu.com\/?p=487\/#orgxmlsaxSAXParseException_systemId_file_opthadoop-277etchadoopcore-sitexml\" title=\"org.xml.sax.SAXParseException; systemId: file:\/opt\/hadoop-2.7.7\/etc\/hadoop\/core-site.xml;\">org.xml.sax.SAXParseException; systemId: file:\/opt\/hadoop-2.7.7\/etc\/hadoop\/core-site.xml;<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-8\" href=\"http:\/\/www.specialwu.com\/?p=487\/#%E5%90%AF%E5%8A%A8hdfs%E6%B2%A1%E6%9C%89datanode%E8%8A%82%E7%82%B9\" title=\"\u542f\u52a8hdfs\u6ca1\u6709datanode\u8282\u70b9\">\u542f\u52a8hdfs\u6ca1\u6709datanode\u8282\u70b9<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-3'><a class=\"ez-toc-link ez-toc-heading-9\" href=\"http:\/\/www.specialwu.com\/?p=487\/#Cannot_create_fileupload1txtCOPYING_Name_node_is_in_safe_mode\" title=\"Cannot create file\/upload1.txt.COPYING. Name node is in safe mode.\">Cannot create file\/upload1.txt.COPYING. Name node is in safe mode.<\/a><\/li><\/ul><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-10\" href=\"http:\/\/www.specialwu.com\/?p=487\/#flume%E7%BB%83%E4%B9%A0\" title=\"flume\u7ec3\u4e60\">flume\u7ec3\u4e60<\/a><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-11\" href=\"http:\/\/www.specialwu.com\/?p=487\/#flume-kafka\" title=\"flume-kafka\">flume-kafka<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-12\" href=\"http:\/\/www.specialwu.com\/?p=487\/#kafka-flume-hdfs\" title=\"kafka-flume-hdfs\">kafka-flume-hdfs<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-13\" href=\"http:\/\/www.specialwu.com\/?p=487\/#hive%E4%B8%8A%E7%9A%84%E6%93%8D%E4%BD%9C\" title=\"hive\u4e0a\u7684\u64cd\u4f5c\">hive\u4e0a\u7684\u64cd\u4f5c<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-14\" href=\"http:\/\/www.specialwu.com\/?p=487\/#%E9%80%9A%E8%BF%87sqoop%E6%8A%8Ahdfs%E4%B8%8A%E7%9A%84%E6%95%B0%E6%8D%AE%E5%AF%BC%E5%85%A5%E5%88%B0MySQL%E4%B8%AD\" title=\"\u901a\u8fc7sqoop\u628ahdfs\u4e0a\u7684\u6570\u636e\u5bfc\u5165\u5230MySQL\u4e2d\">\u901a\u8fc7sqoop\u628ahdfs\u4e0a\u7684\u6570\u636e\u5bfc\u5165\u5230MySQL\u4e2d<\/a><\/li><\/ul><\/li><\/ul><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-1'><a class=\"ez-toc-link ez-toc-heading-15\" href=\"http:\/\/www.specialwu.com\/?p=487\/#%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AEHUE\" title=\"\u5b89\u88c5\u914d\u7f6eHUE\">\u5b89\u88c5\u914d\u7f6eHUE<\/a><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><ul class='ez-toc-list-level-4' ><li class='ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-16\" href=\"http:\/\/www.specialwu.com\/?p=487\/#Couldnt_write_lextab_module\" title=\"Couldn&#8217;t write lextab module &lt;module &#8216;slimit.lextab&#8217; from &#8216;\/usr\/local\/lib\/python2.7\/dist-packages\/slimit\/lextab.pyc&#8217;\">Couldn&#8217;t write lextab module &lt;module &#8216;slimit.lextab&#8217; from &#8216;\/usr\/local\/lib\/python2.7\/dist-packages\/slimit\/lextab.pyc&#8217;<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-17\" href=\"http:\/\/www.specialwu.com\/?p=487\/#%E5%9C%A8hive%E5%90%AF%E5%8A%A8%E6%97%B6%E5%87%BA%E7%8E%B0Resources_are_low_on_NN_Please_add_or_free_up_more_resources_then_turn_off_safe_mode_manually\" title=\"\u5728hive\u542f\u52a8\u65f6\u51fa\u73b0Resources are low on NN. Please add or free up more resources then turn off safe mode manually.\">\u5728hive\u542f\u52a8\u65f6\u51fa\u73b0Resources are low on NN. Please add or free up more resources then turn off safe mode manually.<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-18\" href=\"http:\/\/www.specialwu.com\/?p=487\/#you_are_a_Hue_admin_but_not_a_HDFS_superuser_%E2%80%9Chdfs%E2%80%9D_or_part_of_HDFS_supergroup_%E2%80%9Csupergroup\" title=\"you are a Hue admin but not a HDFS superuser, &#8220;hdfs&#8221; or part of HDFS supergroup, &#8220;supergroup\">you are a Hue admin but not a HDFS superuser, &#8220;hdfs&#8221; or part of HDFS supergroup, &#8220;supergroup<\/a><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-19\" href=\"http:\/\/www.specialwu.com\/?p=487\/#hive%E8%BF%9E%E6%8E%A510000%E7%AB%AF%E5%8F%A3%E5%A4%B1%E8%B4%A5\" title=\"hive\u8fde\u63a510000\u7aef\u53e3\u5931\u8d25\">hive\u8fde\u63a510000\u7aef\u53e3\u5931\u8d25<\/a><ul class='ez-toc-list-level-5' ><li class='ez-toc-heading-level-5'><a class=\"ez-toc-link ez-toc-heading-20\" href=\"http:\/\/www.specialwu.com\/?p=487\/#%E5%90%AF%E5%8A%A8hue_usersharehuebuildenvbin_supervisor\" title=\"\u542f\u52a8hue \/user\/share\/hue\/build\/env\/bin  supervisor\">\u542f\u52a8hue \/user\/share\/hue\/build\/env\/bin  supervisor<\/a><\/li><\/ul><\/li><li class='ez-toc-page-1 ez-toc-heading-level-4'><a class=\"ez-toc-link ez-toc-heading-21\" href=\"http:\/\/www.specialwu.com\/?p=487\/#%E5%90%84%E8%8A%82%E7%82%B9jps%EF%BC%88Java_Virtual_Machine_Process_Status_Tool%EF%BC%89\" title=\"\u5404\u8282\u70b9jps\uff08Java Virtual Machine Process Status Tool\uff09\">\u5404\u8282\u70b9jps\uff08Java Virtual Machine Process Status Tool\uff09<\/a><\/li><\/ul><\/li><\/ul><\/li><\/ul><\/li><\/ul><\/nav><\/div>\n<h1><span class=\"ez-toc-section\" id=\"Linux%E5%B8%B8%E7%94%A8%E5%91%BD%E4%BB%A4\"><\/span>Linux\u5e38\u7528\u547d\u4ee4<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<ul>\n<li>\u67e5\u770b\u8fdb\u7a0b <code>top<\/code><\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">top - 18:18:44 up 5 days, 17:15,  1 user,  load average: 0.15, 0.46, 8.05\nTasks: 222 total,   1 running, 221 sleeping,   0 stopped,   0 zombie\n%Cpu(s):  0.2 us,  0.3 sy,  0.0 ni, 99.6 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st\nKiB Mem :  1511808 total,    71260 free,  1238852 used,   201696 buff\/cache\nKiB Swap:  2097148 total,   923132 free,  1174016 used.   103924 avail Mem \n                         \u865a\u62df\u5185\u5b58 \u5e38\u9a7b\u5185\u5b58  \u5171\u4eab\u5185\u5b58    \u7269\u7406\u5185\u5b58\u767e\u5206\u6bd4                 \u547d\u4ee4\u540d\/\u547d\u4ee4\u884c\n   PID USER      PR  NI    VIRT    RES    SHR S  %CPU %MEM     TIME+ COMMAND                                                                      \n    47 root      39  19       0      0      0 S   5.0  0.0   1:39.64 khugepaged                                                                   \n 17942 root      20   0 2940336  48644   1952 S   4.0  3.2 317:20.25 java                                                                         \n  5268 root      20   0 5418828 309756   5416 S   2.3 20.5 393:51.19 java                                                                         \n   741 root      20   0   90568     60     56 S   1.0  0.0  10:17.27 rngd                                                                         \n 18052 root      20   0 2828980  29392   1956 S   1.0  1.9 129:52.15 java   \n<\/code><\/pre>\n<ul>\n<li>\u67e5\u770b\u6bcf\u4e2a\u7269\u7406CPU\u4e2dcore\u7684\u4e2a\u6570(\u5373\u6838\u6570) <code>cat \/proc\/cpuinfo| grep \"cpu cores\"| uniq<\/code><\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">cpu cores       : 4\n<\/code><\/pre>\n<ul>\n<li>\u67e5\u770bCPU\u4fe1\u606f\uff08\u578b\u53f7\uff09<code>cat \/proc\/cpuinfo | grep name | cut -f2 -d: | uniq -c<\/code><\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\"> 4  Intel(R) Core(TM) i5-4210H CPU @ 2.90GHz\n<\/code><\/pre>\n<ul>\n<li>\u67e5\u770b\u7aef\u53e3\u5360\u7528\u60c5\u51b5\uff1a<code>lsof -i:\u7aef\u53e3\u53f7<\/code><\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">COMMAND  PID  USER   FD   TYPE DEVICE SIZE\/OFF NODE NAME\nmysqld  1561 mysql    4u  IPv6 154188      0t0  TCP wq2:mysql-&gt;wq3:60282 (ESTABLISHED)\nmysqld  1561 mysql   21u  IPv6  29430      0t0  TCP *:mysql (LISTEN)\n<\/code><\/pre>\n<ul>\n<li><code>netstat -tunlp | grep \u7aef\u53e3\u53f7<\/code><\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">tcp6       0      0 :::3306                 :::*                    LISTEN      1561\/mysqld\n<\/code><\/pre>\n<ul>\n<li>\u6740\u6b7b\u8fdb\u7a0b <code>kill -9 pid<\/code><\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">kill -9 1561\n<\/code><\/pre>\n<ul>\n<li>\u67e5\u770b\u5185\u5b58\u60c5\u51b5 <code>free -lh<\/code><\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">              total        used        free      shared  buff\/cache   available\nMem:           1.4G        1.1G        107M         18M        281M        210M\nLow:           1.4G        1.3G        107M\nHigh:            0B          0B          0B\nSwap:          2.0G        3.0M        2.0G\n<\/code><\/pre>\n<ul>\n<li>\u67e5\u770b\u786c\u76d8\u60c5\u51b5 <code>df -lh<\/code><\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">\u6587\u4ef6\u7cfb\u7edf                 \u5bb9\u91cf  \u5df2\u7528  \u53ef\u7528 \u5df2\u7528% \u6302\u8f7d\u70b9\ndevtmpfs                 722M     0  722M    0% \/dev\ntmpfs                    739M     0  739M    0% \/dev\/shm\ntmpfs                    739M   19M  720M    3% \/run\ntmpfs                    739M     0  739M    0% \/sys\/fs\/cgroup\n\/dev\/mapper\/centos-root   50G   18G   33G   35% \/\n\/dev\/sda1               1014M  237M  778M   24% \/boot\ntmpfs                    148M   12K  148M    1% \/run\/user\/42\ntmpfs                    148M     0  148M    0% \/run\/user\/0\n<\/code><\/pre>\n<ul>\n<li>\u67e5\u770b\u6587\u4ef6\u5939\u5927\u5c0f <code>du -h \u76ee\u6807\u6587\u4ef6\/\u6587\u4ef6\u5939<\/code><\/li>\n<\/ul>\n<p><code>du -h -d1 shoppingmallonline<\/code><br \/>\n -d, &#8211;max-depth=N<\/p>\n<pre><code class=\"language-bash line-numbers\">[root@wq1 ~]# du -h --max-depth=1 shoppingmallonline\n4.0K    shoppingmallonline\/categories\n146M    shoppingmallonline\/custom\n1.4G    shoppingmallonline\/employee\n1.2G    shoppingmallonline\/order\n398M    shoppingmallonline\/orderdetail\n416K    shoppingmallonline\/products\n3.1G    shoppingmallonline\n<\/code><\/pre>\n<ul>\n<li>\u67e5\u770b\u5386\u53f2\u64cd\u4f5c<\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">[root@wq1 ~]# history \n   22  ll\n   23  cd flume-1.9.0\/\n<\/code><\/pre>\n<ul>\n<li>\u96c6\u7fa4\u95f4\u6570\u636e\u62f7\u8d1d<\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">scp -r \/opt\/flume-1.9.0 root@wq2:\/opt\/\n<\/code><\/pre>\n<ul>\n<li>\u6a21\u7cca\u67e5\u627e\u67d0\u4e2a\u6587\u4ef6 <code>find \/ -name \"*hadoop*\"<\/code><br \/>\nfind \u67e5\u627e\u7684\u90a3\u4e2a\u76ee\u5f55\u4e0b\uff0c -name \u6587\u4ef6\u540d\/\u6587\u4ef6\u5939\u540d<\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">\/root\/hadoop-2.7.7.tar.gz\n\/root\/hadoop_tmp\n<\/code><\/pre>\n<ul>\n<li>\u641c\u7d22\u67d0\u4e2a\u6587\u4ef6\u4e2d\u7684\u5185\u5bb9<\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">#\u5728\u8be5\u6587\u4ef6\u4e2d\u67e5\u627e\u51fa\u7b26\u5408\u7684\u5b57\u7b26\u5e76\u663e\u793a\u884c\u53f7\ngrep -n \u8981\u67e5\u627e\u7684\u5b57\u7b26 \u8981\u67e5\u627e\u7684\u6587\u4ef6 \n#\u5728\u8be5\u6587\u4ef6\u4e2d\u663e\u793a\u51fa\u6392\u9664\u540e\u7684\u5b57\u7b26\ngrep -v \u8981\u67e5\u627e\u7684\u5b57\u7b26 \u8981\u67e5\u627e\u7684\u6587\u4ef6 \n<\/code><\/pre>\n<h4><span class=\"ez-toc-section\" id=\"vim%E5%91%BD%E4%BB%A4%E7%9B%B8%E5%85%B3\"><\/span>vim\u547d\u4ee4\u76f8\u5173<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<ul>\n<li>vim\u4e2d\u5feb\u901f\u67e5\u627e\u67d0\u4e00\u5b57\u7b26\u4e32<br \/>\n<code>Shift + *<\/code><\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\"> \u6700\u5feb\u7684\u65b9\u5f0f\u662f\u8ba9\u5149\u6807\u505c\u7559\u5728\u60f3\u8981\u67e5\u627e\u7684\u5355\u8bcd\u7684\u4efb\u610f\u4e00\u4e2a\u5b57\u6bcd\u4e0a\u9762\uff0c \u7136\u540e\u8f93\u5165Shift + * \n \u5373\u53ef\u5feb\u901f\u9009\u4e2d\u8be5\u5355\u8bcd\uff0c\u5e76\u4e14\u53ef\u4ee5\u901a\u8fc7 n  \u6216  N \u8fdb\u884c\u4e0a\u4e00\u4e2a\u6216\u4e0b\u4e00\u4e2a\u7684\u5339\u914d\u3002\n<\/code><\/pre>\n<ul>\n<li>\u8bbe\u7f6e\u4e34\u65f6\u884c\u53f7<\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">:set nu\n<\/code><\/pre>\n<ul>\n<li>\u8df3\u8f6c\u5230\u67d0\u4e00\u884c<br \/>\nn\u4ee3\u8868\u8981\u8df3\u8f6c\u7684\u884c\u6570<\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">:n\n<\/code><\/pre>\n<hr \/>\n<ul>\n<li>\u67e5\u770b\u67d0\u51e0\u884c\u7684\u6570\u636e<br \/>\n<code>sed -n 'a,bp' \u76ee\u6807\u6587\u4ef6<\/code>\u67e5\u770ba\u5230b\u884c\u7684\u6570\u636e\u5219\u9700\u8981b\u5927\u4e8ea\u5c31\u884c,\u5982\u679c\u53ea\u67e5\u770ba\u884c\u7684\u6570\u636e\uff0c\u90a3\u4e48b\u5c0f\u4e8ea\u5373\u53ef<\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">[root@wq2 ~]# sed -n '10,1p' w.text\n10\n[root@wq2 ~]# sed -n '10,13p' w.text \n10\n11\n12\n13\n<\/code><\/pre>\n<ul>\n<li>\u7edf\u8ba1\u6587\u4ef6\u884c\u6570<\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">[root@wq2 ~]# wc --help\n\u7528\u6cd5\uff1awc [\u9009\u9879]... [\u6587\u4ef6]...\n\u3000\u6216\uff1awc [\u9009\u9879]... --files0-from=F\n  -c, --bytes            print the byte counts\n  -m, --chars            print the character counts\n  -l, --lines            print the newline counts\n      --files0-from=\u6587\u4ef6        \u4ece\u6307\u5b9a\u6587\u4ef6\u8bfb\u53d6\u4ee5NUL \u7ec8\u6b62\u7684\u540d\u79f0\uff0c\u5982\u679c\u8be5\u6587\u4ef6\u88ab\n                                        \u6307\u5b9a\u4e3a\"-\"\u5219\u4ece\u6807\u51c6\u8f93\u5165\u8bfb\u6587\u4ef6\u540d\n  -L, --max-line-length \u663e\u793a\u6700\u957f\u884c\u7684\u957f\u5ea6\n  -w, --words                   \u663e\u793a\u5355\u8bcd\u8ba1\u6570\n      --help            \u663e\u793a\u6b64\u5e2e\u52a9\u4fe1\u606f\u5e76\u9000\u51fa\n      --version         \u663e\u793a\u7248\u672c\u4fe1\u606f\u5e76\u9000\u51fa\n#\u67e5\u770b\u884c\u6570\uff0c\u5355\u8bcd\u6570\n[root@wq2 ~]# wc -lw w.text \n14 14 w.text\n<\/code><\/pre>\n<ul>\n<li>\u67e5\u770b\u6587\u4ef6\u5939\u4e0b\u6587\u4ef6\u4e2a\u6570\u6216\u6587\u4ef6\u5939\u4e2a\u6570\uff08\u6216\u5b50\u6587\u4ef6\u5939\u4e0b\u7684\uff09<\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">#\u7edf\u8ba1\u5f53\u524d\u6587\u4ef6\u5939\u4e0b\u6587\u4ef6\u7684\u4e2a\u6570\nls -l |grep \"^-\"|wc -l\n#\u7edf\u8ba1\u5f53\u524d\u6587\u4ef6\u5939\u4e0b\u6587\u4ef6\u7684\u4e2a\u6570\uff0c\u5305\u62ec\u5b50\u6587\u4ef6\u5939\u91cc\u7684\nls -lR|grep \"^-\"|wc -l\n#\u7edf\u8ba1\u6587\u4ef6\u5939\u4e0b\u76ee\u5f55\u7684\u4e2a\u6570\uff0c\u5305\u62ec\u5b50\u6587\u4ef6\u5939\u91cc\u7684\nls -lR|grep \"^d\"|wc -l\n[root@wq1 opt]# ls -l|grep \"^d\"|wc -l \n9\n<\/code><\/pre>\n<ul>\n<li>\u5411\u6587\u4ef6\u8ffd\u52a0\u5185\u5bb9 <code>echo \u2018specialwu\u2019 &gt;&gt; ..\/test.txt<\/code><\/li>\n<li>\u8ffd\u8e2a\u6587\u4ef6 <code>tail -f test.txt<\/code>\u6253\u5370\u6700\u540e10\u884c\u518d\u8ffd\u8e2a\u3001\u5982\u679c\u4e0d\u65ad\u5411\u8fd9\u4e2a\u6587\u4ef6\u8ffd\u52a0\u6570\u636e\u901a\u8fc7\u8be5\u547d\u4ee4\u5373\u53ef\u663e\u793a\u51fa\u6765<\/li>\n<li>\u628a\u6587\u4ef6\u4e00\u4e2a\u79fb\u52a8\u5230\u53e6\u4e00\u4e2a\u5730\u65b9 <code>mv test.text \/root.newtest.text<\/code><\/li>\n<li>\u89e3\u538b\u6587\u4ef6<code>tar -zxvf test.tar.gz<\/code><\/li>\n<li>\u538b\u7f29 <code>tar zcvf test.txt<\/code><\/li>\n<\/ul>\n<h4><span class=\"ez-toc-section\" id=\"%E4%BB%A5%E4%B8%8A%E6%96%87%E4%BB%B6%E4%B8%BA%E4%B8%B4%E6%97%B6%E6%89%80%E8%A7%81%EF%BC%8CLinux%E7%9A%84%E5%91%BD%E4%BB%A4%E5%8F%AF%E5%88%86%E4%B8%BA%E4%BB%A5%E4%B8%8B%E5%8D%81%E7%A7%8D%E7%B1%BB%E5%9E%8B\"><\/span>\u4ee5\u4e0a\u6587\u4ef6\u4e3a\u4e34\u65f6\u6240\u89c1\uff0cLinux\u7684\u547d\u4ee4\u53ef\u5206\u4e3a\u4ee5\u4e0b\u5341\u79cd\u7c7b\u578b<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<ol>\n<li>\u6587\u4ef6\u7ba1\u7406<\/li>\n<\/ol>\n<p>&#8211; \u67e5\u770b\u6587\u4ef6\u6240\u5904\u4f4d\u7f6e<\/p>\n<pre><code class=\"language-bash line-numbers\">[root@wq1 ~]# pwd\n\/root\n<\/code><\/pre>\n<ol start=\"2\">\n<li>\u6587\u6863\u7f16\u8f91<\/li>\n<\/ol>\n<pre><code class=\"language-bash line-numbers\">[root@wq1 ~]# vim slaves \nwq1\n~\nESC :wq \u9000\u51fa\u7f16\u8f91\n<\/code><\/pre>\n<ol start=\"3\">\n<li>\u6587\u4ef6\u4f20\u8f93<\/li>\n<\/ol>\n<pre><code class=\"language-bash line-numbers\">[root@wq1 ~]# scp w.text \/root\/tmp\/\n[root@wq1 ~]# cd tmp\/\n[root@wq1 tmp]# ll\n\u603b\u7528\u91cf 4\n-rw-r--r-- 1 root root 54 11\u6708 18 17:24 w.text\n[root@wq1 tmp]# \n<\/code><\/pre>\n<ol start=\"4\">\n<li>\u78c1\u76d8\u7ba1\u7406<\/li>\n<li>\u78c1\u76d8\u7ef4\u62a4<\/li>\n<li>\u7f51\u7edc\u901a\u4fe1<\/li>\n<\/ol>\n<pre><code class=\"language-bash line-numbers\">netstat -a  \/\/\u663e\u793a\u8be6\u7ec6\u7684\u7f51\u7edc\u72b6\u51b5\nnetstat -nu \/\/\u663e\u793aUDP\u7aef\u53e3\u53f7\u7684\u4f7f\u7528\u60c5\u51b5\nnetstat -i      \/\/\u663e\u793a\u7f51\u5361\u5217\u8868\nnetstat -g  \/\/\u663e\u793a\u7ec4\u64ad\u7ec4\u7684\u5173\u7cfb\nnetstat -s  \/\/\u663e\u793a\u7f51\u7edc\u7edf\u8ba1\u4fe1\u606f\nnetstat -l      \/\/\u663e\u793a\u76d1\u542c\u7684\u5957\u63a5\u53e3\nnetstat -ntlp   \/\/\u67e5\u770b\u5f53\u524d\u6240\u6709tcp\u7aef\u53e3\nnetstat -ntulp |grep 80   \/\/\u67e5\u770b\u6240\u670980\u7aef\u53e3\u4f7f\u7528\u60c5\u51b5\nnetstat -an | grep 3306   \/\/\u67e5\u770b\u6240\u67093306\u7aef\u53e3\u4f7f\u7528\u60c5\u51b5\nnetstat -ntp | grep :3306 | wc  \/\/\u67e5\u770b\u67d0\u4e00\u7aef\u53e3\u7684\u8fde\u63a5\u6570\u91cf,\u6bd4\u59823306\u7aef\u53e3\nnetstat -anp |grep 3306 \/\/\u67e5\u770b\u67d0\u4e00\u7aef\u53e3\u7684\u8fde\u63a5\u5ba2\u6237\u7aefIP \u6bd4\u59823306\u7aef\u53e3\n<\/code><\/pre>\n<ol start=\"7\">\n<li>\u7cfb\u7edf\u7ba1\u7406<\/li>\n<\/ol>\n<p>&#8211; \u67e5\u770b\u7cfb\u7edf\u4fe1\u606f<\/p>\n<pre><code class=\"language-bash line-numbers\">cat \/proc\/cpuinfo\nrocessor       : 0\nvendor_id       : GenuineIntel\ncpu family      : 6\nmodel           : 60\nmodel name      : Intel(R) Core(TM) i5-4210H CPU @ 2.90GHz\nstepping        : 3\nmicrocode       : 0x1c\ncpu MHz         : 2893.369\ncache size      : 3072 KB\nphysical id     : 0\nsiblings        : 4\ncore id         : 0\ncpu cores       : 4\napicid          : 0\ninitial apicid  : 0\nfpu             : yes\nfpu_exception   : yes\ncpuid level     : 13\nwp              : yes\n<\/code><\/pre>\n<ol start=\"8\">\n<li>\u7cfb\u7edf\u8bbe\u7f6e<\/li>\n<li>\u5907\u4efd\u538b\u7f29<\/li>\n<li>\u8bbe\u5907\u7ba1\u7406<\/li>\n<\/ol>\n<p>&#8211; \u771f\u5b9e\u5b58\u5728\u7684\u8bbe\u5907<\/p>\n<pre><code class=\"language-bash line-numbers\">[root@wq1 ~]# fdisk -l \n\n\u78c1\u76d8 \/dev\/sda\uff1a107.4 GB, 107374182400 \u5b57\u8282\uff0c209715200 \u4e2a\u6247\u533a\nUnits = \u6247\u533a of 1 * 512 = 512 bytes\n\u6247\u533a\u5927\u5c0f(\u903b\u8f91\/\u7269\u7406)\uff1a512 \u5b57\u8282 \/ 512 \u5b57\u8282\nI\/O \u5927\u5c0f(\u6700\u5c0f\/\u6700\u4f73)\uff1a512 \u5b57\u8282 \/ 512 \u5b57\u8282\n\u78c1\u76d8\u6807\u7b7e\u7c7b\u578b\uff1ados\n\u78c1\u76d8\u6807\u8bc6\u7b26\uff1a0x000e71a9\n\n   \u8bbe\u5907 Boot      Start         End      Blocks   Id  System\n\/dev\/sda1   *        2048     2099199     1048576   83  Linux\n\/dev\/sda2         2099200   111165439    54533120   8e  Linux LVM\n\u78c1\u76d8 \/dev\/mapper\/centos-root\uff1a53.7 GB, 53687091200 \u5b57\u8282\uff0c104857600 \u4e2a\u6247\u533a\nUnits = \u6247\u533a of 1 * 512 = 512 bytes\n\u6247\u533a\u5927\u5c0f(\u903b\u8f91\/\u7269\u7406)\uff1a512 \u5b57\u8282 \/ 512 \u5b57\u8282\nI\/O \u5927\u5c0f(\u6700\u5c0f\/\u6700\u4f73)\uff1a512 \u5b57\u8282 \/ 512 \u5b57\u8282\n\u78c1\u76d8 \/dev\/mapper\/centos-swap\uff1a2147 MB, 2147483648 \u5b57\u8282\uff0c4194304 \u4e2a\u6247\u533a\nUnits = \u6247\u533a of 1 * 512 = 512 bytes\n\u6247\u533a\u5927\u5c0f(\u903b\u8f91\/\u7269\u7406)\uff1a512 \u5b57\u8282 \/ 512 \u5b57\u8282\nI\/O \u5927\u5c0f(\u6700\u5c0f\/\u6700\u4f73)\uff1a512 \u5b57\u8282 \/ 512 \u5b57\u8282\n\n<\/code><\/pre>\n<ul>\n<li>shell\u811a\u672c<br \/>\n<code>\u7528\u6cd5 copyall.sh \/home\/mongodb \/home\/mongodb<\/code><\/li>\n<\/ul>\n<pre><code class=\"language-shell line-numbers\">#\u6839\u636e\u81ea\u5df1\u7684\u673a\u5668\u540d\u4fee\u6539\u540e\u518d\u4f7f\u7528\n#!\/bin\/bash\nHOSTS='wq1 wq2 wq3'\nfor HOST in <span class=\"katex math inline\">{HOSTS}\ndo\n  echo \"++\u6b63\u5728\u590d\u5236<\/span>1\u5230<span class=\"katex math inline\">HOST\u7684<\/span>2\u4e2d...\"\n  scp -rq <span class=\"katex math inline\">{1}<\/span>{HOST}:${2}\n  echo \"--\u53d1\u9001\u5b8c\u6210\uff01\"\ndone\n\n<\/code><\/pre>\n<h1><span class=\"ez-toc-section\" id=\"%E5%A4%A7%E6%95%B0%E6%8D%AE%E7%BB%84%E4%BB%B6%E7%9A%84%E5%B8%B8%E7%94%A8%E5%91%BD%E4%BB%A4\"><\/span>\u5927\u6570\u636e\u7ec4\u4ef6\u7684\u5e38\u7528\u547d\u4ee4<span class=\"ez-toc-section-end\"><\/span><\/h1>\n\n<table id=\"tablepress-10\" class=\"tablepress tablepress-id-10\">\n<tbody class=\"row-hover\">\n<tr class=\"row-1 odd\">\n\t<td class=\"column-1\">\u7ec4\u4ef6\u540d<\/td><td class=\"column-2\">\u64cd\u4f5c<\/td><td class=\"column-3\">\u547d\u4ee4<\/td>\n<\/tr>\n<tr class=\"row-2 even\">\n\t<td class=\"column-1\">hdfs<\/td><td class=\"column-2\">\u542f\u52a8<\/td><td class=\"column-3\">start-dfs.sh<\/td>\n<\/tr>\n<tr class=\"row-3 odd\">\n\t<td class=\"column-1\"><\/td><td class=\"column-2\">\u672c\u5730\u4e0a\u4f20\u6587\u4ef6<\/td><td class=\"column-3\">hdfs dfs -put upload1.txt<\/td>\n<\/tr>\n<tr class=\"row-4 even\">\n\t<td class=\"column-1\"><\/td><td class=\"column-2\">\u683c\u5f0f\u5316<\/td><td class=\"column-3\">hdfs namenode -format<\/td>\n<\/tr>\n<tr class=\"row-5 odd\">\n\t<td class=\"column-1\"><\/td><td class=\"column-2\">\u672c\u5730\u4e0b\u8f7d\u6587\u4ef6<\/td><td class=\"column-3\">hdfs dfs -get \/upload1.txt \/dowtests<\/td>\n<\/tr>\n<tr class=\"row-6 even\">\n\t<td class=\"column-1\">flume<\/td><td class=\"column-2\">\u542f\u52a8\u547d\u4ee4<\/td><td class=\"column-3\">\/bin flume-ng agent \\--conf \/opt\/flume-1.9.0\/conf \\--conf-file \/opt\/flume-1.9.0\/conf\/my_conf\/hdfs-hive.conf \\--name a3 \\-Dflume.root.logger=INFO,console<\/td>\n<\/tr>\n<tr class=\"row-7 odd\">\n\t<td class=\"column-1\">zookeeper<\/td><td class=\"column-2\">\u542f\u52a8\u547d\u4ee4<\/td><td class=\"column-3\">\/bin zkServer.sh start<\/td>\n<\/tr>\n<tr class=\"row-8 even\">\n\t<td class=\"column-1\">kafka<\/td><td class=\"column-2\">\u540e\u53f0\u542f\u52a8<\/td><td class=\"column-3\">kafka-server-start.sh -daemon \/opt\/kafka\/config\/server.properties<\/td>\n<\/tr>\n<tr class=\"row-9 odd\">\n\t<td class=\"column-1\"><\/td><td class=\"column-2\">\u67e5\u770btopic<\/td><td class=\"column-3\">kafka-topics.sh --zookeeper wq1:2181 --list<\/td>\n<\/tr>\n<tr class=\"row-10 even\">\n\t<td class=\"column-1\"><\/td><td class=\"column-2\">\u521b\u5efatopic<\/td><td class=\"column-3\">kafka-topics.sh --zookeeper bd0201:2181 --create --topic topic1 --partitions 3 --replication-factor 2<\/td>\n<\/tr>\n<tr class=\"row-11 odd\">\n\t<td class=\"column-1\"><\/td><td class=\"column-2\">\u6d4b\u8bd5\u6d88\u8d39\u8005<\/td><td class=\"column-3\">kafka-console-consumer.sh --bootstrap-server bd0201:9092 --topic topic1<\/td>\n<\/tr>\n<tr class=\"row-12 even\">\n\t<td class=\"column-1\"><\/td><td class=\"column-2\">\u6d4b\u8bd5\u751f\u4ea7\u8005<\/td><td class=\"column-3\">kafka-console-producer.sh --broker-list bd0201:9092 --topic topic1<\/td>\n<\/tr>\n<tr class=\"row-13 odd\">\n\t<td class=\"column-1\">hive<\/td><td class=\"column-2\">\u542f\u52a8<\/td><td class=\"column-3\">\/bin hive<\/td>\n<\/tr>\n<tr class=\"row-14 even\">\n\t<td class=\"column-1\"><\/td><td class=\"column-2\">\u521b\u5efa\u8868<\/td><td class=\"column-3\">create table testhive1(year String,id int,yearone int,c1 int,c2 int,c3 int,c4 int,c5 int,c6 int,c7 int) row format delimited fields terminated by ',';<\/td>\n<\/tr>\n<tr class=\"row-15 odd\">\n\t<td class=\"column-1\"><\/td><td class=\"column-2\">\u6620\u5c04hdfs\u5230hive<\/td><td class=\"column-3\">load data inpath 'hdfs:\/\/wq1:9000\/hdfs-hive\/flume-hdfs.1603706948079' into table testhive1;<\/td>\n<\/tr>\n<\/tbody>\n<\/table>\n<!-- #tablepress-10 from cache -->\n<h1><span class=\"ez-toc-section\" id=\"hdfs%E7%BB%83%E4%B9%A0\"><\/span>hdfs\u7ec3\u4e60<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<h3><span class=\"ez-toc-section\" id=\"File_profile1txt_could_only_be_replicated_to_0_nodes_instead_of_minReplication_1\"><\/span>File \/profile\/1.txt could only be replicated to 0 nodes instead of minReplication (=1).<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>Hadoop\u96c6\u7fa4\u5b89\u5168\u6a21\u5f0f\u672a\u5173\u95ed<\/p>\n<h3><span class=\"ez-toc-section\" id=\"orgxmlsaxSAXParseException_systemId_file_opthadoop-277etchadoopcore-sitexml\"><\/span>org.xml.sax.SAXParseException; systemId: file:\/opt\/hadoop-2.7.7\/etc\/hadoop\/core-site.xml;<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p><?xml version=\"1.0\" encoding=\"UTF-8\"?>\u6709\u7a7a\u683c\u6216\u8005\u7a7a\u884c<\/p>\n<pre data-language=XML><code class=\"language-markup line-numbers\">&lt;?xml version=\"1.0\" encoding=\"UTF-8\"?&gt;\n&lt;?xml-stylesheet type=\"text\/xsl\" href=\"configuration.xsl\"?&gt;\n&lt;!-- Put site-specific property overrides in this file. --&gt;\n\n&lt;configuration&gt;\n&lt;property&gt;\n    &lt;!-- \u914d\u7f6e\u9ed8\u8ba4\u7684\u6587\u4ef6\u7cfb\u7edf(\u914d\u7f6eNameNode\u670d\u52a1\u542f\u52a8\u7684\u4e3b\u673a\u548c\u7aef\u53e3) --&gt;\n    &lt;name&gt;fs.defaultFS&lt;\/name&gt;\n&lt;value&gt;hdfs:\/\/wq1:9000\/&lt;\/value&gt;\n&lt;\/property&gt;\n&lt;property&gt;\n    &lt;!-- \u914d\u7f6ehdfs\u6587\u4ef6\u4fdd\u5b58\u7684\u76ee\u5f55\uff0c\u5f53\u524d\u9009\u62e9\u653e\u5728Hadoop\u7684\u5b89\u88c5\u76ee\u5f55\u4e0b --&gt;\n    &lt;name&gt;hadoop.tmp.dir&lt;\/name&gt;\n    &lt;value&gt;\/opt\/hadoop-2.7.7\/data&lt;\/value&gt;\n&lt;\/property&gt;\n&lt;\/configuration&gt;\n<\/code><\/pre>\n<h3><span class=\"ez-toc-section\" id=\"%E5%90%AF%E5%8A%A8hdfs%E6%B2%A1%E6%9C%89datanode%E8%8A%82%E7%82%B9\"><\/span>\u542f\u52a8hdfs\u6ca1\u6709datanode\u8282\u70b9<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>cat \/opt\/hadoop-2.7.7\/data\/dfs\/data\/current\/VERSION<br \/>\ncat \/opt\/hadoop-2.7.7\/data\/dfs\/name\/current\/VERSION<br \/>\n\u4f7fdata\/current\/VERSION\u4e2d\u7684clusterID\u4e0ename\/current\/VERSION\u4e2d\u7684clusterID\u76f8\u540c\uff0c\u76f4\u63a5\u5220\u9664dfs\u4e0b\u6240\u6709\u6587\u4ef6\u5728\u521d\u59cb\u5316\u4e5f\u53ef<\/p>\n<h3><span class=\"ez-toc-section\" id=\"Cannot_create_fileupload1txtCOPYING_Name_node_is_in_safe_mode\"><\/span>Cannot create file\/upload1.txt.<em>COPYING<\/em>. Name node is in safe mode.<span class=\"ez-toc-section-end\"><\/span><\/h3>\n<p>hadoop dfsadmin -safemode leave\u547d\u4ee4\u79bb\u5f00\u5b89\u5168\u6a21\u5f0f<\/p>\n<blockquote><p>\n  \u670d\u52a1\u5668\u4e0a\u4e0a\u4f20\u6587\u4ef6hdfs dfs -put upload1.txt \/<br \/>\n  \u901a\u8fc7Java\u4ee3\u7801\u4e0a\u4f20\u6587\u4ef6\n<\/p><\/blockquote>\n<pre><code class=\"language-java line-numbers\">public static void main(String[] args) throws Exception{\n        boolean flag;\n        \/\/ HDFS \u7684\u64cd\u4f5c\n\n        \/\/ HDFS \u7684API\u5165\u53e3\u7c7b\u662fFileSystem\n        \/\/ FileSystem \u6709\u591a\u4e2a\u5b9e\u73b0\u7c7b\n        \/\/ FileSystem \u53ef\u4ee5\u6839\u636eURI\u517c\u5bb9\u5404\u79cd\u5404\u6837\u4e0d\u540c\u7684\u6587\u4ef6\u7cfb\u7edf\n\n        \/\/ 0. \u521b\u5efa\u914d\u7f6e\u6587\u4ef6\u5bf9\u8c61\n\n        \/\/ \u8fd9\u4e2a\u5bf9\u8c61\u53ef\u4ee5\u5728\u7a0b\u5e8f\u8fd0\u884c\u65f6\u4e34\u65f6\u4fee\u6539Hadoop\u7684\u4e00\u4e9b\u914d\u7f6e\u53c2\u6570\n        \/\/ \u4f18\u5148\u7ea7  \u4ee3\u7801\u4e2d  &gt; resource\u7684\u914d\u7f6e\u6587\u4ef6  &gt; \u96c6\u7fa4\n        Configuration conf = new Configuration();\n        conf.set(\"dfs.replication\", \"1\");\n\n        \/\/ 1. \u901a\u8fc7FileSystem\u521b\u5efa  HDFS\u6587\u4ef6\u7cfb\u7edf\u5bf9\u8c61\n\n        FileSystem fs = FileSystem.get(\n                URI.create(\"hdfs:\/\/192.168.236.236:9000\/\"),\n                conf,\n                \"root\"\n        );\n        download(fs);\n        \/\/ 3. \u4e0b\u8f7d\u6587\u4ef6\n        \/\/download(fs);\n\n        \/\/ 4. \u521b\u5efa\u76ee\u5f55\n\/\/         flag = fs.mkdirs(new Path(\"\/2020-8-31\"));\n\/\/        System.out.println(flag);\n\n        \/\/ 5. \u5220\u9664\u6587\u4ef6\n\/\/         flag = fs.delete(new Path(\"\/2020-8-31\"),true);\n\/\/        System.out.println(flag);\n\n        \/\/ 6. \u6587\u4ef6\u540d\u66f4\u6539\n        \/*flag = fs.rename(new Path(\"\/1.txt\"), new Path(\"\/2.txt\"));\n        System.out.println(flag);*\/\n\n        \/\/ 7. \u67e5\u770b\u6587\u4ef6\u8be6\u60c5\n        \/*FileStatus[] fileStatuses = fs.listStatus(new Path(\"\/\"));\n        for (FileStatus fileStatus : fileStatuses) {\n            \/\/TODO  \u6309\u7167Hadoop UI\u7684\u683c\u5f0f\u5c55\u793a\u6587\u4ef6\n\n            \/\/ 8. \u7c7b\u578b\u5224\u65ad\n            boolean isFile = fileStatus.isFile();\n            boolean isDirectory = fileStatus.isDirectory();\n        }*\/\n    }\n private static void upload(FileSystem fs) throws IOException {\n        \/\/ 2. \u4e0a\u4f20\u6587\u4ef6 (\u8986\u76d6)\n        fs.copyFromLocalFile(\n                new Path(\"C:\\\\Users\\\\Administrator\\\\Desktop\\\\up1\"),\n                new Path(\"\/\")\n        );\n    }\n<\/code><\/pre>\n<blockquote><p>\n  \u672c\u5730\u4e0b\u8f7d\u6587\u4ef6 hdfs dfs -get \/upload1.txt \/dowtests\n<\/p><\/blockquote>\n<pre><code class=\"language-java line-numbers\">    private static void download(FileSystem fs) throws IOException {\n        fs.copyToLocalFile(\n                new Path(\"\/up1\/1.txt\"),\n                new Path(\"C:\\\\Users\\\\Administrator\\\\Desktop\\\\up1\")\n        );\n    }\n\n<\/code><\/pre>\n<h1><span class=\"ez-toc-section\" id=\"flume%E7%BB%83%E4%B9%A0\"><\/span>flume\u7ec3\u4e60<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<p><a class=\"wp-editor-md-post-content-link\" href=\"http:\/\/flume.apache.org\/releases\/content\/1.9.0\/FlumeUserGuide.html#kafka-channel\" title=\"\u70b9\u51fb\u67e5\u770bflume\u5b98\u65b9\u914d\u7f6e\u6587\u6863\">\u70b9\u51fb\u67e5\u770bflume\u5b98\u65b9\u914d\u7f6e\u6587\u6863<\/a><br \/>\n0. \u8fd0\u884c\u547d\u4ee4<\/p>\n<pre><code class=\"language-bash line-numbers\"># flume\u5b89\u88c5\u76ee\u5f55\u4e0b\u7684\u547d\u4ee4\nbin\/flume-ng agent \\\n# flume\u5b89\u88c5\u76ee\u5f55\u4e0b\u7684conf\u76ee\u5f55\uff0c\u4e3a\u4e86\u8ba9flume\u627e\u5230\u73af\u5883\u53d8\u91cf\u80e1\u603b\u548clog4j\u4e4b\u7c7b\u7684\u914d\u7f6e\n--conf \/opt\/flume-1.9.0\/conf \\\n# \u6307\u5b9a\u6211\u4eec\u81ea\u5df1\u7f16\u5199\u7684agent\u7684\u914d\u7f6e\u6587\u4ef6\n--conf-file \/opt\/flume-1.9.0\/conf\/my_conf\/netcat_logger.conf \\\n# \u6307\u5b9aagent\u7684\u540d\u5b57\n--name a1 \\\n# \u8bbe\u7f6e\u65e5\u5fd7\u7ea7\u522b\u548c\u8f93\u51fa\u65b9\u5f0f\n-Dflume.root.logger=INFO,console\n<\/code><\/pre>\n<ol>\n<li>\u4ece\u5355\u4e2a\u6587\u4ef6\u4e2d\u5b9e\u65f6\u8bfb\u53d6\u6570\u636e\u5b58\u50a8\u5230HDFS\u4e0a<br \/>\n>\u81ea\u5b9a\u4e49\u7684conf\u914d\u7f6e\u6587\u4ef6<\/li>\n<\/ol>\n<pre><code class=\"language-bash line-numbers\"># Name the components on this agent\n# a1\u662f\u6211\u4eec\u81ea\u5b9a\u4e49\u7684agent\u7684\u540d\u5b57\n# a1.sources  a1\u8fd9\u4e2aagent\u5305\u542b\u7684\u591a\u4e2asource\u7684\u540d\u5b57\n# \u5305\u542b\u51e0\u4e2a\u7ec4\u4ef6\u5c31\u5199\u4e00\u4e2a\u540d\u5b57\u7a7a\u683c\u9694\u5f00\na1.sources = r1\na1.sinks = k1\na1.channels = c1\n# Describe\/configure the source\n# \u914d\u7f6esource\na1.sources.r1.type = exec\na1.sources.r1.command = tail -F \/root\/web.log\n# Describe the sink\n# \u914d\u7f6esink\na1.sinks.k1.type = hdfs\na1.sinks.k1.hdfs.path = hdfs:\/\/bd0201:9000\/flume_test\na1.sinks.k1.hdfs.filePrefix = web-log #\u4e0a\u4f20\u6587\u4ef6\u7684\u524d\u7f00\na1.sinks.k1.hdfs.rollInterval = 0 #\u591a\u4e45\u751f\u6210\u4e00\u4e2a\u6587\u4ef6\na1.sinks.k1.hdfs.rollSize = 133169152 #\u6bcf127M\u751f\u6210\u4e00\u4e2a\u6587\u4ef6\na1.sinks.k1.hdfs.rollCount = 0 #\u6587\u4ef6\u7684\u6eda\u52a8\u4e0eevent\u65e0\u5173\na1.sinks.k1.hdfs.writeFormat = text #\u5199sequence\u6587\u4ef6\u7684\u683c\u5f0f\u3002\u5305\u542b\uff1aText, Writable\uff08\u9ed8\u8ba4\uff09\na1.sinks.k1.hdfs.fileType = DataStream #\u5305\u62ec\uff1aSequenceFile, DataStream,CompressedStream \u4f7f\u7528DataStream\u65f6\u5019\uff0c\u6587\u4ef6\u4e0d\u4f1a\u88ab\u538b\u7f29\n# Use a channel which buffers events in memory\n# \u914d\u7f6echannel\na1.channels.c1.type = memory\na1.channels.c1.capacity = 1000\na1.channels.c1.transactionCapacity = 100\n# Bind the source and sink to the channel\n# \u5c06\u7ec4\u4ef6\u8fde\u63a5\u8d77\u6765\na1.sources.r1.channels = c1\na1.sinks.k1.channel = c1\n<\/code><\/pre>\n<h4><span class=\"ez-toc-section\" id=\"flume-kafka\"><\/span>flume-kafka<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<ul>\n<li>\u83b7\u53d6\u5355\u4e2a\u6587\u4ef6\u7684kafa<\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\"># \u914d\u7f6esource\na1.sources.r1.type = exec\na1.sources.r1.command = tail -F \/root\/cdhdata\/categories\/dt=26\/categories-.1603704061065\n<\/code><\/pre>\n<ul>\n<li>\u83b7\u53d6\u591a\u4e2a\u6587\u4ef6\u5230kafka<\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\"># \u914d\u7f6esource\na1.sources.r1.type = TAILDIR\na1.sources.r1.positionFile = \/var\/log\/flume\/taildir_position.json\na1.sources.r1.filegroups = f1\na1.sources.r1.filegroups.f1 = \/root\/cdhdata\/custom\/.*txt\na1.sources.r1.headers.f1.headerKey1 = value1\na1.sources.r1.fileHeader = true\na1.sources.r1.maxBatchCount = 1000\n<\/code><\/pre>\n<p>\u914d\u7f6e\u4e0a\u9762\u7684\u5199\u6cd5\u51fa\u73b0\u4e86\u9519\u8bef<\/p>\n<pre><code class=\"language-java line-numbers\">2021-03-28 19:24:18,770 (lifecycleSupervisor-1-0) [ERROR - org.apache.flume.source.taildir.ReliableTaildirEventReader.loadPositionFile(ReliableTaildirEventReader.java:147)] Failed loading positionFile: \/root\/opt\/flume-1.9.0\/data\/taildir_position.json\njava.io.EOFException: End of input at line 1 column 1\n        at com.google.gson.stream.JsonReader.nextNonWhitespace(JsonReader.java:954)\n        at com.google.gson.stream.JsonReader.nextValue(JsonReader.java:771)\n        at com.google.gson.stream.JsonReader.peek(JsonReader.java:367)\n        at com.google.gson.stream.JsonReader.expect(JsonReader.java:337)\n        at com.google.gson.stream.JsonReader.beginArray(JsonReader.java:306)\n        at org.apache.flume.source.taildir.ReliableTaildirEventReader.loadPositionFile(ReliableTaildirEventReader.java:111)\n        at org.apache.flume.source.taildir.ReliableTaildirEventReader.&lt;init&gt;(ReliableTaildirEventReader.java:96)\n        at org.apache.flume.source.taildir.ReliableTaildirEventReader.&lt;init&gt;(ReliableTaildirEventReader.java:49)\n        at org.apache.flume.source.taildir.ReliableTaildirEventReader<span class=\"katex math inline\">Builder.build(ReliableTaildirEventReader.java:355)\n        at org.apache.flume.source.taildir.TaildirSource.start(TaildirSource.java:105)\n        at org.apache.flume.source.PollableSourceRunner.start(PollableSourceRunner.java:71)\n        at org.apache.flume.lifecycle.LifecycleSupervisor<\/span>MonitorRunnable.run(LifecycleSupervisor.java:249)\n        at java.util.concurrent.Executors<span class=\"katex math inline\">RunnableAdapter.call(Executors.java:511)\n        at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)\n        at java.util.concurrent.ScheduledThreadPoolExecutor<\/span>ScheduledFutureTask.access<span class=\"katex math inline\">301(ScheduledThreadPoolExecutor.java:180)\n        at java.util.concurrent.ScheduledThreadPoolExecutor<\/span>ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)\n        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)\n        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)\n        at java.lang.Thread.run(Thread.java:748)\n<\/code><\/pre>\n<p>\u4f46\u662f\u76f8\u540c\u7684\u914d\u7f6e\u6761\u4ef6\uff0c\u5728\u7b2c\u4e8c\u6b21\u6570\u636e\u4f20\u8f93\u91cf\u5c0f\u65f6\u5c31\u6ca1\u6709\u51fa\u73b0\u4e0a\u9762\u8fd9\u79cd\u9519\u8bef\u3002<\/p>\n<h4><span class=\"ez-toc-section\" id=\"kafka-flume-hdfs\"><\/span>kafka-flume-hdfs<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<blockquote><p>\n  \u542f\u52a8\n<\/p><\/blockquote>\n<pre><code class=\"language-bash line-numbers\">flume-ng agent \\\n--conf \/opt\/flume-1.9.0\/conf \\\n--conf-file \/opt\/flume-1.9.0\/conf\/my_conf\/hdfs-hive.conf \\\n--name a3 \\\n-Dflume.root.logger=INFO,console\n<\/code><\/pre>\n<blockquote><p>\n  \u81ea\u5b9a\u4e49\u7684\u914d\u7f6e\u6587\u4ef6hdfs-hive.conf\n<\/p><\/blockquote>\n<pre><code class=\"language-bash line-numbers\">#new\n# example.conf: A single-node Flume configuration\n# Name the components on this agent\na3.sources = r1\na3.sinks = k1\na3.channels = c1\n# Describe\/configure the source\na3.sources.r1.type = org.apache.flume.source.kafka.KafkaSource\na3.sources.r1.batchSize = 5000 #\u79ef\u65125000\u4e2aevent\uff0cflush\u5230hdfs\u4e00\u6b21\na3.sources.r1.batchDurationMillis = 2000\na3.sources.r1.kafka.bootstrap.servers = wq1:9092,wq2:9092,wq3:9092\na3.sources.r1.kafka.consumer.auto.offset.reset = earliest\na3.sources.r1.kafka.topics = hdfs-hive\n# Describe the sink\na3.sinks.k1.type = hdfs\na3.sinks.k1.hdfs.path = \/flume-kafka-hdfs\/\na3.sinks.k1.hdfs.filePrefix = flume-hdfs\na3.sinks.k1.hdfs.rollInterval = 0\na3.sinks.k1.hdfs.rollSize = 267386880\na3.sinks.k1.hdfs.rollCount = 0\na3.sinks.k1.hdfs.fileType = DataStream\na3.sinks.k1.hdfs.writeFormat = Text\n# Use a channel which buffers events in memory\na3.channels.c1.type = file\na3.channels.c1.capacity = 10000\na3.channels.c1.byteCapacityBufferPercentage = 20\na3.channels.c1.transactionCapacity = 10000\na3.channels.c1.byteCapacity = 20000000\n# Bind the source and sink to the channel\na3.sources.r1.channels = c1\na3.sinks.k1.channel = c1\n<\/code><\/pre>\n<ul>\n<li>\u5f02\u5e38\u8bb0\u5f55 \uff08\u867d\u51fa\u73b0\u4e86\u8be5\u7c7b\u60c5\u51b5\uff0c\u4f46\u6570\u636e\u8fd8\u662f\u4f20\u5230\u4e86\u96c6\u7fa4\u4e0a\uff09<\/li>\n<\/ul>\n<pre><code class=\"language-bash line-numbers\">org.apache.flume.ChannelFullException: The channel has reached it's capacity. This might be the result of a sink on the channel having too low of batch size, a downstream system running slower than normal, or that the channel capacity is just too low. [channel=c1]\n        at org.apache.flume.channel.file.FileChannel<span class=\"katex math inline\">FileBackedTransaction.doPut(FileChannel.java:505)\n        at org.apache.flume.channel.BasicTransactionSemantics.put(BasicTransactionSemantics.java:93)\n        at org.apache.flume.channel.BasicChannelSemantics.put(BasicChannelSemantics.java:80)\n        at org.apache.flume.channel.ChannelProcessor.processEventBatch(ChannelProcessor.java:191)\n        at org.apache.flume.source.kafka.KafkaSource.doProcess(KafkaSource.java:311)\n        at org.apache.flume.source.AbstractPollableSource.process(AbstractPollableSource.java:60)\n        at org.apache.flume.source.PollableSourceRunner<\/span>PollingRunner.run(PollableSourceRunner.java:133)\n        at java.lang.Thread.run(Thread.java:748)\n<\/code><\/pre>\n<h4><span class=\"ez-toc-section\" id=\"hive%E4%B8%8A%E7%9A%84%E6%93%8D%E4%BD%9C\"><\/span>hive\u4e0a\u7684\u64cd\u4f5c<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<blockquote><p>\n  \u521b\u5efa\u4e00\u5f20\u8868\n<\/p><\/blockquote>\n<pre><code class=\"language-bash line-numbers\">create table testhive1(year String,id int,yearone int,c1 int,c2 int,c3 int,c4 int,c5 int,c6 int,c7 int) row format delimited fields terminated by ',';\n<\/code><\/pre>\n<blockquote><p>\n  \u5c06hdfs\u4e2d\u7684\u6587\u4ef6\u6620\u5c04\u5230\u8868\u4e2d\n<\/p><\/blockquote>\n<pre><code class=\"language-bash line-numbers\">load data inpath 'hdfs:\/\/wq1:9000\/hdfs-hive\/flume-hdfs.1603706948079' into table testhive1;\n<\/code><\/pre>\n<h4><span class=\"ez-toc-section\" id=\"%E9%80%9A%E8%BF%87sqoop%E6%8A%8Ahdfs%E4%B8%8A%E7%9A%84%E6%95%B0%E6%8D%AE%E5%AF%BC%E5%85%A5%E5%88%B0MySQL%E4%B8%AD\"><\/span>\u901a\u8fc7sqoop\u628ahdfs\u4e0a\u7684\u6570\u636e\u5bfc\u5165\u5230MySQL\u4e2d<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<blockquote><p>\n  sqoop-env.sh\n<\/p><\/blockquote>\n<pre><code class=\"language-bash line-numbers\">export HADOOP_COMMON_HOME=\/opt\/cloudera\/parcels\/CDH-6.3.2-1.cdh6.3.2.p0.1605554\/lib\/hadoop\nexport HADOOP_MAPRED_HOME=\/opt\/cloudera\/parcels\/CDH-6.3.2-1.cdh6.3.2.p0.1605554\/lib\/hadoop\nexport HIVE_HOME=\/opt\/cloudera\/parcels\/CDH-6.3.2-1.cdh6.3.2.p0.1605554\/lib\/hive\nexport ZOOCFGDIR=\/opt\/cloudera\/parcels\/CDH-6.3.2-1.cdh6.3.2.p0.1605554\/lib\/zookeeper\n<\/code><\/pre>\n<pre><code class=\"language-bash line-numbers\">ERROR tool.ExportTool: Encountered IOException running export job:\nExport job failed!\n        at org.apache.sqoop.mapreduce.ExportJobBase.runExport(ExportJobBase.java:445)\n        at org.apache.sqoop.manager.SqlManager.exportTable(SqlManager.java:931)\n        at org.apache.sqoop.tool.ExportTool.exportTable(ExportTool.java:80)\n        at org.apache.sqoop.tool.ExportTool.run(ExportTool.java:99)\n        at org.apache.sqoop.Sqoop.run(Sqoop.java:147)\n        at org.apache.hadoop.util.ToolRunner.run(ToolRunner.java:76)\n        at org.apache.sqoop.Sqoop.runSqoop(Sqoop.java:183)\n        at org.apache.sqoop.Sqoop.runTool(Sqoop.java:234)\n        at org.apache.sqoop.Sqoop.runTool(Sqoop.java:243)\n        at org.apache.sqoop.Sqoop.main(Sqoop.java:252)\n<\/code><\/pre>\n<blockquote><p>\n  \/opt\/cloudera\/parcels\/CDH-6.3.2-1.cdh6.3.2.p0.1605554\/lib\/sqoop\/<br \/>\n  sqoop \u5b89\u88c5\u76ee\u5f55\u7684bin\u4e0b\u542f\u52a8\n<\/p><\/blockquote>\n<pre><code class=\"language-bash line-numbers\">sqoop export \\\n--connect jdbc:mysql:\/\/192.168.3.51:3306\/hive \\\n--username root \\\n--password 123456 \\\n--table zuihou \\\n--export-dir \/user\/hive\/warehouse\/ods_database.db\/categories_ods_t\/yt=2020\/mt=10\/dt=26\/categories-.1603704061065 \\\n--fields-terminated-by '\\t' \\\n--m 1 \n<\/code><\/pre>\n<h1><span class=\"ez-toc-section\" id=\"%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AEHUE\"><\/span>\u5b89\u88c5\u914d\u7f6eHUE<span class=\"ez-toc-section-end\"><\/span><\/h1>\n<p>\u4e0b\u8f7d\u5730\u5740http:\/\/archive.cloudera.com\/cdh5\/cdh\/5\/<br \/>\n\u5efa\u8bae\u79bb\u7ebf\u4e0b\u8f7d\u597d\u540e\u518d\u4f20\u5230\u96c6\u7fa4\u4e0a169M 3.9.0\u7248\u672c<br \/>\n<a class=\"wp-editor-md-post-content-link\" href=\"http:\/\/cloudera.github.io\/hue\/docs-4.0.0\/manual.html#_install_hue\">\u70b9\u51fb\u67e5\u770b\u5b98\u65b9\u914d\u7f6e\u6587\u6863<\/a><\/p>\n<blockquote><p>\n  \u6240\u9700\u4f9d\u8d56\n<\/p><\/blockquote>\n<pre><code class=\"language-bash line-numbers\">$ yum install -y gcc libxml2-devel libxslt-devel cyrus-sasl-devel mysql-devel python-devel python-setuptools python-simplejson sqlite-devel ant gmp-devel cyrus-sasl-plain cyrus-sasl-devel cyrus-sasl-gssapi libffi-devel openldap-devel\n<\/code><\/pre>\n<blockquote><p>\n  <code>sudo chown -R hue:hue \/usr\/share\/hue<\/code>\u5426\u5219\u9875\u9762\u8bbf\u95ee\u4e0d\u5230<\/p>\n<p>  Hadoop\u4e0a\u7684\u914d\u7f6e\u6587\u4ef6<br \/>\n  <code>\/opt\/hadoop-2.7.7\/etc\/hadoop<\/code> \u8fd9\u4e09\u4e2a\u914d\u7f6e\u6587\u4ef6\u90fd\u52a0\u4e0a\u5bf9\u5e94\u4f9d\u8d56\n<\/p><\/blockquote>\n<p><code>hdfs-site.xml<\/code><\/p>\n<pre data-language=XML><code class=\"language-markup line-numbers\">&lt;property&gt; \n  &lt;name&gt; dfs.webhdfs.enabled &lt;\/ name&gt; \n  &lt;value&gt; true &lt;\/ value&gt; \n&lt;\/ property&gt;\n<\/code><\/pre>\n<p><code>core-site.xml<\/code><\/p>\n<pre data-language=XML><code class=\"language-markup line-numbers\">&lt;property&gt;\n &lt;name&gt;hadoop.proxyuser.hue.hosts&lt;\/name&gt;\n  &lt;value&gt;*&lt;\/value&gt;\n&lt;\/property&gt;\n&lt;property&gt;\n  &lt;name&gt;hadoop.proxyuser.hue.groups&lt;\/name&gt;\n  &lt;value&gt;*&lt;\/value&gt;\n&lt;\/property&gt;\n<\/code><\/pre>\n<p><code>httpfs-site.xml<\/code><\/p>\n<pre data-language=XML><code class=\"language-markup line-numbers\">&lt;property&gt;\n  &lt;name&gt;httpfs.proxyuser.hue.hosts&lt;\/name&gt;\n  &lt;value&gt;*&lt;\/value&gt;\n&lt;\/property&gt;\n&lt;property&gt;\n  &lt;name&gt;httpfs.proxyuser.hue.groups&lt;\/name&gt;\n  &lt;value&gt;*&lt;\/value&gt;\n&lt;\/property&gt;\n<\/code><\/pre>\n<h4><span class=\"ez-toc-section\" id=\"Couldnt_write_lextab_module\"><\/span>Couldn&#8217;t write lextab module &lt;module &#8216;slimit.lextab&#8217; from &#8216;\/usr\/local\/lib\/python2.7\/dist-packages\/slimit\/lextab.pyc&#8217;<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<p>\u89e3\u51b3\u529e\u6cd5\uff1a \u5728<code>\/usr\/share\/hue\/build\/env\/bin<\/code>\u6267\u884cpip uninstall ply<br \/>\n             pip install ply<span class=\"text-highlighted-inline\" style=\"background-color: #fffd38;\">3.4<\/span><\/p>\n<blockquote><p>\n  \u4fee\u6539hue\u914d\u7f6e\u6587\u4ef6\n<\/p><\/blockquote>\n<p>\u76ee\u524d\u8be5\u9875\u9762\u53ef\u4ee5\u663e\u793a\u53ef\u4ee5\u67e5\u770bhdfs\uff0cMySQL\uff0c\u4f46\u662f\u5199hivesql\u8bed\u53e5\u65f6<code>database is locked<\/code><br \/>\n<code>vim \/usr\/share\/hue\/desktop\/conf\/hue.ini<\/code><\/p>\n<pre><code class=\"language-bash line-numbers\">912\u884chdfs\u914d\u7f6e\n [[hdfs_clusters]]\n    # HA support by using HttpFs\n\n    [[[default]]]\n      # Enter the filesystem uri\n      fs_defaultfs=hdfs:\/\/192.168.236.236:8020\n\n      # NameNode logical name.\n      ## logical_name=192.168.236.236\n\n      # Use WebHdfs\/HttpFs as the communication mechanism.\n      # Domain should be the NameNode or HttpFs host.\n      # Default port is 14000 for HttpFs.\n      webhdfs_url=http:\/\/192.168.236.236:50070\/webhdfs\/v1\n\n      # Change this if your HDFS cluster is Kerberos-secured\n      ## security_enabled=false\n\n      # In secure mode (HTTPS), if SSL certificates from YARN Rest APIs\n      # have to be verified against certificate authority\n      ## ssl_cert_ca_verify=True\n\n      # Directory of the Hadoop configuration\n      hadoop_conf_dir=\/opt\/hadoop-2.7.7\/etc\/hadoop\n     #\u8fd9\u4e24\u6761\u4e3a\u81ea\u5b9a\u4e49\u7684\u90e8\u5206\n     hadoop_hdfs_home=\/opt\/hadoop-2.7.7\n     hadoop_bin=\/opt\/hadoop-2.7.7\/bin\n\n\n     1026\u884chive\u914d\u7f6e\n     [beeswax]\n  # Host where HiveServer2 is running.\n  # If Kerberos security is enabled, use fully-qualified domain name (FQDN).\n  hive_server_host=192.168.236.238\n\n  # Port where HiveServer2 Thrift server runs on.\n  hive_server_port=10000\n\n  # Hive configuration directory, where hive-site.xml is located\n  hive_conf_dir=\/etc\/hive-1.2.2\/conf\n\n  # Timeout in seconds for thrift calls to Hive service\n  server_conn_timeout=120\n\n  # Choose whether to use the old GetLog() thrift call from before Hive 0.14 to retrieve the logs.\n  # If false, use the FetchResults() thrift call from Hive 1.0 or more instead.\n  ## use_get_log_api=false\n\n  # Limit the number of partitions that can be listed.\n  ## list_partitions_limit=10000\n\n  # The maximum number of partitions that will be included in the SELECT * LIMIT sample query for partitioned tables.\n  ## query_partitions_limit=10\n\n  # A limit to the number of rows that can be downloaded from a query before it is truncated.\n  # A value of -1 means there will be no limit.\n  ## download_row_limit=100000\n     [metastore]\n  # Flag to turn on the new version of the create table wizard.\n  enable_new_create_table=true\n\n  # Flag to force all metadata calls (e.g. list tables, table or column details...) to happen via HiveServer2 if available instead of Impala.\n  ## force_hs2_metadata=false\n\n\n1604\u884cMySQL\u4fee\u6539\n    [[[mysql]]]\n      # Name to show in the UI.\n      nice_name=\"wq2mysql\"\n\n      # For MySQL and PostgreSQL, name is the name of the database.\n      # For Oracle, Name is instance of the Oracle server. For express edition\n      # this is 'xe' by default.\n      name=hmysql\n\n      # Database backend to use. This can be:\n      # 1. mysql\n      # 2. postgresql\n      # 3. oracle\n      engine=mysql\n\n      # IP or hostname of the database to connect to.\n      host=192.168.236.237\n\n      # Port the database server is listening to. Defaults are:\n      # 1. MySQL: 3306\n      # 2. PostgreSQL: 5432\n      # 3. Oracle Express Edition: 1521\n      port=3306\n\n      # Username to authenticate with when connecting to the database.\n      user=specialwu\n<\/code><\/pre>\n<h4><span class=\"ez-toc-section\" id=\"%E5%9C%A8hive%E5%90%AF%E5%8A%A8%E6%97%B6%E5%87%BA%E7%8E%B0Resources_are_low_on_NN_Please_add_or_free_up_more_resources_then_turn_off_safe_mode_manually\"><\/span>\u5728hive\u542f\u52a8\u65f6\u51fa\u73b0Resources are low on NN. Please add or free up more resources then turn off safe mode manually.<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<p>\u5b58\u50a8\u7a7a\u95f4\u4e0d\u591f<\/p>\n<h4><span class=\"ez-toc-section\" id=\"you_are_a_Hue_admin_but_not_a_HDFS_superuser_%E2%80%9Chdfs%E2%80%9D_or_part_of_HDFS_supergroup_%E2%80%9Csupergroup\"><\/span>you are a Hue admin but not a HDFS superuser, &#8220;hdfs&#8221; or part of HDFS supergroup, &#8220;supergroup<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<p>\u4e0a\u9762\u4e09\u4e2a\u914d\u7f6e\u6587\u4ef6\u672a\u4fee\u6539\uff0c\u6216\u8005\u662f\u4fee\u6539\u540e\u5f97\u91cd\u65b0\u542f\u52a8<\/p>\n<h4><span class=\"ez-toc-section\" id=\"hive%E8%BF%9E%E6%8E%A510000%E7%AB%AF%E5%8F%A3%E5%A4%B1%E8%B4%A5\"><\/span>hive\u8fde\u63a510000\u7aef\u53e3\u5931\u8d25<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<p><code>vim \/opt\/hive-1.2.2\/conf\/hive-env.sh\u4e2dexport HIVE_SERVER2_THRIFT_PORT=&lt;10000&gt;<\/code><br \/>\nhive\u5b89\u88c5\u76ee\u5f55\u4e0b\u6267\u884c\u4e0b\u9762\u8bed\u53e5 \u542f\u52a8hive\u7684metastore\u4ee5\u53cahiveserver2\u670d\u52a1<\/p>\n<pre><code class=\"language-bash line-numbers\">nohup bin\/hive --service metastore &amp;\nnohup bin\/hive --service hiveserver2 &amp;\n<\/code><\/pre>\n<h5><span class=\"ez-toc-section\" id=\"%E5%90%AF%E5%8A%A8hue_usersharehuebuildenvbin_supervisor\"><\/span>\u542f\u52a8hue <code>\/user\/share\/hue\/build\/env\/bin<\/code>  <code>supervisor<\/code><span class=\"ez-toc-section-end\"><\/span><\/h5>\n<h4><span class=\"ez-toc-section\" id=\"%E5%90%84%E8%8A%82%E7%82%B9jps%EF%BC%88Java_Virtual_Machine_Process_Status_Tool%EF%BC%89\"><\/span>\u5404\u8282\u70b9jps\uff08Java Virtual Machine Process Status Tool\uff09<span class=\"ez-toc-section-end\"><\/span><\/h4>\n<pre><code class=\"language-bash line-numbers\">wq1\n3139 NameNode\n4547 DataNode\n5268 Kafka\n18052 NodeManager\n17942 ResourceManager\n19814 Jps\n2697 SecondaryNameNode\n4940 QuorumPeerMain\n\nwq2\n3972 DataNode\n19654 Jps\n4344 QuorumPeerMain\n2537 SecondaryNameNode\n16570 NodeManager\n\nwq3\n42499 DataNode\n44661 QuorumPeerMain\n115591 NodeManager\n114505 RunJar\n28556 SecondaryNameNode\n114591 RunJar\n117470 Jps\n<\/code><\/pre>\n","protected":false},"excerpt":{"rendered":"<p>Linux\u5e38\u7528\u547d\u4ee4 \u67e5\u770b\u8fdb\u7a0b top top &#8211; 18:18:44 up 5 days, 17:15, 1 u [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":861,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":[],"categories":[7],"tags":[],"_links":{"self":[{"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/posts\/487"}],"collection":[{"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=487"}],"version-history":[{"count":48,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/posts\/487\/revisions"}],"predecessor-version":[{"id":1930,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/posts\/487\/revisions\/1930"}],"wp:featuredmedia":[{"embeddable":true,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=\/wp\/v2\/media\/861"}],"wp:attachment":[{"href":"http:\/\/www.specialwu.com\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=487"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=487"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/www.specialwu.com\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=487"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}