大數(shu)據(ju)架構：搭建CDH5.5.1分布式集群環境
yum install -y ntp gcc make lrzsz wget vim sysstat.x86_64 xinetd screen expect rsync bind-utils iotop dstat nethogs openssl-devel openssh-clients
1) 配(pei)置 /etc/hosts
master 10.17.172.230
node1 10.17.172.231
node2 10.17.172.232
node3 10.17.172.233
node4 10.17.172.234
node5 10.17.172.235
node6 10.17.172.236
node7 10.17.172.237
2) HOSTNAME
vi /etc/sysconfig/network
HOSTNAME=master
hostname master
service network restart
host一次配(pei)好(hao)，不要(yao)(yao)去改；如(ru)果非要(yao)(yao)改，相(xiang)關服務都要(yao)(yao)重啟, authorized_keys要(yao)(yao)處(chu)理, known_hosts要(yao)(yao)刪除
3) 無密碼SSH
在主節點上執行 ssh-keygen -t rsa 一路(lu)回車，生成無密(mi)碼的密(mi)鑰對。
將所(suo)有節(jie)點的公鑰添(tian)加(jia)到認證文件中：
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
scp文件到所有(you)datenode節點：
scp ~/.ssh/authorized_keys root@node1:~/.ssh/
scp ~/.ssh/authorized_keys root@node2:~/.ssh/
scp ~/.ssh/authorized_keys root@node3:~/.ssh/
scp ~/.ssh/authorized_keys root@node4:~/.ssh/
scp ~/.ssh/authorized_keys root@node5:~/.ssh/
scp ~/.ssh/authorized_keys root@node6:~/.ssh/
設置authorized_keys的訪問權限：
chmod 600 ~/.ssh/authorized_keys
4) 修改 history命(ming)令 的(de)記錄(lu)數(shu)量
sed -i 's/^HISTSIZE=1000/HISTSIZE=10000/' /etc/profile
source /etc/profile
5) 安裝 JAVA
rpm -qa | grep java 檢查是否安(an)裝 jdk
rpm -ivh jdk-7u80-linux-x64.rpm 安裝jdk
vi /etc/profile
JAVA_HOME=/usr/java/jdk1.7.0_80
export PATH=$JAVA_HOME/bin:$PATH
export JRE_HOME=${JAVA_HOME}/jre
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
source /etc/profile
6) 配置 防火墻
service iptables stop # 暫時關閉
7) 關(guan)閉SELINUX
臨時生效:
setenforce 0
重啟后永久(jiu)生(sheng)效:
cat /etc/selinux/config | grep "SELINUX="
vi /etc/selinux/config
SELINUX=disabled
8) 時區
vi /etc/sysconfig/clock
ZONE=Asia/Shanghai
rm /etc/localtime
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
9) 配(pei)置 NTP 節點(dian)
yum -y install ntp ntpdate
chkconfig ntpd on # 開(kai)機啟動
chkconfig --list ntpd # 2-5為on狀(zhuang)態(tai)就代表成(cheng)功
master機器:
ntpdate -u 103.226.213.30 # 同步一下時間
vi /etc/ntp.conf
driftfile /var/lib/ntp/drift
restrict 127.0.0.1
restrict -6 ::1
restrict default nomodify notrap
restrict -6 default nomodify notrap
server 103.226.213.30 prefer
server 0.rhel.pool.ntp.org iburst
server 1.rhel.pool.ntp.org iburst
server 2.rhel.pool.ntp.org iburst
server 3.rhel.pool.ntp.org iburst
server 210.72.145.44 iburst
server pool.ntp.org iburst
server time.windows.com iburst
server ntp.sjtu.edu.cn iburst
server time.asia.apple.com iburst
server clock.via.net ntp.nasa.gov iburst
includefile /etc/ntp/crypto/pw
keys /etc/ntp/keys
service ntpd start # 啟(qi)動
ntpstat # 命令查看同步狀態
所(suo)有子節點(dian):
vi /etc/ntp.conf
driftfile /var/lib/ntp/drift
restrict 127.0.0.1
restrict -6 ::1
restrict default kod nomodify notrap nopeer noquery
restrict -6 default kod nomodify notrap nopeer noquery
server master
includefile /etc/ntp/crypto/pw
keys /etc/ntp/keys
ntpdate -u master
service ntpd start
10) MySQL
rpm -qa | grep -i mysql
rpm --nodeps -e mysql-libs-5.1.71-1.el6.x86_64
cd /usr/local
tar -zxvf mysql.tar.gz
cp /data/install_tools/init_server_env/mysql/my.cnf /usr/local/mysql/
cp /data/install_tools/init_server_env/mysql/shutdown_mysql.sh /usr/local/mysql/
cp /data/install_tools/init_server_env/mysql/start_mysql.sh /usr/local/mysql/
# 創建數據(ju)庫相(xiang)應(ying)目錄和設置目錄權限(xian)
mkdir -p /data/mysql-3306/{tmp,data}
groupadd mysql
useradd -r -g mysql mysql -s /sbin/nologin
chmod 755 /usr/local/mysql/*.sh
chown -R mysql:mysql /data/mysql-3306
chown -R mysql:mysql /usr/local/mysql
##初始(shi)化mysql
cd /usr/local/mysql
scripts/mysql_install_db --user=mysql --defaults-file=/usr/local/mysql/my.cnf
/usr/local/mysql/start_mysql.sh
netstat -ant|grep ':3306'|grep 'LISTEN' # 確(que)認 3306 端口可用
/usr/local/mysql/bin/mysql -uroot -S /data/mysql-3306/mysqld.sock
use mysql;
delete from user where user='';
grant all privileges on *.* to root@'%' identified by 'qweasdzxc';
update user set password=password('qweasdzxc') where user='root';
grant select,insert,update,EXECUTE,lock tables on *.* to loguser@'%' identified by 'nagioscheck';
grant usage on *.* to 'zabbixmysql'@'127.0.0.1' identified by 'zabbix123';
flush privileges;
select host,user,password from user;
首先要看本機MySQL的socket套(tao)接字文件在(zai)哪里(li):
mysqld --verbose --help | grep socket
vi /etc/profile
export PATH==$PATH:/usr/local/mysql/bin
source /etc/profile
11)安裝(zhuang)CM==================================================================================================================
1) 主節點解壓(ya)安裝(zhuang)
解壓(ya) cloudera-manager-el6-cm5.5.1_x86_64.tar.gz 到 /opt/目錄
2) 建立(li)數(shu)據庫
cp /data/install_tools/mysql-connector-java-5.1.42.jar /opt/cm-5.5.1/share/cmf/lib/
# 這(zhe)個權限，應該要(yao)
grant all privileges on *.* to scm@'%' identified by 'scm';
flush privileges;
數據庫類型 數據庫名(ming)(ming)稱 -h數據庫主(zhu)(zhu)機(ji)名(ming)(ming) -u數據庫用戶名(ming)(ming) -p數據庫密碼--scm-host cmserver主(zhu)(zhu)機(ji)名(ming)(ming) scm scm scm
/opt/cm-5.5.1/share/cmf/schema/scm_prepare_database.sh mysql cm -h127.0.0.1 -uroot -pqweasdzxc -P3306 --scm-host master scm scm scm
3) Agent配置
vi /opt/cm-5.5.1/etc/cloudera-scm-agent/config.ini
server_host=master
同(tong)步Agent到其他節點
scp -r root@master:/opt/cm-5.5.1/ /opt/
在所有節點創建cloudera-scm用戶(hu)
userdel cloudera-scm
useradd --system --home=/opt/cm-5.5.1/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm
準備Parcels，用以安裝CDH5(主節點)
cp /data/install_tools/CDH/* /opt/cloudera/parcel-repo/
mv /opt/cloudera/parcel-repo/CDH-5.5.1-1.cdh5.5.1.p0.11-el6.parcel.sha1 /opt/cloudera/parcel-repo/CDH-5.5.1-1.cdh5.5.1.p0.11-el6.parcel.sha
啟動:
echo > /opt/cm-5.5.1/log/cloudera-scm-server/cloudera-scm-server.log
echo > /opt/cm-5.5.1/log/cloudera-scm-agent/cloudera-scm-agent.log
主節點:
/opt/cm-5.5.1/etc/init.d/cloudera-scm-server start
tail -f /opt/cm-5.5.1/log/cloudera-scm-server/cloudera-scm-server.log
2017-06-28 11:02:29,570 INFO WebServerImpl:org.mortbay.log: Started SelectChannelConnector@0.0.0.0:7180
2017-06-28 11:02:29,570 INFO WebServerImpl:com.cloudera.server.cmf.WebServerImpl: Started Jetty server.
標(biao)志(zhi)執行成功
所有節點(dian)（包括(kuo)主節點(dian)）: 最好(hao)先啟動子節點(dian)，最后啟動主節點(dian)
/opt/cm-5.5.1/etc/init.d/cloudera-scm-agent start
tail -n20 /opt/cm-5.5.1/log/cloudera-scm-agent/cloudera-scm-agent.log # log 里面有(you)錯(cuo)應該(gai)是(shi)正常的
DnsResolutionMonitor throttling_logger INFO Using java location: '/usr/java/jdk1.7.0_80/bin/java'.
應該是標志(zhi)執行成功(gong),不確認(ren)
4) CDH5的安裝(zhuang)配置
//master:7180/cmf/
初始化:
MySQL 建(jian)庫(ku):
create database hive;
create database oozie;
create database amon;
create database hue;
性能相關:
echo 0 > /proc/sys/vm/swappiness
echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag
echo "" >> /etc/sysctl.conf
echo "vm.swappiness=0" >> /etc/sysctl.conf
sysctl -p
echo "" >> /etc/rc.local
echo "echo never > /sys/kernel/mm/redhat_transparent_hugepage/defrag" >> /etc/rc.local
【群(qun)集(ji)設(she)置 - 審核更改】頁面(mian)時:
# 使用了MySql作為(wei)hive的(de)元數據存儲，hive默認沒有帶mysql的(de)驅(qu)動
cp /opt/cm-5.5.1/share/cmf/lib/mysql-connector-java-5.1.42.jar /opt/cloudera/parcels/CDH-5.5.1-1.cdh5.5.1.p0.11/lib/hive/lib/
【群集設置(zhi) - 首(shou)次運行 命令】頁面時:
cp /opt/cm-5.5.1/share/cmf/lib/mysql-connector-java-5.1.42.jar /var/lib/oozie/
========================================================================================================================
錯誤(wu):
1) Heartbeating to master:7182 failed
由(you)于在主節點上啟動了(le)Agent后，又將Agent scp到了(le)其(qi)他節點上導致(zhi)的，首次啟動Agent，它(ta)會生成(cheng)一個uuid。
mv /usr/bin/host /usr/bin/host.bak
rm -rf /opt/cm-5.5.1/lib/cloudera-scm-agent/uuid
========================================================================================================================
iptables 配置(zhi):
master:
*filter
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [114502:11447075]
-A INPUT -p tcp -m tcp --dport 8088 -j ACCEPT
-A INPUT -p tcp -m tcp --dport 50070 -j ACCEPT
-A INPUT -p tcp -m tcp --dport 7180 -j ACCEPT
-A INPUT -p tcp -m tcp --dport 19888 -j ACCEPT
-A INPUT -s 10.17.172.0/24 -p tcp -j ACCEPT
-A INPUT -p tcp -m tcp --dport 1022 -j ACCEPT
-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
-A INPUT -p icmp -j ACCEPT
-A INPUT -i lo -j ACCEPT
-A INPUT -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT
-A INPUT -j REJECT --reject-with icmp-host-prohibited
-A FORWARD -j REJECT --reject-with icmp-host-prohibited
COMMIT
node[0-6]:
*filter
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [249:74801]
-A INPUT -p tcp -m tcp --dport 10000 -j ACCEPT
-A INPUT -p tcp -m tcp --dport 8042 -j ACCEPT
-A INPUT -s 10.17.172.0/24 -p tcp -j ACCEPT
-A INPUT -p tcp -m tcp --dport 1022 -j ACCEPT
-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT
-A INPUT -p icmp -j ACCEPT
-A INPUT -i lo -j ACCEPT
-A INPUT -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT
-A INPUT -j REJECT --reject-with icmp-host-prohibited
-A FORWARD -j REJECT --reject-with icmp-host-prohibited
COMMIT
========================================================================================================================
卸載:
umount cm_processes
rpm -qa | grep cloudera
rpm -e cloudera-manager-daemons-5.5.1-1.cm551.p0.8.el6.x86_64
ps -ef | grep cmf | grep -v "grep" | awk '{print "kill -9 " $2}' | sh
ps -ef | grep java | grep -v "grep" | awk '{print "kill -9 " $2}' | sh
清理數(shu)據庫數(shu)據
rm -rf /tmp/.scm_prepare_node.lock
rm -rf /tmp/scm_prepare_node
rm -rf /etc/cloudera*
rm -rf /usr/share/cmf /var/lib/cloudera* /var/cache/yum/x86_64/6/cloudera* /var/log/cloudera*
rm -rf /var/run/hadoop* /var/run/flume-ng /var/run/cloudera* /var/run/oozie/ /var/run/sqoop2
rm -rf /var/run/zookeeper /var/run/hbase /var/run/impala /var/run/hive /var/run/hdfs-sockets
rm -rf /var/lib/flume-ng /var/lib/hadoop* /var/lib/hue /var/lib/oozie /var/lib/solr /var/lib/sqoop*
rm -rf /dfs /mapred /yarn
rm -rf /data/dfs
rm -rf /usr/lib/hadoop /usr/lib/hadoop*
rm -rf /usr/lib/hive /usr/lib/hbase /usr/lib/oozie
rm -rf /usr/lib/sqoop* /usr/lib/zookeeper /usr/lib/bigtop*
rm -rf /usr/lib/flume-ng /usr/lib/hcatalog
rm -rf /usr/bin/hadoop* /usr/bin/zookeeper*
rm -rf /usr/bin/hbase* /usr/bin/hive*
rm -rf /usr/bin/hdfs /usr/bin/mapred /usr/bin/yarn /usr/bin/sqoop* /usr/bin/oozie
rm -rf /var/cache/yum/cloudera* /var/lib/zookeeper
rm -rf /opt/cm-5.5.1
rm -rf /opt/cloudera