写于安装前

关于hadoop集群管理软件，我选择了cdh的免费版。本文参照官方文档、网上大神的帖子，加上自己几十次的搭建，踩了无数的坑总结而成。由于每个人的现场环境不同，参照本文档可能会出现报错情况，欢迎留言讨论交流。

1、基础环境

服务器配置:
24核48线程 64GB 600Gx2(raid1)
hadoop存储(4Tx10/每台机器)
目录–>/data1–>/data10

备注：mysql服务器复用计算调度

2、安装依赖

首先安装mysql
sudo su – root
yum install vim lvm2 ssl_mod chkconfig python bind-utils psmisc libxslt zlib sqlite fuse fuse-libs redhat-lsb cyrus-sasl-plain cyrus-sasl-gssapi pstree httpd rsync -y

3、准备离线包

cloudera-manager-centos7-cm5.13.0_x86_64.tar.gz
CDH-5.13.0-1.cdh5.13.0.p0.29-el7.parcel.sha1
CDH-5.13.0-1.cdh5.13.0.p0.29-el7.parcel
manifest.json
mysql-connector-java-5.1.44-bin.jar
oracle jdk 1.8.0_152

下载地址:
http://archive.cloudera.com/cm5/cm/5/
tttp://archive.cloudera.com/cdh5/parcels/latest/

4、 安装

4.1 所有节点

echo "*/20 * * * * /usr/sbin/ntpdate pool.ntp.org > /dev/null 2>&1" > /var/spool/cron/root
systemctl stop NetworkManager
systemctl disable NetworkManager
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config
setenforce 0
systemctl stop firewalld
systemctl disable firewalld

cat >> /etc/hosts <<END
10.10.12.31 cdh01
10.10.12.32 cdh02
10.10.12.33 cdh03
10.10.12.34 cdh04
10.10.12.34 cdh05
END

优化hugepage

echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled

chmod +x /etc/rc.d/rc.local

cat >>/etc/rc.local<<EOF
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
EOF
echo "vm.swappiness = 0">> /etc/sysctl.conf
sysctl -p

4.2 配置主机名

###10.10.12.31
hostnamectl set-hostname cdh01
###10.10.12.32
hostnamectl set-hostname cdh02
###10.10.12.33
hostnamectl set-hostname cdh03
###10.10.12.34
hostnamectl set-hostname cdh04
###10.10.12.35
hostnamectl set-hostname cdh05

cat >> /etc/hosts <<END
10.10.12.31 cdh01
10.10.12.32 cdh02
10.10.12.33 cdh03
10.10.12.34 cdh04
10.10.12.35 cdh05
END

4.3 配置ssh免密登陆

ssh-keygen -f $HOME/.ssh/id_rsa -t rsa -N ''
cat /root/.ssh/id_rsa.pub >/root/.ssh/authorized_keys
rsync -av /root/.ssh/* cdh01:/root/.ssh/
rsync -av /root/.ssh/* cdh02:/root/.ssh/
rsync -av /root/.ssh/* cdh03:/root/.ssh/
rsync -av /root/.ssh/* cdh04:/root/.ssh/
rsync -av /root/.ssh/* cdh05:/root/.ssh/

4.4 所有节点安装oracle jdk 1.8

4.4.1 主节点

yum install mod_ssl

mkdir -p /usr/java/jdk1.8

tar zxvf jdk-8u152-linux-x64.tar.gz -C /usr/java/

mv /usr/java/jdk1.8.0_152/* /usr/java/jdk1.8/

sed -i '$a export JAVA_HOME=/usr/java/jdk1.8/' /etc/profile

sed -i '$a export PATH=$JAVA_HOME/bin:$PATH' /etc/profile

source /etc/profile

java -version

echo $JAVA_HOME

rsync -av jdk-8u152-linux-x64.tar.gz cdh02:/usr/local/
rsync -av jdk-8u152-linux-x64.tar.gz cdh03:/usr/local/
rsync -av jdk-8u152-linux-x64.tar.gz cdh04:/usr/local/
rsync -av jdk-8u152-linux-x64.tar.gz cdh05:/usr/local/

4.4.2 其他节点

`cd /usr/local`

`mkdir -p /usr/java/jdk1.8`

`tar zxvf jdk-8u152-linux-x64.tar.gz -C /usr/java/`

`mv /usr/java/jdk1.8.0_152/* /usr/java/jdk1.8/`

`sed -i '$a export JAVA_HOME=/usr/java/jdk1.8/' /etc/profile`

`sed -i '$a export PATH=$JAVA_HOME/bin:$PATH' /etc/profile`

`source /etc/profile`

`java -version`

`echo $JAVA_HOME`

4.4.3 安装mysql(两个实例3306 3307)

rpm -aq|grep mariadb|xargs rpm -e --nodeps
rpm -ivh http://repo.mysql.com/mysql-community-release-el7-5.noarch.rpm
yum install mysql-server
vi /etc/my.cnf
binlog_format=mixed
systemctl restart mysqld

mysql授权

create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
grant all privileges on *.* to root@"localhost" identified by pwd;
grant all privileges on *.* to root@"%" identified by pwd;
grant all privileges on hive.* to hive@"localhost" identified by pwd;
grant all privileges on hive.* to hive@"%" identified by pwd;
grant all privileges on hue.* to hue@"localhost" identified by pwd;
grant all privileges on hue.* to hue@"%" identified by pwd;
grant all privileges on oozie.* to oozie@"localhost" identified by pwd;
grant all privileges on oozie.* to oozie@"%" identified by pwd;

grant all privileges on cm.* to scm@"%" identified by pwd;

主节点

mkdir -p /var/lib/cloudera-scm-server/
cd /root/app/
tar -zxvf cloudera-manager-centos7-cm5.13.0_x86_64.tar.gz -C /opt
cp mysql-connector-java-5.1.44-bin.jar /opt/cm-5.13.0/share/cmf/lib/

/opt/cm-5.13.0/share/cmf/schema/scm_prepare_database.sh mysql cm -hhost -uroot -ppwd--scm-host cdh01 scm scm scm
vi /opt/cm-5.13.0/etc/cloudera-scm-agent/config.ini 
server_host=cdh01
rsync -av /opt/cm-5.13.0 root@cdh02:/opt/
rsync -av /opt/cm-5.13.0 root@cdh03:/opt/
rsync -av /opt/cm-5.13.0 root@cdh04:/opt/
rsync -av /opt/cm-5.13.0 root@cdh05:/opt/

所有节点

useradd --system --home=/opt/cm-5.13.0/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm

主节点

mkdir -p /opt/cloudera/parcel-repo/
cp /root/app/CDH-5.13.0-1.cdh5.13.0.p0.29-el7.parcel /opt/cloudera/parcel-repo/
cp /root/app/CDH-5.13.0-1.cdh5.13.0.p0.29-el7.parcel.sha1 /opt/cloudera/parcel-repo/
cp /root/app/manifest.json /opt/cloudera/parcel-repo/
mv /opt/cloudera/parcel-repo/CDH-5.13.0-1.cdh5.13.0.p0.29-el7.parcel.sha1 /opt/cloudera/parcel-repo/CDH-5.13.0-1.cdh5.13.0.p0.29-el7.parcel.sha
mkdir -p /usr/share/java/
cp /root/app/mysql-connector-java-5.1.44-bin.jar  /usr/share/java/mysql-connector-java.jar
#cp /root/app/mysql-connector-java-5.1.44-bin.jar /opt/cloudera/parcels/CDH-5.13.0-1.cdh5.13.0.p0.29/lib/hive/lib/
/opt/cm-5.13.0/etc/init.d/cloudera-scm-server start
tailf /opt/cm-5.13.0/log/cloudera-scm-server/cloudera-scm-server.log

Table ‘cm.CM_VERSION’ doesn’t exist

所有节点

/opt/cm-5.13.0/etc/init.d/cloudera-scm-agent start
/opt/cm-5.13.0/etc/init.d/cloudera-scm-agent restart
tailf /opt/cm-5.13.0/log/cloudera-scm-agent/cloudera-scm-agent.log

打开浏览器
http://10.10.12.31:7180

usermod -s /bin/bash hue
echo "hue ALL = (root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/hue
chmod 0440 /etc/sudoers.d/hue

vi /opt/cm-5.13.0/lib64/cmf/service/hue/httpd.sh

add after line No. 129

sudo su – root -c “chown hue: $CLOUDERA_HTTPD_CONF_DIR/*.conf”

5、启用HDFS 的 High Availability

5.1 在HDFS上点击”启用High Availability”

5.2停止hive

更新 Hive Metastore NameNode

写于安装后

至此，hadoop集群能够完整运行。后续会写上spark、hive等组件的升级过程。本文侵删。