额,最近折腾机器学习和大数据,之前一直对hadoop有所了解,但是觉得用不上,现在有机遇可以接触一下。遂研究了一下网上的搭建方式。弄明白原理后找了些docker的搭建方案。发现了https://hub.docker.com/r/uhopper/hadoop/ 这个项目还比较火。可以一件部署hadoop生态环境。自带的容器有namenode 、 datanode、 resourcemanager、nodemanager、spark。如果整合其他的也可以通过环境变量引入配置文件的方式来处理。不过目前这几个容器也够用了。
话不多说。上代码
version: '3'
services:
namenode:
image: uhopper/hadoop-namenode:2.8.1
hostname: namenode
container_name: namenode
domainname: hadoop
net: hadoop
volumes:
- /namenode:/hadoop/dfs/name
environment:
- CLUSTER_NAME=datanode1
- CLUSTER_NAME=datanode2
- CLUSTER_NAME=datanode3
datanode1:
image: uhopper/hadoop-datanode:2.8.1
hostname: datanode1
container_name: datanode1
domainname: hadoop
net: hadoop
volumes:
- /datanode1:/hadoop/dfs/data
environment:
- CORE_CONF_fs_defaultFS=hdfs://namenode:8020
datanode2:
image: uhopper/hadoop-datanode:2.8.1
hostname: datanode2
container_name: datanode2
domainname: hadoop
net: hadoop
volumes:
- /datanode2:/hadoop/dfs/data
environment:
- CORE_CONF_fs_defaultFS=hdfs://namenode:8020
datanode3:
image: uhopper/hadoop-datanode:2.8.1
hostname: datanode3
container_name: datanode3
domainname: hadoop
net: hadoop
volumes:
- /datanode3:/hadoop/dfs/data
environment:
- CORE_CONF_fs_defaultFS=hdfs://namenode:8020
resourcemanager:
image: uhopper/hadoop-resourcemanager:2.8.1
hostname: resourcemanager
container_name: resourcemanager
domainname: hadoop
net: hadoop
environment:
- CORE_CONF_fs_defaultFS=hdfs://namenode:8020
- YARN_CONF_yarn_log___aggregation___enable=true
nodemanager:
image: uhopper/hadoop-nodemanager:2.8.1
hostname: nodemanager
container_name: nodemanager
domainname: hadoop
net: hadoop
environment:
- CORE_CONF_fs_defaultFS=hdfs://namenode:8020
- YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
- YARN_CONF_yarn_log___aggregation___enable=true
- YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs
spark:
image: uhopper/hadoop-spark:2.8.1
hostname: spark
container_name: spark
domainname: hadoop
net: hadoop
environment:
- CORE_CONF_fs_defaultFS=hdfs://namenode:8020
- YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
command: tail -f /var/log/dmesg
```