Docker-Hadoop ALL-IN-ONE

2023年2月17日 354次阅读 来源: xi4okv

额，最近折腾机器学习和大数据，之前一直对hadoop有所了解，但是觉得用不上，现在有机遇可以接触一下。遂研究了一下网上的搭建方式。弄明白原理后找了些docker的搭建方案。发现了https://hub.docker.com/r/uhopper/hadoop/ 这个项目还比较火。可以一件部署hadoop生态环境。自带的容器有namenode 、 datanode、 resourcemanager、nodemanager、spark。如果整合其他的也可以通过环境变量引入配置文件的方式来处理。不过目前这几个容器也够用了。

话不多说。上代码

version: '3'
services:
    namenode:
      image: uhopper/hadoop-namenode:2.8.1
      hostname: namenode
      container_name: namenode
      domainname: hadoop
      net: hadoop
      volumes:
        - /namenode:/hadoop/dfs/name
      environment:
        - CLUSTER_NAME=datanode1
        - CLUSTER_NAME=datanode2
        - CLUSTER_NAME=datanode3

    datanode1:
      image: uhopper/hadoop-datanode:2.8.1
      hostname: datanode1
      container_name: datanode1
      domainname: hadoop
      net: hadoop
      volumes:
        - /datanode1:/hadoop/dfs/data
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
        
    datanode2:
      image: uhopper/hadoop-datanode:2.8.1
      hostname: datanode2
      container_name: datanode2
      domainname: hadoop
      net: hadoop
      volumes:
        - /datanode2:/hadoop/dfs/data
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020

    datanode3:
      image: uhopper/hadoop-datanode:2.8.1
      hostname: datanode3
      container_name: datanode3
      domainname: hadoop
      net: hadoop
      volumes:
        - /datanode3:/hadoop/dfs/data
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
    
    resourcemanager:
      image: uhopper/hadoop-resourcemanager:2.8.1
      hostname: resourcemanager
      container_name: resourcemanager
      domainname: hadoop
      net: hadoop
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
        - YARN_CONF_yarn_log___aggregation___enable=true
        
    nodemanager:
      image: uhopper/hadoop-nodemanager:2.8.1
      hostname: nodemanager
      container_name: nodemanager
      domainname: hadoop
      net: hadoop
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
        - YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
        - YARN_CONF_yarn_log___aggregation___enable=true
        - YARN_CONF_yarn_nodemanager_remote___app___log___dir=/app-logs
        
    spark:
      image: uhopper/hadoop-spark:2.8.1
      hostname: spark
      container_name: spark
      domainname: hadoop
      net: hadoop
      environment:
        - CORE_CONF_fs_defaultFS=hdfs://namenode:8020
        - YARN_CONF_yarn_resourcemanager_hostname=resourcemanager
      command: tail -f /var/log/dmesg

    
    ```
    原文作者：xi4okv
    原文地址: https://www.jianshu.com/p/6bcd72083c08
    本文转自网络文章，转载此文章仅为分享知识，如有侵权，请联系博主进行删除。
点赞