HA环境
| hostname | ip | node |
|---|---|---|
| hp1 | 192.168.1.212 | zookeeper,JournalNode,NameNode,DataNode,NodeManager |
| hp2 | 192.168.1.222 | zookeeper,JournalNode,NameNode,DataNode,ResourceManager,NodeManager |
| hp3 | 192.168.1.232 | zookeeper,JournalNode,DataNode,ResourceManager,NodeManager |
| jf4 | 192.168.1.204 | dnamasq,ntpd |
centos7 minimal
jdk-8u171-linux-x64.rpm
zookeeper-3.4.12
hadoop-2.9.1
部署
# git clone http://git.jfbrother.com/zhengtianbao/jfbigdata-ansible.git
# cat << EOF > hosts
[dns]
hp1
hp2
hp3
[ntp]
hp1
hp2
hp3
[jdk]
hp1
hp2
hp3
[zookeepers]
hp1 myid=1
hp2 myid=2
hp3 myid=3
[hadoop_hosts]
hp1
hp2
hp3
[namenodes]
hp1
hp2
[journalnodes]
hp1
hp2
hp3
[datanodes]
hp1
hp2
hp3
[yarnrmnodes]
hp2
hp3
[yarnnmnodes]
hp1
hp2
hp3
[zookeeper_hosts]
hp1
hp2
hp3
EOF
# cat << EOF > site.yml
- name: sync ntpdate
hosts: zookeepers
remote_user: root
roles:
- ntp
tags:
- ntp
- name: configure local DNS server
hosts: dns
remote_user: root
roles:
- {role: dns, dns_nameservers:['192.168.1.204']}
tags:
- dns
- name: deploy jdk
hosts: jdk
remote_user: root
roles:
- java
tags:
- java
- name: deploy zookeeper cluster
hosts: zookeepers
remote_user: root
roles:
- zookeeper
tags:
- zookeeper
- name: deploy HDFS
hosts: hadoop_hosts
remote_user: root
roles:
- hadoop
tags:
- hadoop
EOF
# ansible-playbook -t ntp,dns,java,zookeeper,hadoop -i hosts site.yml
页面
http://hp1:50070
http://hp3:8088测试
# su - hadoop
# echo 'hello world' >> test.txt
# hdfs dfs -mkdir /input
# hadoop fs -put *.txt /input
# hadoop jar /usr/local/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.9.1.jar wordcount /input /output
# hadoop fs -cat /output/part-r-00000