hadoop安装

hadoop安装

零、hadoop启动命令

start-all.sh
start-dfs start-yarn
hadoop-deamons.sh yarn-deamons.sh
hadoop-deamon.sh yarn-deamon.sh
mr-jobhistoryserver.sh start historyserver
yarn-deamon.sh start timelineserver

一、准备安装包

D:\yilng\bigdata\hadoop\安装包 hadoop-2.6.0.tar.gz

二、使用SCP工具上传到服务器

# 上传路径:/opt/install_bak
# 解压路径:/opt/
tar -zxvf /opt/install_bak/hadoop-2.6.0.tar.gz -C ../

三、修改配置文件

hadoop-env.sh

vim hadoop-env.sh
export JAVA_HOME=/opta/jdk1.8.0_161

core-site.xml

<configuration>
	<property>
		<name>fs.defaultFS</name>
		<value>hdfs://master:8020</value>
	</property>
	<property>
		<name>io.file.buffer.size</name>
		<value>4096</value>
	</property>
	<property>
		<name>hadoop.tmp.dir</name>
		<value>/opt/hadoop-2.6.0/tmp</value>
	</property>
</configuration>

hdfs-site.xml

<configuration>
	<property>
		<name>dfs.replication</name>
		<value>3</value>
	</property>
	<property>
		<name>dfs.block.size</name>
		<value>134217728</value>
	</property>
	<property>
		<name>dfs.namenode.name.dir</name>
		<value>file:///opt/hadoop-2.6.0/hadoopdata/dfs/name</value>
	</property>
	<property>
		<name>dfs.datanode.data.dir</name>
		<value>file:///opt/hadoop-2.6.0/hadoopdata/dfs/data</value>
	</property>
	<property>
		<name>fs.checkpoint.dir</name>
		<value>file:///opt/hadoop-2.6.0/hadoopdata/checkpoint/dfs/cname</value>
	</property>
	<property>
		<name>fs.checkpoint.edits.dir</name>
		<value>file:///opt/hadoop-2.6.0/hadoopdata/checkpoint/dfs/cname</value>
	</property>
	<property>
		<name>dfs.http.address</name>
		<value>master:50070</value>
	</property>
	<property>
		<name>dfs.secondary.http.address</name>
		<value>slave1:50090</value>
	</property>
	<property>
		<name>dfs.webhdfs.enabled</name>
		<value>true</value>
	</property>
	<property>
		<name>dfs.permissions</name>
		<value>false</value>
	</property>
</configuration>

mapred-site.xml

<configuration>
	<property>
		<name>mapreduce.framework.name</name>
		<value>yarn</value>
		<final>true</final>
	</property>
	<property>
		<name>mapreduce.jobhistory.address</name>
		<value>master:10020</value>
	</property>
	<property>
		<name>mapreduce.jobhistory.webapp.address</name>
		<value>master:19888</value>
	</property>
</configuration>

yarn-site.xml

<configuration>
	<property>
		<name>yarn.resourcemanager.hostname</name>
		<value>master</value>
	</property>
	<property>
		<name>yarn.nodemanager.aux-services</name>
		<value>mapreduce_shuffle</value>
	</property>
	<property>
		<name>yarn.resourcemanager.address</name>
		<value>master:8032</value>
	</property>
	<property>
		<name>yarn.resourcemanager.scheduler.address</name>
		<value>master:8030</value>
	</property>
	<property>
		<name>yarn.resourcemanager.resource-tracker.address</name>
		<value>master:8031</value>
	</property>
	<property>
		<name>yarn.resourcemanager.admin.address</name>
		<value>master:8033</value>
	</property>
	<property>
		<name>yarn.resourcemanager.webapp.address</name>
		<value>master:8088</value>
	</property>
	
	<property>
		<name>yarn.nodemanager.pmem-check-enabled</name>
		<value>false</value>
	</property>
	<property>
		<name>yarn.nodemanager.vmem-check-enabled</name>
		<value>false</value>
	</property>



<!--开启日志-->
	<property>
		<description></description>
		<name>yarn.timeline-service.hostname</name>
		<value>master</value>
	</property>
	<property>
		<description>Address for the Timeline server to start the RPC server.</description>
		<name>yarn.timeline-service.address</name>
		<value>${yarn.timeline-service.hostname}:10200</value>
	</property>
	<property>
		<description>The http address of the Timeline service web application.</description>
		<name>yarn.timeline-service.webapp.address</name>
		<value>${yarn.timeline-service.hostname}:8188</value>
	</property>
	<property>
		<description>The https address of the Timeline service web application.</description>
		<name>yarn.timeline-service.webapp.https.address</name>
		<value>${yarn.timeline-service.hostname}:8190</value>
	</property>
	<property>
		<description>Handler thread count to serve the client RPC requests.</description>
		<name>yarn.timeline-service.handler-thread-count</name>
		<value>10</value>
	</property>
	<property>
		<name>yarn.timeline-service.http-cross-origin.enabled</name>  
		<value>false</value>
	</property>
	<property>
		<name>yarn.timeline-service.http-cross-origin.allowed-origins</name> 
		<value>*</value>
	</property>
	<property>
		<name>yarn.timeline-service.http-cross-origin.allowed-methods</name>
		<value>GET,POST,HEAD</value>
	</property>
	<property>
		<name>yarn.timeline-service.http-cross-origin.allowed-headers</name>
		<value>X-Requested-With,Content-Type,Accept,Origin</value>
	</property>
	<property>
		<name>yarn.timeline-service.http-cross-origin.max-age</name>
		<value>1800</value>
	</property>
	<property>
		<name>yarn.timeline-service.generic-application-history.enabled</name>
		<value>true</value>
	</property>
	<property>
		<name>yarn.timeline-service.generic-application-history.store-class</name>
		<value>org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore</value>
	</property>
	<property>
		<name>yarn.timeline-service.enabled</name>
		<value>true</value>
	</property>
	<property>
		<name>yarn.timeline-service.store-class</name>
		<value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value>
	</property>
	<property>
		<name>yarn.timeline-service.ttl-enable</name>
		<value>true</value>
	</property>
	<property>
		<name>yarn.timeline-service.ttl-ms</name>
		<value>6048000000</value>
	</property>
</configuration>

slaves

master
slave1
slave2

master

master

四、将配置好的jdk文件夹和Hadoop文件夹发送给slave1和slave2服务器

scp -r jdk slave1:/opt/jdk
scp -r jdk slave2:/opt/jdk
scp -r /opt/hadoop-2.6.0 slave1:/opt/
scp -r /opt/hadoop-2.6.0 slave2:/opt/

五、格式化

hadoop namenode -format

六、启动hadoop集群

start-dfs.sh
start-yarn.sh
或者
start-all.sh

七、启动之后查看jps

image

image

image