hadoop 初次 - 启动 , 配置 , 测试

时间：2010-08-11 来源：liukaiyi

# HDFS后台程序设置 : 名称节点，第二名称节点和数据节点 conf/hdfs-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>


<configuration>

<property>
   <name>dfs.replication</name>
   <value>2</value>
</property>


    <property>
       <name>dfs.name.dir</name>
       <value>/home/hadoop/hadoop/namenode</value>
    </property>


    <property>
       <name>dfs.data.dir</name>
       <value>/home/hadoop/hadoop/data</value>
    </property>

</configuration>

# hadoop 核心配置 ,比如 mapreduce 中普遍的 I/O 配置 core-site.xml

# mapreduce 后台配置： jobtracker 和 tasktracker # 这特别注意使用 localhost 别使用 IP 地址，具体为什么不知道，但会报错! mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>

<property>
   <name>mapred.job.tracker</name>
   <value>localhost:9001</value>
</property>


<property>
    <name>mapred.system.dir</name>
    <value>/home/hadoop/hadoop/tmp/mapred/system</value>
    <final>true</final>
</property>

</configuration>

# ssh 免登入注意配置 masters , slaves 纯文本文件

在主机的 nfs 上 ./bin/start-all.sh
运行测试 ./bin/hadoop jar ./hadoop-0.20.2-test.jar TestDFSIO -write -nrFiles 5 -fileSize 25

TestFDSIO.0.0.4
10/08/10 23:40:20 INFO mapred.FileInputFormat: nrFiles = 5
10/08/10 23:40:20 INFO mapred.FileInputFormat: fileSize (MB) = 25
10/08/10 23:40:20 INFO mapred.FileInputFormat: bufferSize = 1000000
10/08/10 23:40:20 WARN conf.Configuration: mapred-default.xml:a attempt to override final parameter: mapred.system.dir; Ignoring.
10/08/10 23:40:21 INFO mapred.FileInputFormat: creating control file: 25 mega bytes, 5 files
10/08/10 23:40:21 WARN conf.Configuration: mapred-default.xml:a attempt to override final parameter: mapred.system.dir; Ignoring.
10/08/10 23:40:21 INFO mapred.FileInputFormat: created control files for: 5 files
10/08/10 23:40:22 WARN mapred.JobClient: Use GenericOptionsParser for parsing the arguments. Applications should implement Tool for the same.
10/08/10 23:40:22 INFO mapred.FileInputFormat: Total input paths to process : 5
10/08/10 23:40:23 INFO mapred.JobClient: Running job: job_201008102337_0002
10/08/10 23:40:24 INFO mapred.JobClient: map 0% reduce 0%
10/08/10 23:40:41 INFO mapred.JobClient: map 40% reduce 0%
10/08/10 23:41:03 INFO mapred.JobClient: map 80% reduce 0%
10/08/10 23:41:06 INFO mapred.JobClient: map 80% reduce 13%
10/08/10 23:41:12 INFO mapred.JobClient: map 80% reduce 26%
10/08/10 23:41:15 INFO mapred.JobClient: map 100% reduce 26%
10/08/10 23:41:24 INFO mapred.JobClient: map 100% reduce 100%
10/08/10 23:41:26 INFO mapred.JobClient: Job complete: job_201008102337_0002
10/08/10 23:41:26 INFO mapred.JobClient: Counters: 18
10/08/10 23:41:26 INFO mapred.JobClient: Job Counters
10/08/10 23:41:26 INFO mapred.JobClient: Launched reduce tasks=1
10/08/10 23:41:26 INFO mapred.JobClient: Launched map tasks=5
10/08/10 23:41:26 INFO mapred.JobClient: Data-local map tasks=5
10/08/10 23:41:26 INFO mapred.JobClient: FileSystemCounters
10/08/10 23:41:26 INFO mapred.JobClient: FILE_BYTES_READ=471
10/08/10 23:41:26 INFO mapred.JobClient: HDFS_BYTES_READ=565
10/08/10 23:41:26 INFO mapred.JobClient: FILE_BYTES_WRITTEN=1126
10/08/10 23:41:26 INFO mapred.JobClient: HDFS_BYTES_WRITTEN=131072076
10/08/10 23:41:26 INFO mapred.JobClient: Map-Reduce Framework
10/08/10 23:41:26 INFO mapred.JobClient: Reduce input groups=5
10/08/10 23:41:26 INFO mapred.JobClient: Combine output records=0
10/08/10 23:41:26 INFO mapred.JobClient: Map input records=5
10/08/10 23:41:26 INFO mapred.JobClient: Reduce shuffle bytes=495
10/08/10 23:41:26 INFO mapred.JobClient: Reduce output records=5
10/08/10 23:41:26 INFO mapred.JobClient: Spilled Records=50
10/08/10 23:41:26 INFO mapred.JobClient: Map output bytes=415
10/08/10 23:41:26 INFO mapred.JobClient: Map input bytes=135
10/08/10 23:41:26 INFO mapred.JobClient: Combine input records=0
10/08/10 23:41:26 INFO mapred.JobClient: Map output records=25
10/08/10 23:41:26 INFO mapred.JobClient: Reduce input records=25
10/08/10 23:41:27 INFO mapred.FileInputFormat: ----- TestDFSIO ----- : write
10/08/10 23:41:27 INFO mapred.FileInputFormat: Date & time: Tue Aug 10 23:41:26 CST 2010
10/08/10 23:41:27 INFO mapred.FileInputFormat: Number of files: 5
10/08/10 23:41:27 INFO mapred.FileInputFormat: Total MBytes processed: 125
10/08/10 23:41:27 INFO mapred.FileInputFormat: Throughput mb/sec: 4.095540775203958
10/08/10 23:41:27 INFO mapred.FileInputFormat: Average IO rate mb/sec: 5.74212121963501
10/08/10 23:41:27 INFO mapred.FileInputFormat: IO rate std deviation: 2.875854060066087
10/08/10 23:41:27 INFO mapred.FileInputFormat: Test exec time sec: 64.975
10/08/10 23:41:27 INFO mapred.FileInputFormat: