Hadoop实例WordCount程序一步一步运行[命令行]
虽说现在用Eclipse下开发Hadoop程序很方便了，但是命令行方式对于小程序开发验证很方便。这是初学hadoop时的笔记，记录下来以备查。
<!-- 正文结束 -->
1. 经典的WordCound程序（WordCount.java），可参见
import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MapReduceBase;
import org.apache.hadoop.mapred.Mapper;
import org.apache.hadoop.mapred.OutputCollector;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
public WordCount extends Configured implements Tool
{
public MapClass extends MapReduceBase implements
Mapper<LongWritable, Text, Text, IntWritable>{
private IntWritable
one = IntWritable( 1);
privateText
word = Text();
public map(LongWritable key, Text
value,
OutputCollector <Text, IntWritable> output, Reporter
reporter)
throws IOException {
String line = value.toString();
StringTokenizer itr = StringTokenizer(line);
while (itr.hasMoreTokens())
{
word.set(itr.nextToken());
output.collect(word, one);
}
}
}
publicReduce extends MapReduceBase implements
Reducer<Text, IntWritable, Text, IntWritable>{
public reduce(Text key,
Iterator <IntWritable> values,
OutputCollector <Text, IntWritable> output, Reporter
reporter)
throws IOException
{
int sum =
while(values.hasNext())
{
sum += values.next().get();
}
output.collect(key, new IntWritable(sum));
}
}
static printUsage() {
System.out.println( "wordcount [-m ] [-r ]
");
ToolRunner.printGenericCommandUsage(System.out);
return
}
publicrun(String[]
args) throws Exception {
JobConf conf = JobConf(getConf(),
WordCount. class);
conf.setJobName("wordcount");
//the keys are words
(strings)
conf.setOutputKeyClass(Text.class);
//the values are counts
(ints)
conf.setOutputValueClass(IntWritable.class);
conf.setMapperClass(MapClass.class);
conf.setCombinerClass(Reduce.class);
conf.setReducerClass(Reduce.class);
List<String>other_args = ArrayList <String>();
for( int i = args.length; ++i)
{
try{
if ( "-m".equals(args[i])) {
conf.setNumMapTasks(Integer.parseInt(args[++i]));
}( "-r".equals(args[i])) {
conf.setNumReduceTasks(Integer.parseInt(args[++i]));
}{
other_args.add(args[i]);
}
} catch (NumberFormatException
except) {
System.out.println( "ERROR: Integer expected instead
of
+args[i]);
return printUsage();
} catch (ArrayIndexOutOfBoundsException
except) {
System.out.println( "ERROR: Required parameter missing
from
+args[i -
returnprintUsage();
}
}
// Make sure there are
exactly 2 parameters left.
if(other_args.size() !=
System.out.println("ERROR: Wrong number of parameters:
+other_args.size() + instead
of 2. ");
returnprintUsage();
}
FileInputFormat.setInputPaths(conf, other_args.get( 0));
FileOutputFormat.setOutputPath(conf,Path(other_args.get( 1)));
JobClient.runJob(conf);
return
}
publicmain(String[]
args) throws Exception
{
int res = ToolRunner.run( new Configuration(), new WordCount(),
args);
System.exit(res);
}
}
Mapper<LongWritable, Text, Text, IntWritable>
private
OutputCollector
String line
StringTokenizer itr
word.set(itr.nextToken());
output.collect(word, one);
}
}
}
public
Reducer<Text, IntWritable, Text, IntWritable>
OutputCollector
while
sum
}
output.collect(key,
}
}
System.out.println(
ToolRunner.printGenericCommandUsage
}
public
JobConf conf
conf.setJobName("wordcount");
//
conf.setOutputKeyClass(Text.class);
//
conf.setOutputValueClass(IntWritable.class);
conf.setMapperClass(MapClass.class);
conf.setCombinerClass(Reduce.class);
conf.setReducerClass(Reduce.class);
List<String>
for
try
conf.setNumMapTasks(Integer.parseInt(args[++i]));
}
conf.setNumReduceTasks(Integer.parseInt(args[++i]));
}
other_args.add(args[i]);
}
}
System.out.println(
+
}
System.out.println(
+
return
}
}
if
System.out.println("ERROR: Wrong number of parameters:
+
return
}
FileInputFormat.setInputPaths(conf, other_args.get(
FileOutputFormat.setOutputPath(conf,
JobClient.runJob(conf);
return
}
public
System.exit(res);
}
}
2. 保证hadoop集群是配置好了的，单机的也好。新建一个目录，比如 /home/admin/WordCount
编译WordCount.java程序。
javac
3. 编译完后在/home/admin/WordCount目录会发现三个class文件
WordCount.class，WordCount$Map.class，WordCount$Reduce.class。
cd 进入 /home/admin/WordCount目录,然后执行：
jar cvf
WordCount.jar
就会生成 WordCount.jar 文件。
4. 构造一些输入数据
input1.txt和input2.txt的文件里面是一些单词。如下：
[admin@host WordCount]$ cat
input1.txt
Hello, i love china
are you ok?
[admin@host WordCount]$ cat input2.txt
hello, i love word
You are ok
Hello, i love china
are you ok?
[admin@host WordCount]$ cat input2.txt
hello, i love word
You are ok
在hadoop上新建目录，和put程序运行所需要的输入文件:
hadoop
fs
hadoop fs
hadoop fs
hadoop fs
hadoop fs
hadoop fs
hadoop fs
5. 运行程序,会显示job运行时的一些信息。
[admin@host WordCount]$ hadoop
jar WordCount.jar WordCount
10/09/16WARN mapred.JobClient: Use
GenericOptionsParser for parsing the arguments.
Applications should implement
Tool for the
same.
10/09/16 INFO
mapred.FileInputFormat: Total input paths to process
: 2
10/09/16INFO mapred.JobClient:
Running job: job_201008171228_76165
10/09/16 INFO mapred.JobClient:
map 0% reduce 0%
10/09/16INFO mapred.JobClient:
map 100% reduce 0%
10/09/16INFO mapred.JobClient:
map 100% reduce 100%
10/09/16INFO mapred.JobClient: Job
complete: job_201008171228_76165
10/09/16 INFO mapred.JobClient:
Counters: 16
10/09/16INFO mapred.JobClient:
File Systems
10/09/16 INFO mapred.JobClient:
HDFS bytes read =62
10/09/16INFO mapred.JobClient:
HDFS bytes written =73
10/09/16INFO mapred.JobClient:
Local bytes read =152
10/09/16INFO mapred.JobClient:
Local bytes written =366
10/09/16INFO mapred.JobClient: Job
Counters
10/09/16 INFO mapred.JobClient:
Launched reduce tasks =1
10/09/16INFO mapred.JobClient:
Rack -local map tasks=2
10/09/16INFO mapred.JobClient:
Launched map tasks =2
10/09/16INFO mapred.JobClient:
Map -Reduce Framework
10/09/16INFO mapred.JobClient:
Reduce input groups =11
10/09/16INFO mapred.JobClient:
Combine output records =14
10/09/16INFO mapred.JobClient: Map
input records =4
10/09/16INFO mapred.JobClient:
Reduce output records =11
10/09/16INFO mapred.JobClient: Map
output bytes =118
10/09/16INFO mapred.JobClient: Map
input bytes =62
10/09/16INFO mapred.JobClient:
Combine input records =14
10/09/16INFO mapred.JobClient: Map
output records =14
10/09/16INFO mapred.JobClient:
Reduce input records =14
10/09/16
10/09/16
10/09/16
10/09/16
10/09/16
10/09/16
10/09/16
10/09/16
10/09/16
10/09/16
10/09/16
10/09/16
10/09/16
10/09/16
10/09/16
10/09/16
10/09/16
10/09/16
10/09/16
10/09/16
10/09/16
10/09/16
6. 查看运行结果
[admin@host WordCount]$ hadoop
fs
Founditems
drwxr -x--- admin
admin 0
-rw-r-----admin
admin 102
[admin@host WordCount]$ hadoop fs
Hello,
You
are
china
hello,
i
love
ok
ok?
word
you
Found
drwxr
-rw-r-----
[admin@host WordCount]$ hadoop fs
Hello,
You
are
china
hello,
i
love
ok
ok?
word
you
来自 “ ITPUB博客 ” ，链接：https://blog.itpub.net/24065434/viewspace-1119786/ ，如需转载，请注明出处，否则将追究法律责任。
上一篇：
没有了~
下一篇：
没有了~
请登录后发表评论
登录
全部评论
