利用hdfs的api，可以实现向hdfs的文件、目录读写，利用这一套API可以设计一个简易的山寨版云盘，见下图：

为了方便操作,将常用的文件读写操作封装了一个工具类:

1 import org.apache.hadoop.conf.Configuration;

2 import org.apache.hadoop.fs.*;

3 import org.apache.hadoop.io.IOUtils;

4

5 import java.io.ByteArrayOutputStream;

6 import java.io.IOException;

7 import java.io.InputStream;

8 import java.io.OutputStream;

9

10

15 public class HDFSUtil {

16

17

18 private HDFSUtil() {

19

20 }

21

22

30 public static boolean exits(Configuration conf, String path) throws IOException {

31 FileSystem fs = FileSystem.get(conf);

32 return fs.exists(new Path(path));

33 }

34

35

43 public static void createFile(Configuration conf, String filePath, byte[] contents) throws IOException {

44 FileSystem fs = FileSystem.get(conf);

45 Path path = new Path(filePath);

46 FSDataOutputStream outputStream = fs.create(path);

47 outputStream.write(contents);

48 outputStream.close();

49 fs.close();

50 }

51

52

60 public static void createFile(Configuration conf, String filePath, String fileContent) throws IOException {

61 createFile(conf, filePath, fileContent.getBytes());

62 }

63

64

70 public static void copyFromLocalFile(Configuration conf, String localFilePath, String remoteFilePath) throws IOException {

71 FileSystem fs = FileSystem.get(conf);

72 Path localPath = new Path(localFilePath);

73 Path remotePath = new Path(remoteFilePath);

74 fs.copyFromLocalFile(true, true, localPath, remotePath);

75 fs.close();

76 }

77

78

87 public static boolean deleteFile(Configuration conf, String remoteFilePath, boolean recursive) throws IOException {

88 FileSystem fs = FileSystem.get(conf);

89 boolean result = fs.delete(new Path(remoteFilePath), recursive);

90 fs.close();

91 return result;

92 }

93

94

102 public static boolean deleteFile(Configuration conf, String remoteFilePath) throws IOException {

103 return deleteFile(conf, remoteFilePath, true);

104 }

105

106

115 public static boolean renameFile(Configuration conf, String oldFileName, String newFileName) throws IOException {

116 FileSystem fs = FileSystem.get(conf);

117 Path oldPath = new Path(oldFileName);

118 Path newPath = new Path(newFileName);

119 boolean result = fs.rename(oldPath, newPath);

120 fs.close();

121 return result;

122 }

123

124

132 public static boolean createDirectory(Configuration conf, String dirName) throws IOException {

133 FileSystem fs = FileSystem.get(conf);

134 Path dir = new Path(dirName);

135 boolean result = fs.mkdirs(dir);

136 fs.close();

137 return result;

138 }

139

140

147 public static RemoteIterator listFiles(FileSystem fs, String basePath, boolean recursive) throws IOException {

148

149 RemoteIterator fileStatusRemoteIterator = fs.listFiles(new Path(basePath), recursive);

150

151 return fileStatusRemoteIterator;

152 }

153

154

162 public static RemoteIterator listFiles(Configuration conf, String basePath) throws IOException {

163 FileSystem fs = FileSystem.get(conf);

164 RemoteIterator remoteIterator = fs.listFiles(new Path(basePath), false);

165 fs.close();

166 return remoteIterator;

167 }

168

169

177 public static FileStatus[] listStatus(Configuration conf, String dirPath) throws IOException {

178 FileSystem fs = FileSystem.get(conf);

179 FileStatus[] fileStatuses = fs.listStatus(new Path(dirPath));

180 fs.close();

181 return fileStatuses;

182 }

183

184

185

193 public static String readFile(Configuration conf, String filePath) throws IOException {

194 String fileContent = null;

195 FileSystem fs = FileSystem.get(conf);

196 Path path = new Path(filePath);

197 InputStream inputStream = null;

198 ByteArrayOutputStream outputStream = null;

199 try {

200 inputStream = fs.open(path);

201 outputStream = new ByteArrayOutputStream(inputStream.available());

202 IOUtils.copyBytes(inputStream, outputStream, conf);

203 fileContent = outputStream.toString();

204 } finally {

205 IOUtils.closeStream(inputStream);

206 IOUtils.closeStream(outputStream);

207 fs.close();

208 }

209 return fileContent;

210 }

211 }

简单的测试了一下:

1 @Test

2 public void test() throws IOException {

3 Configuration conf = new Configuration();

4 String newDir = "/test";

5 //01.检测路径是否存在测试

6 if (HDFSUtil.exits(conf, newDir)) {

7 System.out.println(newDir + " 已存在!");

8 } else {

9 //02.创建目录测试

10 boolean result = HDFSUtil.createDirectory(conf, newDir);

11 if (result) {

12 System.out.println(newDir + " 创建成功!");

13 } else {

14 System.out.println(newDir + " 创建失败!");

15 }

16 }

17 String fileContent = "Hi,hadoop. I love you";

18 String newFileName = newDir + "/myfile.txt";

19

20 //03.创建文件测试

21 HDFSUtil.createFile(conf, newFileName, fileContent);

22 System.out.println(newFileName + " 创建成功");

23

24 //04.读取文件内容测试

25 System.out.println(newFileName + " 的内容为:n" + HDFSUtil.readFile(conf, newFileName));

26

27 //05. 测试获取所有目录信息

28 FileStatus[] dirs = HDFSUtil.listStatus(conf, "/");

29 System.out.println("--根目录下的所有子目录---");

30 for (FileStatus s : dirs) {

31 System.out.println(s);

32 }

33

34 //06. 测试获取所有文件

35 FileSystem fs = FileSystem.get(conf);

36 RemoteIterator files = HDFSUtil.listFiles(fs, "/", true);

37 System.out.println("--根目录下的所有文件---");

38 while (files.hasNext()) {

39 System.out.println(files.next());

40 }

41 fs.close();

42

43 //删除文件测试

44 boolean isDeleted = HDFSUtil.deleteFile(conf, newDir);

45 System.out.println(newDir + " 已被删除");

46

47 }

注:测试时,不要忘记了在resources目录下放置core-site.xml文件,不然IDE环境下,代码不知道去连哪里的HDFS

输出结果：

/test 已存在!
/test/myfile.txt 创建成功
/test/myfile.txt 的内容为:
Hi,hadoop. I love you
--根目录下的所有子目录---
FileStatus{path=hdfs://172.28.20.102:9000/jimmy; isDirectory=true; modification_time=1432176691550; access_time=0; owner=hadoop; group=supergroup; permission=rwxrwxrwx; isSymlink=false}
FileStatus{path=hdfs://172.28.20.102:9000/test; isDirectory=true; modification_time=1432181331362; access_time=0; owner=jimmy; group=supergroup; permission=rwxr-xr-x; isSymlink=false}
FileStatus{path=hdfs://172.28.20.102:9000/user; isDirectory=true; modification_time=1431931797244; access_time=0; owner=hadoop; group=supergroup; permission=rwxr-xr-x; isSymlink=false}
--根目录下的所有文件---
LocatedFileStatus{path=hdfs://172.28.20.102:9000/jimmy/input/README.txt; isDirectory=false; length=1366; replication=1; blocksize=134217728; modification_time=1431922483851; access_time=1432174134018; owner=hadoop; group=supergroup; permission=rw-r--r--; isSymlink=false}
LocatedFileStatus{path=hdfs://172.28.20.102:9000/jimmy/output/_SUCCESS; isDirectory=false; length=0; replication=3; blocksize=134217728; modification_time=1432176692454; access_time=1432176692448; owner=jimmy; group=supergroup; permission=rw-r--r--; isSymlink=false}
LocatedFileStatus{path=hdfs://172.28.20.102:9000/jimmy/output/part-r-00000; isDirectory=false; length=1306; replication=3; blocksize=134217728; modification_time=1432176692338; access_time=1432176692182; owner=jimmy; group=supergroup; permission=rw-r--r--; isSymlink=false}
LocatedFileStatus{path=hdfs://172.28.20.102:9000/test/myfile.txt; isDirectory=false; length=21; replication=3; blocksize=134217728; modification_time=1432181331601; access_time=1432181331362; owner=jimmy; group=supergroup; permission=rw-r--r--; isSymlink=false}
/test 已被删除

用spring-mvc结合hdfs api仿造hadoop的文件浏览管理界面，做了一个山寨版：（只完成了文件列表功能）

源代码托管在taobao开源平台上了，有需要的可以参考下：

http://code.taobao.org/p/hdfs-web-client/src/trunk/

ITPub博客

hadoop: hdfs API示例

最新文章