1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
| /**
* 查看HDFS内容
*/
@Test
public void text()throws Exception {
FSDataInputStream in = fileSystem.open(new Path("/cdh_version.properties"));
IOUtils.copyBytes(in, System.out, 1024);
}
/**
* 创建文件
*/
@Test
public void create()throws Exception {
FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/b.txt"));
out.writeUTF("hello hdfs: replication 1");
out.flush();
out.close();
}
/**
* 测试文件名更改
* @throws Exception
*/
@Test
public void rename() throws Exception {
Path oldPath = new Path("/hdfsapi/test/b.txt");
Path newPath = new Path("/hdfsapi/test/c.txt");
boolean result = fileSystem.rename(oldPath, newPath);
System.out.println(result);
}
/**
* 拷贝本地文件到HDFS文件系统
*/
@Test
public void copyFromLocalFile() throws Exception {
Path src = new Path("/Users/rocky/data/hello.txt");
Path dst = new Path("/hdfsapi/test/");
fileSystem.copyFromLocalFile(src,dst);
}
/**
* 拷贝大文件到HDFS文件系统：带进度
*/
@Test
public void copyFromLocalBigFile() throws Exception {
InputStream in = new BufferedInputStream(new FileInputStream(new File("/Users/rocky/tmp/software/jdk-8u91-linux-x64.tar.gz")));
FSDataOutputStream out = fileSystem.create(new Path("/hdfsapi/test/jdk.tgz"),
new Progressable() {
public void progress() {
System.out.print(".");
}
});
IOUtils.copyBytes(in, out ,4096);
}
/**
* 拷贝HDFS文件到本地：下载
*/
@Test
public void copyToLocalFile() throws Exception {
Path src = new Path("/hdfsapi/test/hello.txt");
Path dst = new Path("/Users/rocky/tmp/software");
fileSystem.copyToLocalFile(src, dst);
}
/**
* 查看目标文件夹下的所有文件
*/
@Test
public void listFiles() throws Exception {
FileStatus[] statuses = fileSystem.listStatus(new Path("/hdfsapi/test"));
for(FileStatus file : statuses) {
String isDir = file.isDirectory() ? "文件夹" : "文件";
String permission = file.getPermission().toString();
short replication = file.getReplication();
long length = file.getLen();
String path = file.getPath().toString();
System.out.println(isDir + "\t" + permission
+ "\t" + replication + "\t" + length
+ "\t" + path
);
}
}
/**
* 递归查看目标文件夹下的所有文件
*/
@Test
public void listFilesRecursive() throws Exception {
RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(new Path("/hdfsapi/test"), true);
while (files.hasNext()) {
LocatedFileStatus file = files.next();
String isDir = file.isDirectory() ? "文件夹" : "文件";
String permission = file.getPermission().toString();
short replication = file.getReplication();
long length = file.getLen();
String path = file.getPath().toString();
System.out.println(isDir + "\t" + permission
+ "\t" + replication + "\t" + length
+ "\t" + path
);
}
}
/**
* 查看文件块信息
*/
@Test
public void getFileBlockLocations() throws Exception {
FileStatus fileStatus = fileSystem.getFileStatus(new Path("/hdfsapi/test/jdk.tgz"));
BlockLocation[] blocks = fileSystem.getFileBlockLocations(fileStatus,0,fileStatus.getLen());
for(BlockLocation block : blocks) {
for(String name: block.getNames()) {
System.out.println(name +" : " + block.getOffset() + " : " + block.getLength() + " : " + block.getHosts());
}
}
}
/**
* 删除文件
*/
@Test
public void delete() throws Exception {
boolean result = fileSystem.delete(new Path("/hdfsapi/test/jdk.tgz"), true);
System.out.println(result);
}
@Test
public void testReplication() {
System.out.println(configuration.get("dfs.replication"));
}
|
