HdfsUtil 的 demo

HdfsUtil

package weekend101;



import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;

import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;


public class HdfsUtil {
	
	FileSystem fs = null;
	
	@Before
	public void init() throws Exception{
		
		//讀取classpath下的xxx-site.xml 配置文件，並解析其中內容，封裝到conf對象中
		Configuration conf = new Configuration();
		
		//也能夠在代碼中對 conf中的配置信息進行手動配置，會覆蓋掉配置文件中的讀取的值ֵ
		conf.set("fs.defaultFS", "hdfs://weekend101:9000/");
		
		//更具配置信息，去獲取一個具體文件系統的客戶端操做實例對象
		fs = FileSystem.get(new URI("hdfs://weekend101:9000/"),conf,"hadoop");
		
		
	}
	
	@Test
	public void upload() throws Exception{
		Configuration conf = new Configuration();
		conf.set("fs.defaultFS", "hdfs://weekend101:9000/");
		FileSystem fs = FileSystem.get(conf);
		
		Path dst = new Path("hdfs://weekend101:9000/aa/qingshu.txt");
		
		FSDataOutputStream os = fs.create(dst);
		
		FileInputStream is = new FileInputStream("d:/qingshu.txt");
		
		IOUtils.copy(is,os);
	}
	
public void upload2() throws Exception, IOException{
		
		fs.copyFromLocalFile(new Path("d:/qingshu.txt"), new Path("hdfs://weekend101:9000/aaa/bbb/ccc/qingshu2.txt"));
		
	}

/**
 * 下載
 * @throws Exception 
 * @throws IllegalArgumentException 
 */
@Test
public void download() throws Exception {
	
	fs.copyToLocalFile(new Path("hdfs://weekend101:9000/aa/qingshu2.txt"), new Path("c:/qingshu2.txt"));

}

/**
 * 建立文件夾
 * @throws Exception 
 * @throws IllegalArgumentException 
 */
@Test
public void mkdir() throws IllegalArgumentException, Exception {

	fs.mkdirs(new Path("/aaa/bbb/ccc"));
	
	
}

/**
 * 刪除文件
 * @throws IOException 
 * @throws IllegalArgumentException 
 */
@Test
public void rm() throws IllegalArgumentException, IOException {

	fs.delete(new Path("/aa"), true);
	
}

/**
 * 查看文件
 * @throws IOException 
 * @throws IllegalArgumentException 
 * @throws FileNotFoundException 
 * 
 */
@Test
public void listFiles() throws FileNotFoundException, IllegalArgumentException, IOException {

	// listFiles�г������ļ���Ϣ�������ṩ�ݹ����
	RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path("/"), true);
	
	while(files.hasNext()){
		
		LocatedFileStatus file = files.next();
		Path filePath = file.getPath();
		String fileName = filePath.getName();
		System.out.println(fileName);
		
	}
	
	System.out.println("---------------------------------");
	
	//listStatus 能夠列出文件和文件夾的信息，可是不提供自帶的遞歸遍歷
	FileStatus[] listStatus = fs.listStatus(new Path("/"));
	for(FileStatus status: listStatus){
		
		String name = status.getPath().getName();
		System.out.println(name + (status.isDirectory()?" is dir":" is file"));
		
	}
	
}

public static void main(String[] args) throws Exception {

	Configuration conf = new Configuration();
	conf.set("fs.defaultFS", "hdfs://weekend110:9000/");
	
	FileSystem fs = FileSystem.get(conf);
	
	FSDataInputStream is = fs.open(new Path("/jdk-7u65-linux-i586.tar.gz"));
	
	FileOutputStream os = new FileOutputStream("c:/jdk7.tgz");
	
	IOUtils.copy(is, os);
}

	
}