【Hadoop】- HDFS API 基本用法

HDFS: Hadoop設計的一種分佈式文件系統，本文主要介紹hdfs文件系統的JavaAPI的使用

①：公共代碼：設置hdfs的訪問接口

private static String hdfsPath = "hdfs://192.168.174.131:9090";
private static Configuration conf = new Configuration();

②：獲取HDFS分佈式文件系統內部的DataNode節點信息

public static DatanodeInfo[] getDataNodeInfos(){
		FileSystem hdfs = null;
		DatanodeInfo[] datanodeInfos= null;
		try {
			hdfs = FileSystem.get(URI.create(hdfsPath),conf);
			DistributedFileSystem dbfs = (DistributedFileSystem) hdfs;
			datanodeInfos = dbfs.getDataNodeStats();
			hdfs.close();
		} catch (IOException e) {
			return null;
		}
		return datanodeInfos;
	}

③：判斷HDFS是否存儲目標文件

public static boolean judgeFileExist(String hdfsFile){
		FileSystem hdfs = null;
        boolean isSuccess = false;
		try {
			hdfs = FileSystem.get(URI.create(hdfsPath),conf);
			isSuccess = hdfs.exists(new Path(hdfsFile));
			hdfs.close();
		} catch (IOException e) {
			return false;
		}
		return isSuccess;
	}

④：獲取指定HDFS目錄下的文件列表信息

public static FileStatus[] getFilesByDir(String hdfsFileDir){
		FileSystem hdfs = null;
        FileStatus[] fileStatus = null;
		try {
			hdfs = FileSystem.get(URI.create(hdfsPath),conf);
			fileStatus = hdfs.listStatus(new Path(hdfsFileDir));
			hdfs.close();
		} catch (IOException e) {
			return null;
		}
		return fileStatus;
	}

⑤：HDFS文件系統中建立目錄

public static boolean mkdirFromHdfs(String hdfsFileDir){
    	
        FileSystem hdfs = null;
        boolean isSuccess = false;
		try {
			hdfs = FileSystem.get(URI.create(hdfsPath),conf);
			isSuccess = hdfs.mkdirs(new Path(hdfsFileDir));
			hdfs.close();
		} catch (IOException e) {
			return false;
		}
		return isSuccess;
    }

⑥：將文本信息寫入HDFS的指定文件

public static boolean writeInfoToHdfsFile(String dest,byte[] content){
		
		    FileSystem hdfs = null;
		    FSDataOutputStream fsDataOutputStream = null;
		    
	        try {
	        	hdfs = FileSystem.get(URI.create(hdfsPath),conf);
	        	fsDataOutputStream = hdfs.create(new Path(dest));
	        	fsDataOutputStream.write(content);
	        	fsDataOutputStream.flush();
	        	fsDataOutputStream.close();
			    hdfs.close();
			} catch (IOException e) {
				return false;
			}
	        return true;
	}

⑦：讀取HDFS文件系統文本文件的內容

public static String readFile(String hdfsFilePath) throws IOException{
    	
    	FileSystem hdfs = null;
    	InputStream is = null;
    	StringBuffer sb = new StringBuffer();
        try {
        	hdfs = FileSystem.get(URI.create(hdfsPath),conf);
            is = hdfs.open(new Path(hdfsFilePath));
            int length = 0;
            byte[] buf = new byte[1024];
            while((length=is.read(buf))!=-1){
            	sb.append(new String(buf,0,length));
            }
        } finally {
            IOUtils.closeStream(is);
            hdfs.close();
        }
        return sb.toString();
    }

⑧：刪除HDFS指定文件

public static boolean deleteHdfsFile(String hdfsFilePath){
    	
        FileSystem hdfs = null;
        boolean isSuccess = false;
		try {
			hdfs = FileSystem.get(URI.create(hdfsPath),conf);
			
			//hdfs.delete(path,true)  遞歸刪除
			isSuccess = hdfs.deleteOnExit(new Path(hdfsFilePath));
		    hdfs.close();
		} catch (IOException e) {
			return false;
		}
		return isSuccess;
    }

⑨：重命名HDFS文件

public static boolean renameHdfsFile(String oldName,String newName){
		
	        FileSystem hdfs = null;
	        boolean isSuccess = false;
			try {
				hdfs = FileSystem.get(URI.create(hdfsPath),conf);
				isSuccess = hdfs.rename(new Path(oldName), new Path(newName));
			    hdfs.close();
			} catch (IOException e) {
				return false;
			}
	       return isSuccess;
	}

⑩：上傳本地文件到HDFS文件系統

public static boolean uploadLocalFileToHDFS(boolean delSrc,boolean override,String src,String dest){

		FileSystem hdfs = null;
		try {
			hdfs = FileSystem.get(URI.create(hdfsPath),conf);
			//注意：目標地址能夠寫全路徑，若是不寫則默認在當前訪問的用戶主目錄下操做
			hdfs.copyFromLocalFile(delSrc,override,new Path(src), new Path(dest));
			hdfs.close();
		} catch (IOException e) {
			return false;
		}
		return true;
	}

注意：可能出現「Permission Denied」等錯誤信息，設置hadoop目錄的訪問權限便可：hadoop dfs -chmod -R 777 hadoop文件路徑 (例)