hadoop之旅5-idea經過maven搭建hdfs環境

相信你們經過以前的作法都已經搭建起了一個hadoop的開發環境。今天帶你們經過java api來訪問hdfs文件系統

首先啓動hadoop集羣

start-dfs.sh
或者
start-all.sh  //一鍵啓動hadoop集羣和yarn集羣
複製代碼

打開idea

在pom.xml文件里加入hadoop的依賴，我這裏使用的是我搭建的同樣版本的依賴 hadoop 2.7.3

<properties>
	<hadoop.version>2.7.3</hadoop.version>
</properties>

<dependency>
	<groupId>org.apache.hadoop</groupId>
	<artifactId>hadoop-common</artifactId>
	<version>${hadoop.version}</version>
</dependency>
<dependency>
	<groupId>org.apache.hadoop</groupId>
	<artifactId>hadoop-client</artifactId>	
	<version>${hadoop.version}</version>
</dependency>
<dependency>
        <groupId>org.apache.hadoop</groupId>
	<artifactId>hadoop-hdfs</artifactId>
	<version>${hadoop.version}</version>
</dependency>
複製代碼

Java api在hdfs上建立一個文件目錄

//建立配置文件
Configuration conf = new Configuration();
 //windows下沒法找到對應的環境變量，須要設置。把hadoop解壓下來的根目錄
System.setProperty("hadoop.home.dir", "F:\\linux\\hadoop-2.7.3");
// 指定hadoop fs的地址
conf.set("fs.defaultFS", "hdfs://master:9000");
//定義訪問的根目錄
String userRootPath = "/userSpace"
//拿到文件操做對象
FileSystem fs = FileSystem.get(conf);
//建立一個path對象，hdfs上的目錄都須要用path對象來訪問
Path dir = new Path(userRootPath);  //表示根目錄下的 userSpace目錄
boolean result = fs.mkdirs(dir);
if(result){
    System.out.println("建立目錄成功！");
}
複製代碼

其實就和java訪問文件同樣的操做相似，很是簡單，不過操做hdfs主要是經過FileSystem類。下面給你們貼一下完整代碼

package com.mmcc.springboothadoop.hadoop;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;

import java.io.ByteArrayOutputStream;
import java.io.IOException;

public class HdfsUtils {
    private static Configuration conf = new Configuration();
    public static String userRootPath = "/userSpace";

    static {
        //windows下沒法找到對應的環境變量，須要設置
        System.setProperty("hadoop.home.dir", "F:\\linux\\hadoop-2.7.3");
        // 指定hadoop fs的地址
        conf.set("fs.defaultFS", "hdfs://master:9000");
    }

    //判斷路徑是否存在
    public static boolean exists(String path) throws IOException {
        FileSystem fileSystem = FileSystem.get(conf);
        return fileSystem.exists(new Path(path));
    }

    //建立文件
    public static void createFile(String filePath, byte[] contents) throws IOException {
        FileSystem fileSystem = FileSystem.get(conf);
        Path path = new Path(filePath);
        FSDataOutputStream fdo = fileSystem.create(path);
        fdo.write(contents);
        fdo.close();
        fileSystem.close();

    }

    /**
     * 建立文件
     *
     * @param filePath
     * @param fileContent
     * @throws IOException
     */
    public static void createFile(String filePath, String fileContent)
            throws IOException {
        createFile(filePath, fileContent.getBytes());
    }

    //從本地複製到hdfs上
    public static void copyFromLocalFile(String localFilePath, String remoteFilePath) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        Path localPath = new Path(localFilePath);
        Path remotePath = new Path(remoteFilePath);
        fs.copyFromLocalFile(false, true, localPath, remotePath);
        fs.close();
    }


    /**
     * 刪除目錄或文件
     *
     * @param remoteFilePath
     * @param recursive
     * @return
     * @throws IOException
     */
    public static boolean deleteFile(String remoteFilePath, boolean recursive)
            throws IOException {
        FileSystem fs = FileSystem.get(conf);
        boolean result = fs.delete(new Path(remoteFilePath), recursive);
        fs.close();
        return result;
    }
    /**
     * 刪除目錄或文件(若是有子目錄,則級聯刪除)
     *
     * @param remoteFilePath
     * @return
     * @throws IOException
     */
    public static boolean deleteFile(String remoteFilePath) throws IOException {
        return deleteFile(remoteFilePath, true);
    }
    /**
     * 文件重命名
     *
     * @param oldFileName
     * @param newFileName
     * @return
     * @throws IOException
     */
    public static boolean renameFile(String oldFileName, String newFileName)
            throws IOException {
        FileSystem fs = FileSystem.get(conf);
        Path oldPath = new Path(oldFileName);
        Path newPath = new Path(newFileName);
        boolean result = fs.rename(oldPath, newPath);
        fs.close();
        return result;
    }
    /**
     * 建立目錄
     *
     * @param dirName
     * @return
     * @throws IOException
     */
    public static boolean createDirectory(String dirName) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        Path dir = new Path(dirName);
        boolean result = false;
        if (!fs.exists(dir)) {
            result = fs.mkdirs(dir);
        }
        fs.close();
        return result;
    }

    //列出指定路徑下的文件
    public static  RemoteIterator<LocatedFileStatus> listFiles(String dirPath,boolean recursive) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        RemoteIterator<LocatedFileStatus> listFiles = fs.listFiles(new Path(dirPath), recursive);//不進行遞歸
        fs.close();
        return listFiles;
    }
    /**
     * 列出指定路徑下的文件（非遞歸）
     *
     * @param basePath
     * @return
     * @throws IOException
     */
    public static RemoteIterator<LocatedFileStatus> listFiles(String basePath)
            throws IOException {
        FileSystem fs = FileSystem.get(conf);
        RemoteIterator<LocatedFileStatus> remoteIterator = fs.listFiles(
                new Path(basePath), false);
        fs.close();
        return remoteIterator;
    }

    /**
     * 列出指定目錄下的文件\子目錄信息（非遞歸）
     *
     * @param dirPath
     * @return
     * @throws IOException
     */
    public static FileStatus[] listStatus(String dirPath) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        FileStatus[] fileStatuses = fs.listStatus(new Path(dirPath));
        fs.close();
        return fileStatuses;
    }

    //讀取文件內容
    public static byte[] readFile(String filePath) throws IOException {
        byte[] fileContent = null;
        FileSystem fs = FileSystem.get(conf);
        Path path = new Path(filePath);

        if (fs.exists(path)){
            FSDataInputStream fsin = fs.open(path);
            ByteArrayOutputStream bos = new ByteArrayOutputStream();
            IOUtils.copyBytes(fsin,bos,conf);
            fileContent = bos.toByteArray();
        }
        return fileContent;
    }

    //下載hdfs上的文件
    public static void download(String remote,String local) throws IOException {
        FileSystem fs = FileSystem.get(conf);
        //遠程hdfs上的文件
        Path remotePath = new Path(remote);
        //本地的文件
        Path localPath = new Path(local);
        fs.copyToLocalFile(remotePath,localPath);
        fs.close();
    }
}
複製代碼