客戶端遠程訪問高可用（HA）hdfs

當hadoop namenode是HA集羣時，hdfs可能動態切換hdfs主節點，客戶端遠程訪問hdfs有兩種實現方法：java

方法1：配置多個hdfs地址，每次操做先判斷可用的hdfs地址。

形如：hdfs://192.168.2.102:9000,hdfs://192.168.2.101:9000，以逗號（,）隔開node

private void hdfsInit(String hdfs) { HdfsPath.setHdfs(hdfs); String[] pathes = HdfsPath.getHdfs().split(","); for(String path: pathes) { Configuration conf = new Configuration(); conf.set("fs.defaultFS", path); conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName()); try (FileSystem fs = FileSystem.newInstance(conf)) { fs.listStatus(new Path("/")); HdfsPath.setHdfs(path); return ; } catch (IOException e) { logger.warn(path+"拒絕鏈接；"+e); } } logger.error("hdfs配置錯誤！"+hdfs); }

方法2：將全部關於namenode的參數寫入Configuration對象中。代碼：apache

import java.io.BufferedInputStream; import java.io.FileInputStream; import java.io.InputStream; import java.io.OutputStream; import java.net.URI; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; public class Demo { private static void uploadToHdfs() throws Exception { //本地文件地址
String localSrc = "d:/PowerDesigner15_Evaluation.rar"; //存放在hdfs的目的地址
String dest = "/user/PowerDesigner15_Evaluation.rar"; InputStream in = new BufferedInputStream(new FileInputStream(localSrc)); //獲得配置對象
Configuration conf = new Configuration(); conf.set("fs.defaultFS", "hdfs://ns1"); conf.set("dfs.nameservices", "ns1"); conf.set("dfs.ha.namenodes.ns1", "nn1,nn2"); conf.set("dfs.namenode.rpc-address.ns1.nn1", "192.168.2.101:9000"); conf.set("dfs.namenode.rpc-address.ns1.nn2", "192.168.2.102:9000"); conf.set("dfs.client.failover.proxy.provider.ns1", "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"); //文件系統
FileSystem fs = FileSystem.get(new URI("hdfs://ns1"), conf, "hadoop"); //輸出流
OutputStream out = fs.create(new Path(dest)); //鏈接兩個流，造成通道，使輸入流向輸出流傳輸數據
IOUtils.copyBytes(in, out, 4096, true); } public static void main(String[] args) throws Exception{ // TODO Auto-generated method stub
uploadToHdfs(); } }