Ubuntu16.04 下 Spark2.0.2+Hadoop2.7.3+Zookeeper3.4.9+HBase1.1.7集羣環境搭建--5

時間  2019-11-12
標籤 ubuntu16.04 ubuntu spark2.0.2+hadoop2.7.3+zookeeper3.4.9+hbase1.1.7 spark hadoop zookeeper hbase 集羣 環境 搭建 欄目 Ubuntu 简体版
原文   原文鏈接

接着1上篇：http://www.javashuo.com/article/p-swxkwhye-gs.htmljava

接着2上篇：http://www.javashuo.com/article/p-yuphtrbr-hc.htmlsql

接着3上篇：http://www.javashuo.com/article/p-xqxhmddb-hh.htmlapache

接着4上篇：https://my.oschina.net/u/988386/blog/802073windows

 

 

Eclipse遠程調試。api

環境信息 win10+eclipse neon+maven服務器

windows下要winutils.exe文件，在hadoop-common-bin中eclipse

下載地址（不是我傳的，要分和我不要緊）：maven

（http://download.csdn.net/detail/speedgoddeer/9347013）oop

  • 服務器上hbase-site.xml 文件要複製一份到eclipse的src目錄。

      eclipse 調試 會出現各類 jar包版本問題，注意查看log信息。url

        調試代碼以下：僅供參考

import java.util.ArrayList;
import java.util.List;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.protobuf.generated.ClientProtos;
import org.apache.hadoop.hbase.util.Base64;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaSparkContext;

public class SparkHBaseDemo {

	private static final String master = "spark://d155:7077";

	//hbase解壓目錄下的lib
	private static final String HBASE_PATH = "F:\\BACKUP_SOFTWARE\\hadoop\\hbase\\hbase-1.1.7\\lib\\";
	
	public static String[] sparkJars() {
		
		List<String> list = new ArrayList<String>();
		
		list.add(HBASE_PATH+"zookeeper-3.4.6.jar");
		list.add(HBASE_PATH+"hbase-common-1.1.7.jar");
		list.add(HBASE_PATH+"hbase-client-1.1.7.jar");
		list.add(HBASE_PATH+"hbase-server-1.1.7.jar");
		list.add(HBASE_PATH+"hbase-protocol-1.1.7.jar");
		list.add(HBASE_PATH+"htrace-core-3.1.0-incubating.jar");
		
		//當前程序jar
		list.add("D:\\WIBZPLACE7\\hdp\\target\\hdp-0.0.1-SNAPSHOT.jar");
		
		
		String[] array =new String[list.size()];
        list.toArray(array);
		

		return array;
	}

	public static void debug() {
		System.setProperty("hadoop.home.dir", "E:\\DEVELOP\\SOFTWARE\\hadoop-common-bin");

		SparkConf sparkConf = new SparkConf();
		sparkConf.setAppName("HBASE").setMaster(master);

		sparkConf.setJars(sparkJars());

		JavaSparkContext sc = new JavaSparkContext(sparkConf);

		Configuration conf = HBaseConfiguration.create();
		Scan scan = new Scan();
		scan.addFamily(Bytes.toBytes("liezu1"));

		try {
			String tableName = "tb_demo1";
			conf.set(TableInputFormat.INPUT_TABLE, tableName);
			ClientProtos.Scan proto = ProtobufUtil.toScan(scan);
			String ScanToString = Base64.encodeBytes(proto.toByteArray());
			conf.set(TableInputFormat.SCAN, ScanToString);

			JavaPairRDD<ImmutableBytesWritable, Result> myRDD = sc.newAPIHadoopRDD(conf, TableInputFormat.class,
					ImmutableBytesWritable.class, Result.class);

			System.out.println(myRDD.count());

		} catch (Exception ex) {
			ex.printStackTrace();
		} finally {
			sc.close();
		}

	}

	public static void main(String[] args) {
		debug();
	}
}
  • pom.xml文件
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<groupId>net.ibizsys.hdp</groupId>
	<artifactId>hdp</artifactId>
	<version>0.0.1-SNAPSHOT</version>
	<repositories>
		<repository>
			<id>cloudera-repo-releases</id>
			<name>Spark-Support</name>
			<url>http://maven.aliyun.com/nexus/content/groups/public/</url>
		</repository>
	</repositories>
	<properties>
		<jackson.version>2.6.7</jackson.version>
		<scala.binary.version>2.11</scala.binary.version>
		<spark.version>2.0.2</spark.version>
		<scala.lang.version>2.11.8</scala.lang.version>
		<hadoop.version>2.7.3</hadoop.version>
		<hbase.version>1.2.4</hbase.version>
	</properties>
	<dependencies>

		<dependency>
			<groupId>io.netty</groupId>
			<artifactId>netty-all</artifactId>
			<version>4.0.29.Final</version>
		</dependency>
		<!-- <dependency> <groupId>io.netty</groupId> <artifactId>netty</artifactId> 
			</dependency> -->
		<dependency>
			<groupId>jdk.tools</groupId>
			<artifactId>jdk.tools</artifactId>
			<version>1.7</version>
			<scope>system</scope>
			<systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-common</artifactId>
			<version>${hadoop.version}</version>
			<exclusions>
				<exclusion>
					<groupId>io.netty</groupId>
					<artifactId>netty-all</artifactId>
				</exclusion>
				<exclusion>
					<groupId>io.netty</groupId>
					<artifactId>netty</artifactId>
				</exclusion>
				<exclusion>
					<groupId>org.jboss.netty</groupId>
					<artifactId>netty</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-client</artifactId>
			<version>${hadoop.version}</version>
			<exclusions>
				<exclusion>
					<groupId>org.jboss.netty</groupId>
					<artifactId>netty</artifactId>
				</exclusion>
				<exclusion>
					<groupId>io.netty</groupId>
					<artifactId>netty-all</artifactId>
				</exclusion>
				<exclusion>
					<groupId>io.netty</groupId>
					<artifactId>netty</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-hdfs</artifactId>
			<version>${hadoop.version}</version>
			<exclusions>
				<exclusion>
					<groupId>org.jboss.netty</groupId>
					<artifactId>netty</artifactId>
				</exclusion>
				<exclusion>
					<groupId>io.netty</groupId>
					<artifactId>netty-all</artifactId>
				</exclusion>
				<exclusion>
					<groupId>io.netty</groupId>
					<artifactId>netty</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-hdfs</artifactId>
			<version>${hadoop.version}</version>
			<exclusions>
				<exclusion>
					<groupId>org.jboss.netty</groupId>
					<artifactId>netty</artifactId>
				</exclusion>
				<exclusion>
					<groupId>io.netty</groupId>
					<artifactId>netty-all</artifactId>
				</exclusion>
				<exclusion>
					<groupId>io.netty</groupId>
					<artifactId>netty</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
		<dependency>
			<groupId>org.apache.hadoop</groupId>
			<artifactId>hadoop-mapreduce-client-core</artifactId>
			<version>${hadoop.version}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hbase</groupId>
			<artifactId>hbase-client</artifactId>
			<version>${hbase.version}</version>
			<exclusions>
				<exclusion>
					<groupId>org.jboss.netty</groupId>
					<artifactId>netty</artifactId>
				</exclusion>
				<exclusion>
					<groupId>io.netty</groupId>
					<artifactId>netty-all</artifactId>
				</exclusion>
				<exclusion>
					<groupId>io.netty</groupId>
					<artifactId>netty</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
		<dependency>
			<groupId>org.apache.hbase</groupId>
			<artifactId>hbase-common</artifactId>
			<version>${hbase.version}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.hbase</groupId>
			<artifactId>hbase-server</artifactId>
			<version>${hbase.version}</version>
			<exclusions>
				<exclusion>
					<groupId>org.jboss.netty</groupId>
					<artifactId>netty</artifactId>
				</exclusion>
				<exclusion>
					<groupId>io.netty</groupId>
					<artifactId>netty-all</artifactId>
				</exclusion>
				<exclusion>
					<groupId>io.netty</groupId>
					<artifactId>netty</artifactId>
				</exclusion>
			</exclusions>
		</dependency>



		<dependency>
			<groupId>org.scala-lang</groupId>
			<artifactId>scala-library</artifactId>
			<version>${scala.lang.version}</version>
		</dependency>
		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-core_2.11</artifactId>
			<version>2.0.2</version>
			<exclusions>
				<exclusion>
					<groupId>org.jboss.netty</groupId>
					<artifactId>netty</artifactId>
				</exclusion>
				<exclusion>
					<groupId>io.netty</groupId>
					<artifactId>netty-all</artifactId>
				</exclusion>
				<exclusion>
					<groupId>io.netty</groupId>
					<artifactId>netty</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
		<dependency>
			<groupId>org.apache.spark</groupId>
			<artifactId>spark-sql_2.11</artifactId>
			<version>2.0.2</version>
			<exclusions>
				<exclusion>
					<groupId>org.jboss.netty</groupId>
					<artifactId>netty</artifactId>
				</exclusion>
				<exclusion>
					<groupId>io.netty</groupId>
					<artifactId>netty-all</artifactId>
				</exclusion>
				<exclusion>
					<groupId>io.netty</groupId>
					<artifactId>netty</artifactId>
				</exclusion>
			</exclusions>
		</dependency>
	</dependencies>
</project>
相關文章
相關標籤/搜索
每日一句
    每一个你不满意的现在，都有一个你没有努力的曾经。
本站公眾號
   歡迎關注本站公眾號,獲取更多信息
聯繫我們 最近搜索 最新文章 沪ICP备13005482号-10 MyBatis教程 SQL 教程 MySQL教程 Java 教程 Thymeleaf 教程 Hibernate教程 Spring教程 Redis教程