本地Java大數據環境基礎配置

創建項目

準備pom.xml文件配置(附在文檔最後)

在下載jar過程中極其緩慢,簡直不能忍受,查閱了下資料,得到下面的辦法:

1、 maven倉庫中沒有支持cdh的相關依賴。cloudera自己建立了一個相關的倉庫。要想利用maven添加相關依賴,則必須單獨添加cloudera倉庫

本地Java大數據環境基礎配置

2、 在國內使用maven中央倉庫一般會有網絡問題,大部分人會使用aliyun倉庫或者其他開源的倉庫。所以需要修改setting.xml (以下配置中 *,!cloudera 表示除了aliyun倉庫還使用cloudera倉庫)

配置:

<code>

<

mirror

>

        

<

id

>

nexus-aliyun

id

>

        

<

mirrorOf

>

*,!cloudera

mirrorOf

>

        

<

name

>

Nexus aliyun

name

>

                             

<

url

>

           http://maven.aliyun.com/nexus/content/groups/public         

url

>

mirror

>

/<code>
本地Java大數據環境基礎配置

配置環境

下面兩個配置表示是在本地運行還是到Linux環境中運行

dev

本地Java大數據環境基礎配置

編寫內容

本地Java大數據環境基礎配置

local

本地Java大數據環境基礎配置

編寫內容

本地Java大數據環境基礎配置

準備log4j的日誌配置,這個網上很多,可以直接複製一份,改下關鍵地方就可以

本地Java大數據環境基礎配置

HDFS連接地址的配置

本地Java大數據環境基礎配置

Hbase的連接配置

本地Java大數據環境基礎配置

配置文件transformer-env.xml

本地Java大數據環境基礎配置

文件內容

本地Java大數據環境基礎配置

本地Java大數據環境基礎配置


pom.xml文件配置

<code>

<

project

xmlns

=

"http://maven.apache.org/POM/4.0.0"

xmlns:xsi

=

"http://www.w3.org/2001/XMLSchema-instance"

xsi:schemaLocation

=

"http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"

>

 

<

modelVersion

>

4.0.0

modelVersion

>

 

<

groupId

>

com.xlgl.wzy

groupId

>

 

<

artifactId

>

localenv

artifactId

>

 

<

version

>

0.0.1-SNAPSHOT

version

>

  

<

properties

>

   

<

hadoop.version

>

2.7.3

hadoop.version

>

   

<

hbase.version

>

1.2.5

hbase.version

>

   

<

hive.version

>

2.1.1

hive.version

>

   

<

mysql.version

>

5.1.6

mysql.version

>

   

<

gson.version

>

2.6.2

gson.version

>

   

<

uasparser.version

>

0.6.1

uasparser.version

>

 

properties

>

          

<

repositories

>

                                   

<

repository

>

                         

<

id

>

aliyun

id

>

                         

<

url

>

http://maven.aliyun.com/nexus/content/groups/public/

url

>

                 

repository

>

                 

<

repository

>

                         

<

id

>

cloudera

id

>

                         

<

url

>

https://repository.cloudera.com/artifactory/cloudera-repos/

url

>

                 

repository

>

                 

<

repository

>

                         

<

id

>

jboss

id

>

                         

<

url

>

http://repository.jboss.org/nexus/content/groups/public

url

>

                 

repository

>

        

repositories

>

   

<

dependencies

>

   

<

dependency

>

     

<

groupId

>

org.apache.hadoop

groupId

>

     

<

artifactId

>

hadoop-client

artifactId

>

     

<

version

>

${hadoop.version}

version

>

   

dependency

>

     

<

dependency

>

     

<

groupId

>

org.apache.hbase

groupId

>

     

<

artifactId

>

hbase-client

artifactId

>

     

<

version

>

${hbase.version}

version

>

   

dependency

>

     

<

dependency

>

     

<

groupId

>

org.apache.hbase

groupId

>

     

<

artifactId

>

hbase-server

artifactId

>

     

<

version

>

${hbase.version}

version

>

   

dependency

>

     

<

dependency

>

     

<

groupId

>

org.apache.hive

groupId

>

     

<

artifactId

>

hive-exec

artifactId

>

     

<

version

>

${hive.version}

version

>

   

dependency

>

         

<

dependency

>

     

<

groupId

>

mysql

groupId

>

     

<

artifactId

>

mysql-connector-java

artifactId

>

     

<

version

>

${mysql.version}

version

>

   

dependency

>

         

<

dependency

>

     

<

groupId

>

cz.mallat.uasparser

groupId

>

     

<

artifactId

>

uasparser

artifactId

>

     

<

version

>

${uasparser.version}

version

>

   

dependency

>

     

<

dependency

>

     

<

groupId

>

com.google.code.gson

groupId

>

     

<

artifactId

>

gson

artifactId

>

     

<

version

>

${gson.version}

version

>

   

dependency

>

 

dependencies

>

   

<

profiles

>

       

<

profile

>

           

<

id

>

local

id

>

     

<

activation

>

       

<

activeByDefault

>

true

activeByDefault

>

     

activation

>

     

<

build

>

       

<

resources

>

               

<

resource

>

                        

<

directory

>

configuration/local

directory

>

               

resource

>

               

<

resource

>

                        

<

directory

>

src/main/resources

directory

>

               

resource

>

       

resources

>

       

<

plugins

>

         

<

plugin

>

                         

<

groupId

>

org.codehaus.mojo

groupId

>

           

<

artifactId

>

build-helper-maven-plugin

artifactId

>

           

<

version

>

1.4

version

>

           

<

executions

>

             

<

execution

>

               

<

id

>

add-source

id

>

                 

<

phase

>

generate-sources

phase

>

                 

<

goals

>

                   

<

goal

>

add-source

goal

>

                 

goals

>

                 

<

configuration

>

                   

<

sources

>

                     

<

source

>

${basedir}/src/main/java

source

>

                   

<

source

>

${basedir}/src/main/extr

source

>

                 

sources

>

               

configuration

>

               

execution

>

             

executions

>

           

plugin

>

       

plugins

>

     

build

>

   

profile

>

     

<

profile

>

           

<

id

>

dev

id

>

     

<

build

>

       

<

resources

>

               

<

resource

>

                        

<

directory

>

configuration/dev

directory

>

               

resource

>

               

<

resource

>

                        

<

directory

>

src/main/resources

directory

>

               

resource

>

       

resources

>

        

<

plugins

>

         

<

plugin

>

             

<

groupId

>

org.codehaus.mojo

groupId

>

           

<

artifactId

>

build-helper-maven-plugin

artifactId

>

           

<

version

>

1.4

version

>

           

<

executions

>

             

<

execution

>

               

<

id

>

add-source

id

>

                 

<

phase

>

generate-sources

phase

>

                 

<

goals

>

                   

<

goal

>

add-source

goal

>

                 

goals

>

                 

<

configuration

>

                   

<

sources

>

                     

<

source

>

${basedir}/src/main/java

source

>

                 

sources

>

               

configuration

>

               

execution

>

             

executions

>

           

plugin

>

           

<

plugin

>

                       

<

groupId

>

org.apache.maven.plugins

groupId

>

           

<

artifactId

>

maven-shade-plugin

artifactId

>

           

<

version

>

2.1

version

>

           

<

executions

>

             

<

execution

>

               

<

phase

>

package

phase

>

               

<

goals

>

                 

<

goal

>

shade

goal

>

               

goals

>

               

<

configuration

>

                 

<

artifactSet

>

                   

<

includes

>

                     

<

include

>

cz.mallat.uasparser:uasparser

include

>

                     

<

include

>

net.sourceforge.jregex:jregex

include

>

                     

<

include

>

mysql:mysql-connector-java

include

>

                   

includes

>

                 

artifactSet

>

               

configuration

>

             

execution

>

           

executions

>

         

plugin

>

                

plugins

>

     

build

>

   

profile

>

 

profiles

>

   

<

build

>

   

<

testSourceDirectory

>

src/test/java

testSourceDirectory

>

   

<

plugins

>

   

<

plugin

>

               

<

artifactId

>

maven-assembly-plugin

artifactId

>

               

<

configuration

>

                   

<

descriptorRefs

>

                       

<

descriptorRef

>

jar-with-dependencies

descriptorRef

>

                   

descriptorRefs

>

               

configuration

>

               

<

executions

>

                   

<

execution

>

                       

<

id

>

make-assembly

id

>

                       

<

phase

>

package

phase

>

                       

<

goals

>

                           

<

goal

>

assembly

goal

>

                       

goals

>

                    

execution

>

               

executions

>

         

plugin

>

     

<

plugin

>

       

<

artifactId

>

maven-compiler-plugin

artifactId

>

       

<

version

>

3.3

version

>

       

<

configuration

>

         

<

source

>

1.8

source

>

         

<

target

>

1.8

target

>

         

<

encoding

>

utf8

encoding

>

       

configuration

>

     

plugin

>

   

plugins

>

   

<

pluginManagement

>

            

<

plugins

>

                                         

<

plugin

>

                             

<

groupId

>

org.eclipse.m2e

groupId

>

                             

<

artifactId

>

lifecycle-mapping

artifactId

>

                             

<

version

>

1.0.0

version

>

                             

<

configuration

>

                                      

<

lifecycleMappingMetadata

>

                                               

<

pluginExecutions

>

                                                        

<

pluginExecution

>

                                                                

<

pluginExecutionFilter

>

                                                                         

<

groupId

>

org.codehaus.mojo

groupId

>

                                                                         

<

artifactId

>

                                                                                   build-helper-maven-plugin                                                                          

artifactId

>

                                                                         

<

versionRange

>

[1.4,)

versionRange

>

                                                                         

<

goals

>

                                                                                  

<

goal

>

add-source

goal

>

                                                                         

goals

>

                                                                

pluginExecutionFilter

>

                                                                

<

action

>

                                                                         

<

ignore

>

ignore

>

                                                                

action

>

                                                       

pluginExecution

>

                                               

pluginExecutions

>

                                      

lifecycleMappingMetadata

>

                             

configuration

>

                    

plugin

>

            

plugins

>

   

pluginManagement

>

 

build

>

 /<code>


分享到:


相關文章: