創建項目
準備pom.xml文件配置(附在文檔最後)
在下載jar過程中極其緩慢,簡直不能忍受,查閱了下資料,得到下面的辦法:
1、 maven倉庫中沒有支持cdh的相關依賴。cloudera自己建立了一個相關的倉庫。要想利用maven添加相關依賴,則必須單獨添加cloudera倉庫
2、 在國內使用maven中央倉庫一般會有網絡問題,大部分人會使用aliyun倉庫或者其他開源的倉庫。所以需要修改setting.xml (以下配置中 *,!cloudera 表示除了aliyun倉庫還使用cloudera倉庫)
配置:
<code><
mirror
><
id
>nexus-aliyunid
><
mirrorOf
>*,!clouderamirrorOf
><
name
>Nexus aliyunname
><
url
> http://maven.aliyun.com/nexus/content/groups/publicurl
>mirror
>/<code>
配置環境
下面兩個配置表示是在本地運行還是到Linux環境中運行
dev
編寫內容
local
編寫內容
準備log4j的日誌配置,這個網上很多,可以直接複製一份,改下關鍵地方就可以
HDFS連接地址的配置
Hbase的連接配置
配置文件transformer-env.xml
文件內容
pom.xml文件配置
<code><
project
xmlns
="http://maven.apache.org/POM/4.0.0"
xmlns:xsi
="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation
="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
><
modelVersion
>4.0.0modelVersion
><
groupId
>com.xlgl.wzygroupId
><
artifactId
>localenvartifactId
><
version
>0.0.1-SNAPSHOTversion
><
properties
><
hadoop.version
>2.7.3hadoop.version
><
hbase.version
>1.2.5hbase.version
><
hive.version
>2.1.1hive.version
><
mysql.version
>5.1.6mysql.version
><
gson.version
>2.6.2gson.version
><
uasparser.version
>0.6.1uasparser.version
>properties
><
repositories
><
repository
><
id
>aliyunid
><
url
>http://maven.aliyun.com/nexus/content/groups/public/url
>repository
><
repository
><
id
>clouderaid
><
url
>https://repository.cloudera.com/artifactory/cloudera-repos/url
>repository
><
repository
><
id
>jbossid
><
url
>http://repository.jboss.org/nexus/content/groups/publicurl
>repository
>repositories
><
dependencies
><
dependency
><
groupId
>org.apache.hadoopgroupId
><
artifactId
>hadoop-clientartifactId
><
version
>${hadoop.version}version
>dependency
><
dependency
><
groupId
>org.apache.hbasegroupId
><
artifactId
>hbase-clientartifactId
><
version
>${hbase.version}version
>dependency
><
dependency
><
groupId
> org.apache.hbasegroupId
><
artifactId
>hbase-serverartifactId
><
version
>${hbase.version}version
>dependency
><
dependency
><
groupId
>org.apache.hivegroupId
><
artifactId
>hive-execartifactId
><
version
>${hive.version}version
>dependency
><
dependency
><
groupId
>mysqlgroupId
><
artifactId
>mysql-connector-javaartifactId
><
version
>${mysql.version}version
>dependency
><
dependency
><
groupId
>cz.mallat.uasparsergroupId
><
artifactId
>uasparserartifactId
><
version
>${uasparser.version}version
>dependency
><
dependency
><
groupId
>com.google.code.gsongroupId
><
artifactId
>gsonartifactId
><
version
>${gson.version}version
>dependency
>dependencies
><
profiles
><
profile
><
id
>localid
><
activation
><
activeByDefault
>trueactiveByDefault
>activation
><
build
><
resources
><
resource
><
directory
>configuration/localdirectory
>resource
><
resource
><
directory
>src/main/resourcesdirectory
>resource
>resources
><
plugins
><
plugin
><
groupId
>org.codehaus.mojogroupId
><
artifactId
>build-helper-maven-pluginartifactId
><
version
>1.4version
><
executions
><
execution
><
id
>add-sourceid
><
phase
>generate-sourcesphase
><
goals
><
goal
>add-sourcegoal
>goals
><
configuration
><
sources
><
source
>${basedir}/src/main/javasource
><
source
>${basedir}/src/main/extrsource
>sources
>configuration
>execution
>executions
>plugin
>plugins
>build
>profile
><
profile
><
id
>devid
><
build
><
resources
><
resource
><
directory
>configuration/devdirectory
>resource
><
resource
><
directory
>src/main/resourcesdirectory
>resource
>resources
><
plugins
><
plugin
><
groupId
>org.codehaus.mojogroupId
><
artifactId
>build-helper-maven-pluginartifactId
><
version
>1.4version
><
executions
><
execution
><
id
>add-sourceid
><
phase
>generate-sourcesphase
><
goals
><
goal
>add-sourcegoal
>goals
><
configuration
><
sources
><
source
>${basedir}/src/main/javasource
>sources
>configuration
>execution
>executions
>plugin
><
plugin
><
groupId
> org.apache.maven.pluginsgroupId
><
artifactId
>maven-shade-pluginartifactId
><
version
>2.1version
><
executions
><
execution
><
phase
>packagephase
><
goals
><
goal
>shadegoal
>goals
><
configuration
><
artifactSet
><
includes
><
include
>cz.mallat.uasparser:uasparserinclude
><
include
>net.sourceforge.jregex:jregexinclude
><
include
>mysql:mysql-connector-javainclude
>includes
>artifactSet
>configuration
>execution
>executions
>plugin
>plugins
>build
>profile
>profiles
><
build
><
testSourceDirectory
>src/test/javatestSourceDirectory
><
plugins
><
plugin
><
artifactId
>maven-assembly-pluginartifactId
><
configuration
><
descriptorRefs
><
descriptorRef
>jar-with-dependenciesdescriptorRef
>descriptorRefs
>configuration
><
executions
><
execution
><
id
>make-assemblyid
><
phase
>packagephase
><
goals
><
goal
>assemblygoal
>goals
>execution
>executions
>plugin
><
plugin
><
artifactId
>maven-compiler-pluginartifactId
><
version
>3.3version
><
configuration
><
source
>1.8source
><
target
>1.8target
><
encoding
>utf8encoding
>configuration
>plugin
>plugins
><
pluginManagement
><
plugins
><
plugin
><
groupId
>org.eclipse.m2egroupId
><
artifactId
>lifecycle-mappingartifactId
><
version
>1.0.0version
><
configuration
><
lifecycleMappingMetadata
><
pluginExecutions
><
pluginExecution
><
pluginExecutionFilter
><
groupId
>org.codehaus.mojo
groupId
><
artifactId
> build-helper-maven-pluginartifactId
><
versionRange
>[1.4,)versionRange
><
goals
><
goal
>add-sourcegoal
>goals
>pluginExecutionFilter
><
action
><
ignore
>ignore
>action
>pluginExecution
>pluginExecutions
>lifecycleMappingMetadata
>configuration
>plugin
>plugins
>pluginManagement
>build
> /<code>