在前面《centos7 快速配置手册1~4》文档中，描述了在centos7服务器上部署OGG bd的初始化过程
本章节重点描述如何配置oracle GoldenGate for Bigdata -hdfs 增量数据同步，从oracle 到 hdfs

下面是前一章节的回顾：
hadoop目录
/usr/hdp/2.6.0.3-8/hadoop
ogg主目录
export OGG_HOME=/ogg/oggbd
# export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ORACLE_HOME/lib
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$OGG_HOME:$OGG_HOME/ggjava
----------------------------------------------------------
ogg配置文件中
#Sample gg.classpath for HDP 打开这个预置的设置
gg.classpath=/usr/hdp/current/hadoop-client/client/*:/etc/hadoop/conf
----------------------------------------------------------
ogg 初始化配置文件 (tt)
scp slave03:/ogg/oggbd/dirdat/initld ./
cd ..
./replicat paramfile dirprm/irhdfs.prm reportfile dirrpt/initld.rpt
cat dirrpt/initld.rpt 

ogg 初始化检查 (tt)
[root@slave02 dirprm]#  hdfs dfs -ls /ogg1
Found 1 items
drwxrwxrwx   - root hdfs          0 2017-07-21 17:38 /ogg1/bdtest.tt
[root@slave02 dirprm]#  hdfs dfs -ls /ogg1/bdtest.tt

beeline 连接方式
# beeline -u jdbc:hive2://slave02:10000
0: jdbc:hive2://slave02:10000> select * from bdtest.tt;
0: jdbc:hive2://slave02:10000> select count(*) from bdtest.tt; 共35条记录，要等待78秒
----------------------------------------------------------
ogg 初始化数据准备 (userinfo)
# scp slave03:/ogg/oggbd/dirdat/* ./

ogg 初始化配置文件准备和启动 (userinfo)
cat repfils.prm 
    REPLICAT REPFILS
    -- Trail file for this example is located in "AdapterExamples/trail" directory
    -- Command to add REPLICAT
    -- add replicat repfils, exttrail AdapterExamples/trail/tr
    TARGETDB LIBFILE libggjava.so SET property=dirprm/hdfs.props
    REPORTCOUNT EVERY 1 MINUTES, RATE
    GROUPTRANSOPS 10000
    MAP ggtest.*, TARGET bdtest.*;

add replicat repfils, exttrail /ogg/oggbd/dirdat/in
start repfils

124622273 导入完成。

导入后空间占用
    hdfs://hadoopHA/ogg1/bdtest.yth_client_userinfo 占用了多大磁盘空间？
    # hdfs dfs -du -s -h hdfs://hadoopHA/ogg1/bdtest.yth_client_userinfo
    51.9 G  hdfs://hadoopHA/ogg1/bdtest.yth_client_userinfo

添加外部表
create external table bdtest.userinfo(
  optype     VARCHAR(1),
  tabname    VARCHAR(26),
  date1      VARCHAR(26),
  date2      VARCHAR(26),
  id         VARCHAR(32),
  qyid       VARCHAR(32),
  khsh       VARCHAR(32),
  dlzh       VARCHAR(32),
  cps        VARCHAR(1024),
  khshs      VARCHAR(1024),
  khshs2     VARCHAR(1024), 
  clientver  VARCHAR(32),
  createtime string,
  clientip   VARCHAR(100),
  browser    VARCHAR(32),
  os         VARCHAR(32),
  memory     VARCHAR(32),
  clientid   VARCHAR(100)
)
stored as textfile location '/ogg1/bdtest.yth_client_userinfo';

查询外部表
select 
userinfo.khsh,
userinfo.clientver ,
userinfo.createtime,
userinfo.clientip,
userinfo.browser,
userinfo.os,
userinfo.memory 
from bdtest.userinfo
limit 2;

~~~~~~~~~~~~~~~~~~~~~~~~~~ 增量数据同步 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
参考《ogg-mongodb_00.docx》 文档

源端配置
vi /etc/hosts
...
10.3.254.52 slave02

su - oracle
cd /ogg/oggora/dirprm
vi ggext.prm
extract ggext
userid ogg@spark,password AACAAAAAAAAAAAHAYBGFCDZCJHWCEIHH, BLOWFISH, ENCRYPTKEY DEFAULT
FETCHOPTIONS FETCHPKUPDATECOLS
--RMTHOSTOPTIONS
RMTHOST slave02, MGRPORT 7839
RMTFILE /ogg/oggbd/dirdat/oa, MEGABYTES 2, PURGE
DDL include objname ggtest.*
TABLE ggtest.*;

dblogin userid ogg@spark,password htjsogg
delete extract ggext
unregister extract ggext database

register extract GGEXT database
add extract ggext, INTEGRATED TRANLOG, BEGIN NOW
源端配置完毕。

目标端配置
[root@slave02 dirprm]# cat hdfs.props 
gg.handlerlist=hdfs
gg.handler.hdfs.type=hdfs
gg.handler.hdfs.includeTokens=false
gg.handler.hdfs.maxFileSize=1g
gg.handler.hdfs.rootFilePath=/ogg1
gg.handler.hdfs.fileRollInterval=0
gg.handler.hdfs.inactivityRollInterval=0
gg.handler.hdfs.fileSuffix=.txt
gg.handler.hdfs.partitionByTable=true
gg.handler.hdfs.rollOnMetadataChange=true
gg.handler.hdfs.authType=none
gg.handler.hdfs.format=delimitedtext
# gg.handler.hdfs.format.includeColumnNames=true
gg.handler.hdfs.mode=tx
goldengate.userexit.timestamp=utc
goldengate.userexit.writers=javawriter
javawriter.stats.display=TRUE
javawriter.stats.full=TRUE
gg.log=log4j
gg.log.level=INFO
gg.report.time=30sec

#Sample gg.classpath for HDP
gg.classpath=/usr/hdp/current/hadoop-client/client/*:/etc/hadoop/conf
javawriter.bootoptions=-Xmx512m -Xms32m -Djava.class.path=ggjava/ggjava.jar

[root@slave02 dirprm]# cat rhdfs.prm
REPLICAT rhdfs
-- Trail file for this example is located in "AdapterExamples/trail" directory
-- Command to add REPLICAT
-- add replicat rhdfs, exttrail AdapterExamples/trail/tr
setenv HADOOP_COMMON_LIB_NATIVE_DIR=/usr/hdp/2.6.0.3-8/hadoop/lib/native
DDL include all
TARGETDB LIBFILE libggjava.so SET property=dirprm/hdfs.props
REPORTCOUNT EVERY 1 MINUTES, RATE
GROUPTRANSOPS 10000
MAP ggtest.*, TARGET bdtest.*;
目标端配置结束

源端启动进程
    start ggext

目标端添加进程,启动进程
    add replicat rhdfs, exttrail dirdat/oa
    start replicat rhdfs

源端测试表中添加一些数据
insert into tt (owner,table_name) values ('ww','tt01');
insert into tt (owner,table_name) values ('ww','tt02');
insert into tt (owner,table_name) values ('ww','tt03');
commit;

insert into yth_client_userinfo 
(id,qyid,khsh,dlzh,cps,khshs)
values ('1','1','1','1','1','1');
insert into yth_client_userinfo 
(id,qyid,khsh,dlzh,cps,khshs)
values ('2','2','2','2','2','2');
commit;

delete from tt where owner='ww' and table_name='tt03';
commit;


检查复制情况
源端检查
GGSCI (crmdb2 as ogg@spark) 13> stats ggext
目标端检查
GGSCI (slave02) 10> view report rhdfs
GGSCI (slave02) 11> stats rhdfs
hive中检查
select * from bdtest.tt;
select count(*) from bdtest.userinfo;

增量数据同步配置结束。