ITPub博客

首页 > 大数据 > Hadoop > flume线上配置

flume线上配置

原创 Hadoop 作者:lcz393537415 时间:2018-08-02 16:25:41 0 删除 编辑

hdfs_agent.sources = r1

hdfs_agent.sinks = k1

hdfs_agent.channels = c1


# Describe/configure the source

hdfs_agent.sources.r1.type = org.apache.flume.source.kafka.KafkaSource

#hdfs_agent.sources.r1.zookeeperConnect = 10.69.40.17:2181,10.69.40.12:2181,10.69.40.7:2181,10.69.40.11:2181,10.69.40.16:2181

#hdfs_agent.sources.r1.groupId = online_kafka_to_hdfs_gk_groupid_000001

#hdfs_agent.sources.r1.topic = IRT_Dev_Status_STD

hdfs_agent.sources.r1.kafka.bootstrap.servers = 10.69.40.10:9092,10.69.40.13:9092,10.69.40.14:9092,10.69.40.15:9092

hdfs_agent.sources.r1.kafka.consumer.group.id = online_kafka_to_hdfs_gk_groupid_000001

hdfs_agent.sources.r1.kafka.topics = workcondition_info_flume

hdfs_agent.sources.r1.batchSize = 2000

hdfs_agent.sources.r1.interceptors = i1

hdfs_agent.sources.r1.interceptors.i1.type = host

# Describe the sink

hdfs_agent.sinks.k1.type = hdfs

hdfs_agent.sinks.k1.hdfs.path = hdfs://rootcloud/ods/%Y%m%d/%H

hdfs_agent.sinks.k1.hdfs.filePrefix = log-%{host}

hdfs_agent.sinks.k1.hdfs.fileType = DataStream

hdfs_agent.sinks.k1.hdfs.rollInterval = 3600

hdfs_agent.sinks.k1.hdfs.rollSize = 104857600

hdfs_agent.sinks.k1.hdfs.rollCount = 0

hdfs_agent.sinks.k1.hdfs.batchSize = 2000

hdfs_agent.sinks.k1.hdfs.callTimeout = 300000

hdfs_agent.sinks.k1.hdfs.idleTimeout = 600


# Use a channel which buffers events in memory

hdfs_agent.channels.c1.type = memory

hdfs_agent.channels.c1.capacity = 2000000

hdfs_agent.channels.c1.transactionCapacity = 3000


# Bind the source and sink to the channel

hdfs_agent.sources.r1.channels = c1

hdfs_agent.sinks.k1.channel = c1


来自 “ ITPUB博客 ” ,链接:https://blog.itpub.net/31347383/viewspace-2168986/ ,如需转载,请注明出处,否则将追究法律责任。

下一篇: MR 博客记录
请登录后发表评论 登录
全部评论

注册时间:2016-05-23

  • 博文量
    65
  • 访问量
    84700