wget http://www.apache.org/dyn/closer.cgi/flume/1.5.0/apache-flume-1.5.0-bin.tar.gz

tar zxvf apache-flume-1.5.0-bin.tar.gz

mv apache-flume-1.5.0-bin flume
cd conf

vim flume.conf
内容如下：

点击(此处)折叠或打开

agent1.sources=source1
agent1.sinks=sink1
agent1.channels=channel1

agent1.sources.source1.type=exec
agent1.sources.source1.command=tail -f /usr/local/nginx/logs/access.log
agent1.sources.source1.channels=channel1
agent1.sources.source1.fileHeader=true

agent1.sources.source1.interceptors=i
agent1.sources.source1.interceptors.i.regex= .*\.(css|js|jpg|jpeg|png|gif|ico|swf|flv).*
agent1.sources.source1.interceptors.i.type= org.apache.flume.interceptor.RegexFilteringInterceptor$Builder
agent1.sources.source1.interceptors.i.excludeEvents=true

agent1.sinks.sink1.type=hdfs
agent1.sinks.sink1.hdfs.path=hdfs://192.168.0.237:9000/flumelog/%Y/%m/%d
agent1.sinks.sink1.hdfs.filePrefix=nginx-%Y-%m-%d-%H
agent1.sinks.sink1.hdfs.fileType=DataStream
agent1.sinks.sink1.hdfs.useLocalTimeStamp=true
agent1.sinks.sink1.hdfs.writeFormat=TEXT

agent1.sinks.sink1.hdfs.batchSize=1000

agent1.sinks.sink1.hdfs.rollSize=0
agent1.sinks.sink1.hdfs.rollCount=0
agent1.sinks.sink1.hdfs.rollInterval=0
agent1.sinks.sink1.hdfs.threadsPoolSize=4
agent1.sinks.sink1.hdfs.callTimeout=30000
agent1.sinks.sink1.channel=channel1

agent1.channels.channel1.type=file
agent1.channels.channel1.checkpointDir=/usr/flume_temp/flume_tmp123
agent1.channels.channel1.dataDirs=/usr/flume_temp/flume_tmp

创建目录
mkdir -p /usr/flume_temp/flume_tmp123
mkdir -p /usr/flume_temp/flume_tmp

mv flume-env.sh.template flume-env.sh
vi flume-env.sh
修改JAVA_HOME后保存退出

修改环境变量 vi /etc/profile
1.配置好JAVA_HOME
2.添加export PATH=$PATH:/usr/flume/flume/bin
source /etc/profile

进入到flume根目录：
flume-ng agent -c conf -f conf/flume.conf --name agent1 -Dflume.root.logger=INFO,console
(必然会报类找不到的错误，将相应hadoop的jar包复制到/lib目录下再启动即可。)

ITPub博客

Flume监听Nginx日志流向HDFS安装配置

wget http://www.apache.org/dyn/closer.cgi/flume/1.5.0/apache-flume-1.5.0-bin.tar.gz

最新文章