flume在linux環境下的安裝與配置

tar -zxvf apache-flume-1.8.0-bin.tar.gz

配置環境變量

vim /etc/profile

#set flume environment
export FLUME_HOME=/usr/local/apache-flume-1.8.0
export PATH=$FLUME_HOME/bin:$PATH

source /etc/profile

修改 配置

vim flume-conf.properties

agent.sources = tail-source
agent.channels = tail-channel
agent.sinks = tail-sink

agent.sources.tail-source.type = TAILDIR
agent.sources.tail-source.channels = tail-channel

#須要tail的文件指定group,可指定多個，用空格隔開
agent.sources.tail-source.filegroups = debug
agent.sources.tail-source.filegroups.debug = /usr/local/wujintao/debug.log

#對於taildir,須要間歇性的保存tail文件的位置，以便中斷後能夠繼續從中斷點tail，json格式文件
agent.sources.tail-source.positionFile = /usr/local/wujintao/taildir_position.json

#每一個tail的文件,建立一個kafka topic
agent.sources.tail-source.headers.debug.topic = agent-debug
agent.sources.tail-source.skipToEnd = true
agent.sources.tail-source.interceptors = i1 i2
agent.sources.tail-source.interceptors.i1.type = timestamp
agent.sources.tail-source.interceptors.i2.type = host
agent.sources.tail-source.interceptors.i2.useIP = true
agent.sources.tail-source.interceptors.i2.hostHeader = host

agent.channels.tail-channel.type = memory
agent.channels.tail-channel.capacity = 1000
agent.channels.tail-channel.transactionCapacity = 100

agent.sinks.tail-sink.type = org.apache.flume.sink.kafka.KafkaSink
agent.sinks.tail-sink.channel = tail-channel
agent.sinks.tail-sink.kafka.bootstrap.servers = localhost:9092
agent.sinks.tail-sink.flumeBatchSize = 32
agent.sinks.tail-sink.kafka.producer.acks = 1
agent.sinks.tail-sink.useFlumeEventFormat = false

啓動

flume-ng agent --conf conf --conf-file conf/flume-conf.properties --name agent -Dflume.root.logger=INFO,console &