[Logstash-1.5.6 Pipeline]

https://www.elastic.co/guide/en/logstash/1.5/pipeline.html

1、每個input，會啓動一個input thread

input{
	generator {
	}
	generator {
	}
}

filter{
	kv{
	}
}

output{
	null{
	}
}

 top -H

  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
 6910 root      20   0 3570m 223m  14m R 73.1  2.8   0:20.11 |worker
 6908 root      20   0 3570m 223m  14m S 59.5  2.8   0:15.36 <generator
 6909 root      20   0 3570m 223m  14m S 52.5  2.8   0:15.16 <generator
 6912 root      20   0 3570m 223m  14m S 48.5  2.8   0:12.06 >output

input{
#       generator {
#       }
#       generator {
#       }
        file{
                path => ["/opt/1.txt"]
        }
        file{
                path => ["/opt/2.txt"]
        }
}

filter{
        kv{
        }
}

output{
        null{
        }
}

for i in `seq 1 5000`; do echo "b:2" >> /opt/1.txt; echo "a:1" >> /opt/2.txt;done;

top -H

 6959 root      20   0 3570m 235m  14m S 28.6  3.0   0:03.83 |worker
 6957 root      20   0 3570m 235m  14m S 26.6  3.0   0:03.70 <file
 6958 root      20   0 3570m 235m  14m S 24.6  3.0   0:03.52 <file
 6961 root      20   0 3570m 235m  14m S 17.3  3.0   0:02.06 >output

 

input{
#	generator {
#	}
#	generator {
#	}
#	file{
#		path => ["/opt/1.txt"]
#	}
	file{
		path => ["/opt/1.txt","/opt/2.txt"]
	}
}

filter{
	kv{
	}
}

output{
	null{
	}
}

top -H

 7057 root      20   0 3504m 230m  14m R 84.7  2.9   0:05.46 <file
 7058 root      20   0 3504m 230m  14m R 71.4  2.9   0:04.42 |worker
 7060 root      20   0 3504m 230m  14m R 43.2  2.9   0:02.35 >output

2、pipeline worker threads  -w  (default:1)

input{
#	generator {
#	}
#	generator {
#	}
#	file{
#		path => ["/opt/1.txt"]
#	}
	file{
		path => ["/opt/1.txt","/opt/2.txt"]
	}
}

filter{
	kv{
	}
}

output{
	null{
	}
}

../bin/logstash -f ./1.conf -w 5

top -H

  7107 root      20   0 3768m 236m  14m R  3.0  3.0   0:05.03 <file
 7108 root      20   0 3768m 236m  14m S  1.3  3.0   0:01.72 |worker
 7109 root      20   0 3768m 236m  14m R  1.0  3.0   0:01.44 |worker
 7110 root      20   0 3768m 236m  14m R  1.0  3.0   0:01.37 |worker
 7111 root      20   0 3768m 236m  14m S  1.0  3.0   0:01.42 |worker
 7112 root      20   0 3768m 236m  14m R  0.7  3.0   0:01.36 |worker
 7114 root      20   0 3768m 236m  14m S  0.7  3.0   0:02.08 >output

 

3、性能測試

http://kibana.logstash.es/content/logstash/performance/generator.html