Size: 3148
Comment:
|
Size: 6002
Comment:
|
Deletions are marked like this. | Additions are marked like this. |
Line 22: | Line 22: |
* pip install graypy . | |
Line 27: | Line 28: |
import datetime | |
Line 72: | Line 74: |
} end rule "Timestamp" when m : GELFMessage( fullMessage matches "^.*\\s\\d+.\\d+.\\d+-\\d+.\\d+.\\d+.*" ) then Matcher matcher = Pattern.compile("\\s(\\d+).(\\d+).(\\d+)-(\\d+).(\\d+).(\\d+)").matcher(m.getFullMessage()); if (matcher.find()) { m.addAdditionalData("_year", matcher.group(1) ); m.addAdditionalData("_month", matcher.group(2) ); m.addAdditionalData("_day", matcher.group(3) ); m.addAdditionalData("_hour", matcher.group(4) ); m.addAdditionalData("_minute", matcher.group(5) ); m.addAdditionalData("_second", matcher.group(6) ); |
|
Line 123: | Line 143: |
}}} |
curl -XPOST "http://localhost:9200/_search" -d' { "query": { "filtered": { "query": { "match_all": { } } , "filter": { "and": [ {"range":{"_DateTimeX": {"from":"2010-01-01 00:00:00","to":"2010-06-07 23:59:59"} }} , {"term":{"fieldx":"valuex"}} ] } } } } ' }}} == 2016 test graylog == Elasticsearch {{{#!highlight bash cd /tmp wget https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.4.2/elasticsearch-2.4.2.tar.gz tar xvzf elasticsearch-2.4.2.tar.gz cd elasticsearch-2.4.2 nano config/elasticsearch.yml # cluster.name: graylog bin/elasticsearch }}} {{{#!highlight bash cd /tmp wget https://packages.graylog2.org/releases/graylog/graylog-2.1.2.tgz tar xvzf graylog-2.1.2.tgz cd graylog-2.1.2 mkdir -p /etc/graylog/server cp /tmp/graylog-2.1.2/graylog.conf.example /etc/graylog/server/server.conf uuidgen 90f7????-2c8b-????-9c2e-????b3282589 # set password_secret in /etc/graylog/server/server.conf root_username = admin # echo -n 12345678 | shasum -a 256 --> root_password_sha2 elasticsearch_cluster_name = graylog nano /etc/graylog/server/node-id # nodex mongod & bin/graylogctl start bin/graylogctl status tail -f log/graylog-server.log http://localhost:9000/gettingstarted }}} {{{ http://localhost:9000/system/inputs # create input for GELF HTTP, launch new input # Title: GelfHttpTest Node: nodex/localhost bind addr: 0.0.0.0 port 12201 curl -XPOST http://localhost:12201/gelf -p0 -d '{"short_message":"Hello there", "host":"example.org", "facility":"test", "_foo":"bar"}' }}} {{{ # added input GelfUdp port 12202 import logging import graypy import datetime my_logger = logging.getLogger('test_logger') my_logger.setLevel(logging.DEBUG) handler = graypy.GELFHandler('127.0.0.1', 12202) my_logger.addHandler(handler) my_logger.debug('Hello Graylog2.') my_logger.debug('Hello Graylog2, %s.'%(datetime.datetime.now() )) my_logger.info('Inf hello Graylog2, %s.'%(datetime.datetime.now() )) }}} |
Graylog2
Graylog2 is an open source log management solution that stores your logs in ElasticSearch.
Clean up DB
http://wiki.hackspherelabs.com/index.php?title=Graylog2#Clean_Out_Graylog2_DB
"Cure" for high CPU usage:
- service graylog2 stop
- cd /opt/elasticsearch-0.19.9/data/graylog2
- rm * -rf
- /opt/mongo/bin/mongo
- use graylog2
- db.message_counts.remove()
- db.hosts.remove()
- exit
- service graylog2 start
Send log from python to graylog2 through GELF
See details in https://pypi.python.org/pypi/graypy
Install with easy_install graypy . * pip install graypy .
1 #file name testGelf.py
2 import logging
3 import graypy
4 import datetime
5
6 my_logger = logging.getLogger('test_logger')
7 my_logger.setLevel(logging.DEBUG)
8
9 handler = graypy.GELFHandler('192.168.1.123', 12201)
10 my_logger.addHandler(handler)
11
12 my_logger.debug('Hello Graylog2.')
13 my_logger.debug('Hello Graylog2, %s.'%(datetime.datetime.now() ))
14 my_logger.info('Inf hello Graylog2, %s.'%(datetime.datetime.now() ))
On graylog2 the following columns are used:
- From: hostx
- Date: Tue Dec 10 13:14:50 +0000 2013
- Severity: Debug
- Facility: test_logger
- File: testGelf.py:10
thread_name: MainThread
function: <module>
process_name: MainProcess
- pid: 27663
Drools example .drl
rule "IMEI" when m : GELFMessage( shortMessage matches ".*\\s\\d{15}\\s.*" ) then Matcher matcher = Pattern.compile("\\s(\\d{15})\\s").matcher(m.getShortMessage()); if (matcher.find()) { m.addAdditionalData("_imei", matcher.group(1) ); } end rule "IP Port" when m : GELFMessage( shortMessage matches "^.*\\s\\d+.\\d+.\\d+.\\d+:\\d+\\s.*" ) then Matcher matcher = Pattern.compile("\\s(\\d+.\\d+.\\d+.\\d+):(\\d+)\\s").matcher(m.getShortMessage()); if (matcher.find()) { m.addAdditionalData("_ipaddr", matcher.group(1) ); m.addAdditionalData("_port", matcher.group(2) ); } end rule "Timestamp" when m : GELFMessage( fullMessage matches "^.*\\s\\d+.\\d+.\\d+-\\d+.\\d+.\\d+.*" ) then Matcher matcher = Pattern.compile("\\s(\\d+).(\\d+).(\\d+)-(\\d+).(\\d+).(\\d+)").matcher(m.getFullMessage()); if (matcher.find()) { m.addAdditionalData("_year", matcher.group(1) ); m.addAdditionalData("_month", matcher.group(2) ); m.addAdditionalData("_day", matcher.group(3) ); m.addAdditionalData("_hour", matcher.group(4) ); m.addAdditionalData("_minute", matcher.group(5) ); m.addAdditionalData("_second", matcher.group(6) ); } end
ElasticSearch Queries
http://www.elasticsearchtutorial.com/elasticsearch-in-5-minutes.html
http://joelabrahamsson.com/elasticsearch-101/
curl 'http://localhost:9200/blog/post/_search?q=user:dilbert&pretty=true' curl -XGET 'http://localhost:9200/blog/_search?pretty=true' -d ' { "query" : { "range" : { "postDate" : { "from" : "2011-12-10", "to" : "2011-12-12" } } } }' curl -XPOST "http://localhost:9200/_search" -d' { "query": { "filtered": { "query": { "match_all": { } }, "filter": { "term": { "FieldX": "value1234" } } } } }' curl -XPOST "http://localhost:9200/_search" -d' { "query": { "filtered": { "query": { "query_string": { "query": "other text" } }, "filter": { "term": { "field": "asdf" } } } } }' curl -XPOST "http://localhost:9200/_search" -d' { "query": { "filtered": { "query": { "match_all": { } } , "filter": { "and": [ {"range":{"_DateTimeX": {"from":"2010-01-01 00:00:00","to":"2010-06-07 23:59:59"} }} , {"term":{"fieldx":"valuex"}} ] } } } } '
2016 test graylog
Elasticsearch
1 cd /tmp
2 wget https://packages.graylog2.org/releases/graylog/graylog-2.1.2.tgz
3 tar xvzf graylog-2.1.2.tgz
4 cd graylog-2.1.2
5 mkdir -p /etc/graylog/server
6 cp /tmp/graylog-2.1.2/graylog.conf.example /etc/graylog/server/server.conf
7 uuidgen
8 90f7????-2c8b-????-9c2e-????b3282589
9 # set password_secret in /etc/graylog/server/server.conf
10 root_username = admin
11 # echo -n 12345678 | shasum -a 256 --> root_password_sha2
12 elasticsearch_cluster_name = graylog
13 nano /etc/graylog/server/node-id # nodex
14 mongod &
15 bin/graylogctl start
16 bin/graylogctl status
17 tail -f log/graylog-server.log
18 http://localhost:9000/gettingstarted
http://localhost:9000/system/inputs # create input for GELF HTTP, launch new input # Title: GelfHttpTest Node: nodex/localhost bind addr: 0.0.0.0 port 12201 curl -XPOST http://localhost:12201/gelf -p0 -d '{"short_message":"Hello there", "host":"example.org", "facility":"test", "_foo":"bar"}'
# added input GelfUdp port 12202 import logging import graypy import datetime my_logger = logging.getLogger('test_logger') my_logger.setLevel(logging.DEBUG) handler = graypy.GELFHandler('127.0.0.1', 12202) my_logger.addHandler(handler) my_logger.debug('Hello Graylog2.') my_logger.debug('Hello Graylog2, %s.'%(datetime.datetime.now() )) my_logger.info('Inf hello Graylog2, %s.'%(datetime.datetime.now() ))