-
Notifications
You must be signed in to change notification settings - Fork 0
/
stomp_output.conf
86 lines (76 loc) · 1.76 KB
/
stomp_output.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
input {
http_poller {
urls => {
b56d => {
method => "GET"
url => "https://data.cincinnati-oh.gov/resource/b56d-ydmm.json?$limit=50000&$offset=0&$order=time"
headers => {
Accept => "application/json"
}
}
}
request_timeout => 60
schedule => { cron => "* * * * * UTC"}
codec => "json"
metadata_target => "[@metadata][response_metadata]"
}
}
filter {
#Store Processing date time
mutate { rename => [ "@timestamp", "processing_time" ]}
# The main @timestamp of the event
date {
match => ["time", "yyyy-MM-dd'T'HH:mm:ss'.'SSS"]
target => "@timestamp"
}
date {
match => ["loadts", "yyyy-MM-dd'T'HH:mm:ss'.'SSS"]
target => "loadts"
}
# Some conversions
mutate {
convert => {
"latitude" => "float"
"longitude" => "float"
"speed" => "float"
"distance_traveled" => "float"
"odometer" => "float"
}
}
#"device_position": { "type": "geo_point" }
mutate {
rename => {
"latitude" => "[device_position][lat]"
"longitude" => "[device_position][lon]"
}
}
# Unique Id the metric
fingerprint {
method => "MD5"
key => "secret hashing key"
concatenate_sources => true
concatenate_all_fields => false
source => ["asset", "@timestamp"]
target => "[@metadata][document_id]"
}
#remove unecessary fields added by logstash
mutate { remove_field => ["host", "@version", "time"] }
}
output {
elasticsearch {
hosts => ["http://127.0.0.1:9000"]
index => "sensors-data-%{+YYYY-MM-dd}"
document_id => "%{[@metadata][document_id]}"
action => "index"
}
stomp {
host => "127.0.0.1"
port => 61613
user => system
password => manager
destination => "sensors-data"
headers => {
"persistent" => true
}
}
}