From 0237f30135667585316b2d74b48600917fb64348 Mon Sep 17 00:00:00 2001 From: yo Date: Sat, 30 Mar 2024 17:28:20 +0100 Subject: [PATCH] outputs: examples and heavy load tests --- config/example_streamed_output.yml | 59 ++++++++++++++++++++++++++ config/heavy-load_cmd_to_redis.yml | 50 ++++++++++++++++++++++ config/heavy-load_write_to_redis.yml | 62 ++++++++++++++++++++++++++++ 3 files changed, 171 insertions(+) create mode 100644 config/example_streamed_output.yml create mode 100644 config/heavy-load_cmd_to_redis.yml create mode 100644 config/heavy-load_write_to_redis.yml diff --git a/config/example_streamed_output.yml b/config/example_streamed_output.yml new file mode 100644 index 0000000..6141234 --- /dev/null +++ b/config/example_streamed_output.yml @@ -0,0 +1,59 @@ +--- +concurrency: 0 + +# patterns are substitued in regexes. +# when a filter performs an action, it replaces the found pattern +patterns: + ip: + # reaction regex syntax is defined here: https://github.com/google/re2/wiki/Syntax + # simple version: regex: '(?:(?:[0-9]{1,3}\.){3}[0-9]{1,3})|(?:[0-9a-fA-F:]{2,90})' + regex: '(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|(?:(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(?::[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(?::[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(?::[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(?::[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:(?:(?::[0-9a-fA-F]{1,4}){1,6})|:(?:(?::[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(?::[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])|(?:[0-9a-fA-F]{1,4}:){1,4}:(?:(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(?:25[0-5]|(?:2[0-4]|1{0,1}[0-9]){0,1}[0-9]))' + ignore: + - 127.0.0.1 + - ::1 + # Patterns can be ignored based on regexes, it will try to match the whole string detected by the pattern + # ignoreregex: + # - '10\.0\.[0-9]{1,3}\.[0-9]{1,3}' + login: + regex: '[a-zA-Z0-9_\-\.]*' + + method: + regex: '.*' + + port: + regex: '[0-9]{1,5}' + +# Outputs are commands returning stdin you can use in write actions. +# This can ben used to get a persistent connection to p.e. a KV database you will write into, +# eliminating the overhead of executing a process each time action is trigged. +outputs: + redis: + start: ['redis-cli', '-h', 'redis.example.org', '-a', 'mypasswordoncmdlinedontdothis'] +# tee: +# start: ['tee', 'output.log'] + + +# streams are commands +# they are run and their ouptut is captured +# *example:* `tail -f /var/log/nginx/access.log` +# their output will be used by one or more filters +streams: + # streams have a user-defined name + ssh: + # note that if the command is not in environment's `PATH` + # its full path must be given. + cmd: ['tail', '-f', '/var/log/auth.log'] + # filters run actions when they match regexes on a stream + filters: + # filters have a user-defined name + acceptedlogin: + # reaction's regex syntax is defined here: https://github.com/google/re2/wiki/Syntax + regex: + - 'Accepted for from port ' + # actions are run by the filter when regexes are matched + actions: + # actions have a user-defined name + store2redis: + write: + output: redis + text: ['XADD', 'logins', '*', 'username', '', 'method', '', 'ip', '', 'port', ''] diff --git a/config/heavy-load_cmd_to_redis.yml b/config/heavy-load_cmd_to_redis.yml new file mode 100644 index 0000000..b9ace49 --- /dev/null +++ b/config/heavy-load_cmd_to_redis.yml @@ -0,0 +1,50 @@ +--- +patterns: + num: + regex: '[0-9]+' + idx: + regex: '[0-9]+' + ip: + regex: '(?:(?:[0-9]{1,3}\.){3}[0-9]{1,3})|(?:[0-9a-fA-F:]{2,90})' + ignore: + - 1.0.0.1 + +concurrency: 0 + +streams: + tailDown1: + cmd: [ 'sh', '-c', 'sleep 2; seq 100010 | while read i; do echo found $(($i % 100)) for test 1; done' ] + filters: + findIP: + regex: + - '^found for test $' + actions: + store2redis: + cmd: ['redis-cli', '-h', 'redis.example.org', '-a', 'mypasswordoncmdlinedontdothis', 'XADD', 'teststream', '*', 'found', '', 'test', ''] + tailDown2: + cmd: [ 'sh', '-c', 'sleep 2; seq 100010 | while read i; do echo prout $(($i % 100)) for test 2; done' ] + filters: + findIP: + regex: + - '^prout for test $' + actions: + store2redis: + cmd: ['redis-cli', '-h', 'redis.example.org', '-a', 'mypasswordoncmdlinedontdothis', 'XADD', 'teststream', '*', 'found', '', 'test', ''] + tailDown3: + cmd: [ 'sh', '-c', 'sleep 2; seq 100010 | while read i; do echo nanana $(($i % 100)) for test 3; done' ] + filters: + findIP: + regex: + - '^nanana for test $' + actions: + store2redis: + cmd: ['redis-cli', '-h', 'redis.example.org', '-a', 'mypasswordoncmdlinedontdothis', 'XADD', 'teststream', '*', 'found', '', 'test', ''] + tailDown4: + cmd: [ 'sh', '-c', 'sleep 2; seq 100010 | while read i; do echo nanana $(($i % 100)) for test 4; done' ] + filters: + findIP: + regex: + - '^nomatch for test $' + actions: + store2redis: + cmd: ['redis-cli', '-h', 'redis.example.org', '-a', 'mypasswordoncmdlinedontdothis', 'XADD', 'teststream', '*', 'found', '', 'test', ''] diff --git a/config/heavy-load_write_to_redis.yml b/config/heavy-load_write_to_redis.yml new file mode 100644 index 0000000..fc01e49 --- /dev/null +++ b/config/heavy-load_write_to_redis.yml @@ -0,0 +1,62 @@ +--- +patterns: + num: + regex: '[0-9]+' + idx: + regex: '[0-9]+' + ip: + regex: '(?:(?:[0-9]{1,3}\.){3}[0-9]{1,3})|(?:[0-9a-fA-F:]{2,90})' + ignore: + - 1.0.0.1 + +concurrency: 0 + +outputs: + redis: + start: ['redis-cli', '-h', 'redis.example.org', '-a', 'mypasswordoncmdlinedontdothis'] + +streams: + tailDown1: + cmd: [ 'sh', '-c', 'seq 100010 | while read i; do echo found $(($i % 100)) for test 1; done' ] + filters: + findIP: + regex: + - '^found for test $' + actions: + store2redis: + write: + output: redis + text: ['XADD', 'teststream', '*', 'found', '', 'test', ''] + tailDown2: + cmd: [ 'sh', '-c', 'seq 100010 | while read i; do echo prout $(($i % 100)) for test 2; done' ] + filters: + findIP: + regex: + - '^prout for test $' + actions: + store2redis: + write: + output: redis + text: ['XADD', 'teststream', '*', 'prout', '', 'test', ''] + tailDown3: + cmd: [ 'sh', '-c', 'seq 100010 | while read i; do echo nanana $(($i % 100)) for test 3; done' ] + filters: + findIP: + regex: + - '^nanana for test $' + actions: + store2redis: + write: + output: redis + text: ['XADD', 'teststream', '*', 'nanana', '', 'test', ''] + tailDown4: + cmd: [ 'sh', '-c', 'seq 100010 | while read i; do echo nanana $(($i % 100)) for test 4; done' ] + filters: + findIP: + regex: + - '^nomatch for test $' + actions: + store2redis: + write: + output: redis + text: ['XADD', 'teststream', '*', 'nomatch', '', 'test', '']