v1.0.3: Support string list, with TTL

README.md

@@ -1,6 +1,6 @@
 # RedisLookupPlugin Plugin for Graylog
 
-Plugin to add Redis Data Adapter in read/write to graylog so you can store and retrieve key/values from pipelines  
+Plugin to add Redis Data Adapter in read/write to graylog so you can store and retrieve key/values and lists of string values from pipelines  
 Support Redis authentication (with password and username/password)  
 
 **Required Graylog version:** 5.0 and later
@@ -8,7 +8,7 @@ Support Redis authentication (with password and username/password)
 Installation
 ------------
 
-[Download the plugin](https://git.nosd.in/yo/graylog-redis-lookup-plugin/releases/download/v1.0.0/graylog-plugin-redis-lookup-1.0.0.jar)
+[Download the plugin](https://git.nosd.in/yo/graylog-redis-lookup-plugin/releases/download/v1.0.3/graylog-plugin-redis-lookup-1.0.3.jar)
 and place the `.jar` file in your Graylog plugin directory. The plugin directory
 is the `plugins/` folder relative from your `graylog-server` directory by default
 and can be configured in your `graylog.conf` file.
@@ -35,8 +35,18 @@ Usage
 * Use 'lookup_clear_key(lookup_table, key)' to remove key  
 * Use 'lookup_has_value(lookup_table, key)' to test key existence  
 * Use 'lookup_assign_ttl(lookup_table, key, ttl)' to change TTL of existing key  
+* Use 'lookup_set_string_list(lookup_table, key, value, [ttl])' to create a list named "key"  
+* Use 'lookup_add_string_list(lookup_table, key, value, [keep_duplicates])' to add value list to existing list  
+* Use 'lookup_remove_string_list(lookup_table, key, value) to remove a string from list "key"  
 
-By default keys will be created in Redis with the default TTL defined at data adapter creation time  
+By default single value keys will be created in Redis with the default TTL defined at data adapter creation time  
+
+Be aware that only setting TTL with 'lookup_assign_ttl' of 'lookup_set_string_list' alter TTL value in Redis ; so a list created with a TTL of 3600 will expire in 3600 seconds, even if it was updated with 'lookup_add_string_list' some seconds before expiration.  
+
+Known bugs
+----------
+
+Deletion via lookup_remove_string_list, lookup_clear_key or keep_duplicates=false sometimes not done.  
 
 
 Getting started

dependency-reduced-pom.xml

@@ -10,7 +10,7 @@
   <groupId>in.nosd.redis</groupId>
   <artifactId>graylog-plugin-redis-lookup</artifactId>
   <name>${project.artifactId}</name>
-  <version>1.0.2</version>
+  <version>1.0.3</version>
   <description>Graylog ${project.artifactId} plugin.</description>
   <url>https://www.graylog.org</url>
   <developers>

pom.xml

@@ -31,7 +31,7 @@
 
     <groupId>in.nosd.redis</groupId>
     <artifactId>graylog-plugin-redis-lookup</artifactId>
-    <version>1.0.2</version>
+    <version>1.0.3</version>
     <packaging>jar</packaging>
 
     <name>${project.artifactId}</name>

src/main/java/in/nosd/redis/dataadapters/RedisLookupDataAdapter.java

@@ -77,8 +77,14 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
     private final Meter redisDelRequestErrors;
     private final Timer redisAssignTtlRequestTimer;
     private final Meter redisAssignTtlRequestErrors;
+    private final Timer redisAddStringListRequestTimer;
+    private final Meter redisAddStringListRequestErrors;
     private final Timer redisSetStringListRequestTimer;
     private final Meter redisSetStringListRequestErrors;
+    private final Timer redisSetStringListWithTtlRequestTimer;
+    private final Meter redisSetStringListWithTtlRequestErrors;
+    private final Timer redisRemoveStringListRequestTimer;
+    private final Meter redisRemoveStringListRequestErrors;
 
     @Inject
     public RedisLookupDataAdapter(@Assisted("dto") DataAdapterDto dto,
@@ -107,8 +113,14 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
         this.redisDelRequestErrors = metricRegistry.meter(MetricRegistry.name(getClass(), "redisDelRequestErrors"));
         this.redisAssignTtlRequestTimer = metricRegistry.timer(MetricRegistry.name(getClass(), "redisAssignTtlRequestTime"));
         this.redisAssignTtlRequestErrors = metricRegistry.meter(MetricRegistry.name(getClass(), "redisAssignTtlRequestErrors"));
+        this.redisAddStringListRequestTimer = metricRegistry.timer(MetricRegistry.name(getClass(), "redisAddStringListRequestTime"));
+        this.redisAddStringListRequestErrors = metricRegistry.meter(MetricRegistry.name(getClass(), "redisAddStringListRequestErrors"));
         this.redisSetStringListRequestTimer = metricRegistry.timer(MetricRegistry.name(getClass(), "redisSetStringListRequestTime"));
         this.redisSetStringListRequestErrors = metricRegistry.meter(MetricRegistry.name(getClass(), "redisSetStringListRequestErrors"));
+        this.redisSetStringListWithTtlRequestTimer = metricRegistry.timer(MetricRegistry.name(getClass(), "redisSetStringListWithTtlRequestTime"));
+        this.redisSetStringListWithTtlRequestErrors = metricRegistry.meter(MetricRegistry.name(getClass(), "redisSetStringListWithTtlRequestErrors"));
+        this.redisRemoveStringListRequestTimer = metricRegistry.timer(MetricRegistry.name(getClass(), "redisRemoveStringListRequestTime"));
+        this.redisRemoveStringListRequestErrors = metricRegistry.meter(MetricRegistry.name(getClass(), "redisRemoveStringListRequestErrors"));
     }
 
     @Override
@@ -135,6 +147,11 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
         cachePurge.purgeAll();
     }
 
+    private String getSingleValue(String key) {
+        final String value = this.commands.get(key);
+        return value;
+    }
+
     @Override
     protected LookupResult doGet(Object key) {
         final Timer.Context time = redisGetRequestTimer.time();
@@ -144,15 +161,22 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
             return getEmptyResult();
         }
         try {
-            final String value = this.commands.get(trimmedKey);
+            // Get item type and existence
-            if (value == null) {
+            final String type = this.commands.type(trimmedKey);
-                LOG.debug("Redis GET request for key <{}> returned null, key do not exists.", trimmedKey);
+            switch(type) {
-                redisGetRequestErrors.mark();
+                case "none":
-                return LookupResult.empty();
+                    LOG.debug("Redis TYPE request for key <{}> returned null, key do not exists.", trimmedKey);
+                    redisGetRequestErrors.mark();
+                    return LookupResult.empty();
+                case "list":
+                    final List<String> result = this.commands.lrange(trimmedKey, 0, -1);
+                    return LookupResult.withoutTTL().stringListValue(result).build();
+                default:
+                    final String value = getSingleValue(trimmedKey);
+                    return LookupResult.single(value);
             }
-            return LookupResult.single(value);
         } catch (Exception e) {
-            LOG.error("Redis GET request error for key <{}>", trimmedKey, e);
+            LOG.error("Exception: Redis GET request error for key <{}>", trimmedKey, e);
             redisGetRequestErrors.mark();
             return LookupResult.empty();
         } finally {
@@ -188,7 +212,7 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
             }
             return LookupResult.single(value.toString());
         } catch (Exception e) {
-            LOG.error("Redis SET(EX) key <{}> to value <{}> with TTL <{}> returned an exception: {}", key, value, ttlSec, e);
+            LOG.error("Exception: Redis SET(EX) key <{}> to value <{}> with TTL <{}> returned {}", key, value, ttlSec, e);
             redisSetRequestErrors.mark();
             return LookupResult.withError();
         } finally {
@@ -201,14 +225,14 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
         final Timer.Context time = redisDelRequestTimer.time();
         final String trimmedKey = StringUtils.trimToNull(key.toString());
         try {
-            final Long result = this.commands.del(key.toString());
+            final Long result = this.commands.del(trimmedKey);
             if (result != 1) {
-                LOG.debug("Redis DEL key <{}> returned {}", key, result);
+                LOG.debug("Redis DEL key <{}> returned {}", trimmedKey, result);
                 redisDelRequestErrors.mark();
             }
             return;
         } catch (Exception e) {
-            LOG.error("Redis DEL key <{}> returned {}", key, e);
+            LOG.error("Exception: Redis DEL key <{}> returned {}", trimmedKey, e);
             redisDelRequestErrors.mark();
             return;
         } finally {
@@ -218,30 +242,32 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
 
     private LookupResult setExpire(String key, Long ttl) {
         try {
-            final Boolean result = this.commands.expire(key, ttl);
+            if (!this.commands.expire(key, ttl)) {
-            if (!result) {
+                LOG.warn("Redis EXPIRE key <{}> to <{}> returned false (key does not exist or the timeout could not be set)", key, ttl);
-                LOG.warn("Redis EXPIRE key <{}> to <{}> returned {}", key, ttl, result);
                 return LookupResult.withError();
             }
-            final String value = this.commands.get(key);
+            return LookupResult.single(this.commands.get(key).toString());
-            return LookupResult.single(value.toString());
-        } catch (Exception e) {
-            LOG.error("Redis EXPIRE key <{}> to <{}> returned {}", key, ttl, e);
-            return LookupResult.withError(e.toString());
         }
+        catch (Exception e) {
+            // lettuce 6.3.0 returns "WRONGTYPE Operation against a key holding the wrong kind of value" when EXPIRE on a list key, but do the job.
+            if (e.getMessage().startsWith("WRONGTYPE Operation against a key holding the wrong kind of value")) {
+            } else {
+                LOG.error("Exception: Redis EXPIRE key <{}> to <{}> returned {}", key, ttl, e);
+                return LookupResult.withError(e.toString());
+            }
+        }
+        return LookupResult.single(this.commands.get(key).toString());
     }
 
     private LookupResult setPersist(String key) {
         try {
-            final Boolean result = this.commands.persist(key);
+            if (!this.commands.persist(key)) {
-            if (!result) {
+                LOG.debug("Redis PERSIST key <{}> returned false (key does not exist or does not have an associated timeout)", key);
-                LOG.warn("Redis PERSIST key <{}> returned {}", key, result);
                 return LookupResult.withError();
             }
-            final String value = this.commands.get(key);
+            return LookupResult.single(this.commands.get(key).toString());
-            return LookupResult.single(value.toString());
         } catch (Exception e) {
-            LOG.error("Redis PERSIST key <{}> returned {}", key, e);
+            LOG.error("Exception: Redis PERSIST key <{}> returned {}", key, e);
             return LookupResult.withError(e.toString());
         }
     }
@@ -257,9 +283,40 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
                 return setPersist(trimmedKey);
             }
         } catch (Exception e) {
-            LOG.error("assignTtl <{}> to key <{}> returned {}", ttlSec, trimmedKey, e);
+            // lettuce 6.3.0 returns "WRONGTYPE Operation against a key holding the wrong kind of value" when TTL on a list key, but do the job.
-            redisAssignTtlRequestErrors.mark();
+            if (e.getMessage().startsWith("WRONGTYPE Operation against a key holding the wrong kind of value")) {
-            return LookupResult.withError();
+            } else {
+                LOG.error("Exception: assignTtl <{}> to key <{}> returned {}", ttlSec, trimmedKey, e);
+                redisAssignTtlRequestErrors.mark();
+                return LookupResult.withError();
+            }
+        } finally {
+            time.stop();
+        }
+        return LookupResult.single(this.commands.get(trimmedKey));
+    }
+
+    @Override
+    public LookupResult addStringList(Object key, List<String> listValue, boolean keepDuplicates) {
+        final Timer.Context time = redisAddStringListRequestTimer.time();
+        final String trimmedKey = StringUtils.trimToNull(key.toString());
+        if (trimmedKey == null) {
+            LOG.debug("A blank key was supplied");
+            return getEmptyResult();
+        }
+        try {
+            if (!keepDuplicates) {
+                removeStringList(trimmedKey, listValue);
+            }
+            final Long len = this.commands.rpush(trimmedKey, listValue.toArray(new String[0]));
+            if (len > 0) {
+                return LookupResult.withoutTTL().stringListValue(this.commands.lrange(trimmedKey, 0, -1)).build();
+            }
+            return LookupResult.empty();
+        } catch (Exception e) {
+            LOG.error("Exception: Redis RPUSH request error for key <{}>: <{}>", trimmedKey, e);
+            redisAddStringListRequestErrors.mark();
+            return LookupResult.empty();
         } finally {
             time.stop();
         }
@@ -267,7 +324,70 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
 
     @Override
     public LookupResult setStringList(Object key, List<String> listValue) {
-        return LookupResult.empty();
+        final Timer.Context time = redisSetStringListRequestTimer.time();
+        final String trimmedKey = StringUtils.trimToNull(key.toString());
+        if (trimmedKey == null) {
+            LOG.debug("A blank key was supplied");
+            return getEmptyResult();
+        }
+        try {
+            // We need to replace list, so we delete it first
+            this.commands.ltrim(trimmedKey, 1, 0);
+            final Long len = this.commands.rpush(trimmedKey, listValue.toArray(new String[0]));
+            if (len > 0) {
+                return LookupResult.withoutTTL().stringListValue(this.commands.lrange(trimmedKey, 0, -1)).build();
+            }
+            return LookupResult.empty();
+        } catch (Exception e) {
+            LOG.error("Exception: Redis RPUSH request error for key <{}>: <{}>", trimmedKey, e);
+            redisSetStringListRequestErrors.mark();
+            return LookupResult.empty();
+        } finally {
+            time.stop();
+        }
+    }
+
+    public LookupResult setStringListWithTtl(Object key, List<String> listValue, Long ttlSec) {
+        final Timer.Context time = redisSetStringListWithTtlRequestTimer.time();
+        try {
+            setStringList(key, listValue);
+            return(assignTtl(key, ttlSec));
+        } catch (Exception e) {
+            // This exception comes from assignTtl
+            if (e.getMessage().startsWith("WRONGTYPE Operation against a key holding the wrong kind of value")) {
+            } else {
+                LOG.error("Exception: Redis RPUSH request error for key <{}>: <{}>", key, e);
+                redisSetStringListWithTtlRequestErrors.mark();
+                return LookupResult.empty();
+            }
+        } finally {
+            time.stop();
+        }
+        return LookupResult.withoutTTL().stringListValue(this.commands.lrange(StringUtils.trimToNull(key.toString()), 0, -1)).build();
+    }
+
+    @Override
+    public LookupResult removeStringList(Object key, List<String> listValue) {
+        final Timer.Context time = redisRemoveStringListRequestTimer.time();
+        final String trimmedKey = StringUtils.trimToNull(key.toString());
+        if (trimmedKey == null) {
+            LOG.debug("A blank key was supplied");
+            return getEmptyResult();
+        }
+        try {
+            long removed = 0;
+            for (String value : listValue) {
+                removed += this.commands.lrem(trimmedKey, 0, value);
+            }
+            LOG.debug("Redis LREM for key <{}> and value <{}> deleted <{}> items", trimmedKey, listValue, removed);
+            return LookupResult.withoutTTL().stringListValue(this.commands.lrange(trimmedKey, 0, -1)).build();
+        } catch (Exception e) {
+            LOG.error("Exception: Redis LREM request error for key <{}> and value <{}>: <{}>", trimmedKey, listValue, e);
+            redisRemoveStringListRequestErrors.mark();
+            return LookupResult.empty();
+        } finally {
+            time.stop();
+        }
     }
 
     public interface Factory extends LookupDataAdapter.Factory2<RedisLookupDataAdapter> {