WIP : Select data type from dropdown list

src/main/java/in/nosd/redis/RedisLookupPluginModule.java

@@ -26,6 +26,7 @@ import org.graylog2.plugin.PluginModule;
 
 import in.nosd.redis.dataadapters.RedisLookupDataAdapter;
 import in.nosd.redis.functions.RedisLookupPluginFunction;
+import in.nosd.redis.migrations.V104_MigrateRedisType;
 
 import java.util.Collections;
 import java.util.Set;
@@ -64,8 +65,10 @@ public class RedisLookupPluginModule extends PluginModule {
          *
          * addConfigBeans();
          */
+
+        addMigration(V104_MigrateRedisType.class);
          
-         addMessageProcessorFunction(RedisLookupPluginFunction.NAME, RedisLookupPluginFunction.class);
+        addMessageProcessorFunction(RedisLookupPluginFunction.NAME, RedisLookupPluginFunction.class);
 
         installLookupDataAdapter2(RedisLookupDataAdapter.NAME, RedisLookupDataAdapter.class, 
             RedisLookupDataAdapter.Factory.class, RedisLookupDataAdapter.Config.class);

src/main/java/in/nosd/redis/dataadapters/RedisLookupDataAdapter.java

@@ -60,16 +60,15 @@ import java.util.StringJoiner;
 
 public class RedisLookupDataAdapter extends LookupDataAdapter {    
     private static final Logger LOG = LoggerFactory.getLogger(RedisLookupDataAdapter.class);
-
+    
     public static final String NAME = "RedisLookup";
-
+    
     private static final Duration REFRESH_INTERVAL_DURATION = Duration.ZERO;
-
+    
     private final Config config;
     private final RedisClient client;
     private RedisCommands<String, String> commands;
     private StatefulRedisConnection<String, String> connection;
-
     private final Timer redisGetRequestTimer;
     private final Meter redisGetRequestErrors;
     private final Timer redisSetRequestTimer;
@@ -124,9 +123,6 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
         this.redisRemoveStringListRequestErrors = metricRegistry.meter(MetricRegistry.name(getClass(), "redisRemoveStringListRequestErrors"));
     }
 
-/**************************************************************************
-* LookupDataAdapter functions
-**************************************************************************/
     @Override
     protected void doStart() throws Exception {        
         connection = this.client.connect();
@@ -151,6 +147,11 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
         cachePurge.purgeAll();
     }
 
+    private String getSingleValue(String key) {
+        final String value = this.commands.get(key);
+        return value;
+    }
+
     @Override
     protected LookupResult doGet(Object key) {
         final Timer.Context time = redisGetRequestTimer.time();
@@ -239,11 +240,6 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
         }
     }
 
-    private String getSingleValue(String key) {
-        final String value = this.commands.get(key);
-        return value;
-    }
-
     private LookupResult setExpire(String key, Long ttl) {
         try {
             if (!this.commands.expire(key, ttl)) {
@@ -312,10 +308,6 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
             if (!keepDuplicates) {
                 removeStringList(trimmedKey, listValue);
             }
-            
-            LOG.info("Redis addStringList: List is <{}> items", listValue.size());
-            LOG.info("Redis addStringList: List is <{}>", listValue);
-            
             final Long len = this.commands.rpush(trimmedKey, listValue.toArray(new String[0]));
             if (len > 0) {
                 return LookupResult.withoutTTL().stringListValue(this.commands.lrange(trimmedKey, 0, -1)).build();
@@ -452,7 +444,10 @@ public class RedisLookupDataAdapter extends LookupDataAdapter {
         public abstract int redisDB();
         
         @JsonProperty("redis_type")
-        @NotEmpty
+        /* FIXME: This should be notEmpty, but migration crash when dbDataAdapterService.findAll() with error 
+        * "Missing required properties: redisType"
+        * so dont flag NoteEmpty and put Nullable. */
+        @Nullable
         public abstract String redisType();
 
         @JsonProperty("redis_ttl")

src/main/java/in/nosd/redis/migrations/V104_MigrateRedisType.java

@@ -0,0 +1,140 @@
+/*
+ * Copyright (C) 2024 johan@nosd.in
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+package in.nosd.redis.migrations;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.auto.value.AutoValue;
+import com.google.common.collect.ImmutableSet;
+import org.graylog.autovalue.WithBeanGetter;
+import in.nosd.redis.dataadapters.RedisLookupDataAdapter;
+import in.nosd.redis.dataadapters.RedisLookupDataAdapter.Config;
+import org.graylog2.events.ClusterEventBus;
+import org.graylog2.lookup.db.DBDataAdapterService;
+import org.graylog2.lookup.dto.DataAdapterDto;
+import org.graylog2.lookup.events.DataAdaptersUpdated;
+import org.graylog2.migrations.Migration;
+import org.graylog2.plugin.cluster.ClusterConfigService;
+import org.graylog2.plugin.lookup.LookupDataAdapterConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.time.ZonedDateTime;
+import java.util.Collections;
+import java.util.Collection;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static com.google.common.base.Strings.isNullOrEmpty;
+
+// From graylog-plugin-threatintel/src/main/java/org/graylog/plugins/threatintel/migrations/V20170821100300_MigrateOTXAPIToken.java
+
+/* Manually :
+*
+* Go into mongodb instance
+* Locate "lut_data_adapters" table
+* Edit every entry for which "config.type" == "RedisLookup"
+*  Add "redis_type" key with value "strings"
+*/
+
+
+public class V104_MigrateRedisType extends Migration {
+    private static final Logger LOG = LoggerFactory.getLogger(V104_MigrateRedisType.class);
+    private static final ImmutableSet<String> REDIS_DATA_ADAPTER_NAMES = ImmutableSet.of("redis-lookup");
+
+    private final ClusterConfigService clusterConfigService;
+    private final DBDataAdapterService dbDataAdapterService;
+    private final ClusterEventBus clusterBus;
+
+    @Inject
+    public V104_MigrateRedisType(ClusterConfigService clusterConfigService,
+                                 DBDataAdapterService dbDataAdapterService,
+                                 ClusterEventBus clusterBus) {
+        this.clusterConfigService = clusterConfigService;
+        this.dbDataAdapterService = dbDataAdapterService;
+        this.clusterBus = clusterBus;
+    }
+
+    @Override
+    public ZonedDateTime createdAt() {
+        return ZonedDateTime.parse("2024-03-03T15:42:42Z");
+    }
+
+    @Override
+    public void upgrade() {
+        Config redisConf;
+
+        // List every instance of lookup DB adapter, and find instance with Config.type == "RedisLookup" (RedisLookupDataAdapter.NAME)
+        Collection<DataAdapterDto> DataAdapters = dbDataAdapterService.findAll(); 
+        for (DataAdapterDto dataAdapterDto : DataAdapters) {
+            if (dataAdapterDto.config().type().equals(RedisLookupDataAdapter.NAME)) {
+                redisConf = (Config) dataAdapterDto.config();
+                
+                final Config newConf = Config.builder()
+                                            .type(redisConf.type())
+                                            .redisHost(redisConf.redisHost())
+                                            .redisPort(redisConf.redisPort())
+                                            .redisDB(redisConf.redisDB())
+                                            .redisType("strings")
+                                            .redisKeyTTL(redisConf.redisKeyTTL())
+                                            .redisUsername(redisConf.redisUsername())
+                                            .redisPassword(redisConf.redisPassword())
+                                            .build();
+                
+                final DataAdapterDto newDto = DataAdapterDto.builder()
+                    .id(dataAdapterDto.id())
+                    .config(newConf)
+                    .title(dataAdapterDto.title())
+                    .description(dataAdapterDto.description())
+                    .name(dataAdapterDto.name())
+                    .build();
+
+                final DataAdapterDto saved = dbDataAdapterService.save(newDto);
+                clusterBus.post(DataAdaptersUpdated.create(saved.id()));
+                LOG.debug("Redis data adapter <{}> migrated: data_type added", dataAdapterDto.name());
+            }
+        }
+    }
+
+    @JsonAutoDetect
+    @AutoValue
+    @WithBeanGetter
+    public static abstract class MigrationCompleted {
+        @JsonProperty("converted_redis_type")
+        public abstract boolean convertedRedisType();
+
+        @JsonProperty("data_adapter_ids")
+        public abstract Set<String> dataAdapterIds();
+
+        @JsonCreator
+        public static MigrationCompleted create(@JsonProperty("data_adapter_ids") final Set<String> dataAdapterIds,
+                                                @JsonProperty("converted_redis_type") final boolean convertedRedisType) {
+            return new AutoValue_V104_MigrateRedisType_MigrationCompleted(convertedRedisType, dataAdapterIds);
+        }
+
+        public static MigrationCompleted convertedType(@JsonProperty("data_adapter_ids") final Set<String> dataAdapterIds) {
+            return create(dataAdapterIds, true);
+        }
+
+        public static MigrationCompleted notConvertedType() {
+            return create(Collections.emptySet(), false);
+        }
+    }
+}
+ 

src/web/dataadapters/RedisLookupAdapterFieldSet.jsx

@@ -17,7 +17,7 @@
 import React, { useCallback, useEffect, useRef, useState } from 'react';
 import PropTypes from 'prop-types';
 
-import ObjectUtils from 'util/ObjectUtils';
+//import ObjectUtils from 'util/ObjectUtils';
 import { Input } from 'components/bootstrap';
 // "Missing or invalid plugin" in "dataAdapter create/Data adapter type" list when imported
 //import { Select } from './components/common';

src/web/dataadapters/RedisLookupAdapterSummary.jsx

@@ -39,6 +39,8 @@ class RedisLookupAdapterSummary extends React.Component {
       <dt>Redis database</dt>
       <dd>{config.redis_database}</dd>
       <dt>Redis key TTL</dt>
+      <dd>{config.redis_type || 'n/a'}</dd>
+      <dt>Redis data type</dt>
       <dd>{config.redis_ttl || 'n/a'}</dd>
       <dt>Redis username</dt>
       <dd>{config.redis_username || 'n/a'}</dd>