From 861f8e456d730531da42f396cc3c5895947344c3 Mon Sep 17 00:00:00 2001 From: yo000 Date: Sun, 3 Mar 2024 19:28:58 +0100 Subject: [PATCH] Add Migration to add data type to existing data adapter --- .../migrations/V104_MigrateRedisType.java | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 src/main/java/in/nosd/redis/migrations/V104_MigrateRedisType.java diff --git a/src/main/java/in/nosd/redis/migrations/V104_MigrateRedisType.java b/src/main/java/in/nosd/redis/migrations/V104_MigrateRedisType.java new file mode 100644 index 0000000..072ea83 --- /dev/null +++ b/src/main/java/in/nosd/redis/migrations/V104_MigrateRedisType.java @@ -0,0 +1,140 @@ +/* + * Copyright (C) 2024 johan@nosd.in + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the Server Side Public License, version 1, + * as published by MongoDB, Inc. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * Server Side Public License for more details. + * + * You should have received a copy of the Server Side Public License + * along with this program. If not, see + * . + */ +package in.nosd.redis.migrations; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableSet; +import org.graylog.autovalue.WithBeanGetter; +import in.nosd.redis.dataadapters.RedisLookupDataAdapter; +import in.nosd.redis.dataadapters.RedisLookupDataAdapter.Config; +import org.graylog2.events.ClusterEventBus; +import org.graylog2.lookup.db.DBDataAdapterService; +import org.graylog2.lookup.dto.DataAdapterDto; +import org.graylog2.lookup.events.DataAdaptersUpdated; +import org.graylog2.migrations.Migration; +import org.graylog2.plugin.cluster.ClusterConfigService; +import org.graylog2.plugin.lookup.LookupDataAdapterConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Inject; +import java.time.ZonedDateTime; +import java.util.Collections; +import java.util.Collection; +import java.util.Set; +import java.util.stream.Collectors; + +import static com.google.common.base.Strings.isNullOrEmpty; + +// From graylog-plugin-threatintel/src/main/java/org/graylog/plugins/threatintel/migrations/V20170821100300_MigrateOTXAPIToken.java + +/* Manually : +* +* Go into mongodb instance +* Locate "lut_data_adapters" table +* Edit every entry for which "config.type" == "RedisLookup" +* Add "redis_type" key with value "strings" +*/ + + +public class V104_MigrateRedisType extends Migration { + private static final Logger LOG = LoggerFactory.getLogger(V104_MigrateRedisType.class); + private static final ImmutableSet REDIS_DATA_ADAPTER_NAMES = ImmutableSet.of("redis-lookup"); + + private final ClusterConfigService clusterConfigService; + private final DBDataAdapterService dbDataAdapterService; + private final ClusterEventBus clusterBus; + + @Inject + public V104_MigrateRedisType(ClusterConfigService clusterConfigService, + DBDataAdapterService dbDataAdapterService, + ClusterEventBus clusterBus) { + this.clusterConfigService = clusterConfigService; + this.dbDataAdapterService = dbDataAdapterService; + this.clusterBus = clusterBus; + } + + @Override + public ZonedDateTime createdAt() { + return ZonedDateTime.parse("2024-03-03T15:42:42Z"); + } + + @Override + public void upgrade() { + Config redisConf; + + // List every instance of lookup DB adapter, and find instance with Config.type == "RedisLookup" (RedisLookupDataAdapter.NAME) + Collection DataAdapters = dbDataAdapterService.findAll(); + for (DataAdapterDto dataAdapterDto : DataAdapters) { + if (dataAdapterDto.config().type().equals(RedisLookupDataAdapter.NAME)) { + redisConf = (Config) dataAdapterDto.config(); + + final Config newConf = Config.builder() + .type(redisConf.type()) + .redisHost(redisConf.redisHost()) + .redisPort(redisConf.redisPort()) + .redisDB(redisConf.redisDB()) + .redisType("strings") + .redisKeyTTL(redisConf.redisKeyTTL()) + .redisUsername(redisConf.redisUsername()) + .redisPassword(redisConf.redisPassword()) + .build(); + + final DataAdapterDto newDto = DataAdapterDto.builder() + .id(dataAdapterDto.id()) + .config(newConf) + .title(dataAdapterDto.title()) + .description(dataAdapterDto.description()) + .name(dataAdapterDto.name()) + .build(); + + final DataAdapterDto saved = dbDataAdapterService.save(newDto); + clusterBus.post(DataAdaptersUpdated.create(saved.id())); + LOG.debug("Redis data adapter <{}> migrated: data_type added", dataAdapterDto.name()); + } + } + } + + @JsonAutoDetect + @AutoValue + @WithBeanGetter + public static abstract class MigrationCompleted { + @JsonProperty("converted_redis_type") + public abstract boolean convertedRedisType(); + + @JsonProperty("data_adapter_ids") + public abstract Set dataAdapterIds(); + + @JsonCreator + public static MigrationCompleted create(@JsonProperty("data_adapter_ids") final Set dataAdapterIds, + @JsonProperty("converted_redis_type") final boolean convertedRedisType) { + return new AutoValue_V104_MigrateRedisType_MigrationCompleted(convertedRedisType, dataAdapterIds); + } + + public static MigrationCompleted convertedType(@JsonProperty("data_adapter_ids") final Set dataAdapterIds) { + return create(dataAdapterIds, true); + } + + public static MigrationCompleted notConvertedType() { + return create(Collections.emptySet(), false); + } + } +} +