From d5d73f3e6f2aaf174b1e96c62bdedfe6df7d1ff5 Mon Sep 17 00:00:00 2001
From: ppom <>
Date: Sat, 21 Oct 2023 12:00:00 +0200
Subject: [PATCH] Reduce database size
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

...and it's still not a breaking change!
fix #42

Benchmark: using config/heavy-load.yml
From 8.1MB to 4.8MB → 40% of size reduction!

75% of size reduction if we cumulate both optimizations, waow!
---
 app/daemon.go  |  2 +-
 app/persist.go | 13 +++++++++++++
 app/pipe.go    |  2 +-
 app/types.go   |  1 +
 4 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/app/daemon.go b/app/daemon.go
index c8ddcbc..b66a4a5 100644
--- a/app/daemon.go
+++ b/app/daemon.go
@@ -187,7 +187,7 @@ func MatchesManager() {
 			matchesManagerHandleFlush(fo)
 		case pft = <-matchesC:
 
-			entry := LogEntry{pft.t, pft.p, pft.f.stream.name, pft.f.name, 0, false}
+			entry := LogEntry{pft.t, 0, pft.p, pft.f.stream.name, pft.f.name, 0, false}
 
 			entry.Exec = matchesManagerHandleMatch(pft)
 
diff --git a/app/persist.go b/app/persist.go
index 237d4a3..1e82a66 100644
--- a/app/persist.go
+++ b/app/persist.go
@@ -163,6 +163,7 @@ func rotateDB(c *Conf, logDec *gob.Decoder, flushDec *gob.Decoder, logEnc *gob.E
 		flushes[PSF{entry.Pattern, entry.Stream, entry.Filter}] = entry.T
 	}
 
+	lastTimeCpt := int64(0)
 	now := time.Now()
 	for {
 		var entry LogEntry
@@ -207,6 +208,12 @@ func rotateDB(c *Conf, logDec *gob.Decoder, flushDec *gob.Decoder, logEnc *gob.E
 			continue
 		}
 
+		// restore time
+		if entry.T.IsZero() {
+			entry.T = time.Unix(entry.S, lastTimeCpt)
+		}
+		lastTimeCpt++
+
 		// store matches
 		if !entry.Exec && entry.T.Add(filter.retryDuration).Unix() > now.Unix() {
 			if startup {
@@ -229,6 +236,7 @@ func rotateDB(c *Conf, logDec *gob.Decoder, flushDec *gob.Decoder, logEnc *gob.E
 }
 
 func encodeOrFatal(enc *gob.Encoder, entry LogEntry, writeSF2int map[SF]int, writeCounter *int) {
+	// Stream/Filter reduction
 	sf, ok := writeSF2int[SF{entry.Stream, entry.Filter}]
 	if ok {
 		entry.SF = sf
@@ -239,6 +247,11 @@ func encodeOrFatal(enc *gob.Encoder, entry LogEntry, writeSF2int map[SF]int, wri
 		writeSF2int[SF{entry.Stream, entry.Filter}] = *writeCounter
 		*writeCounter++
 	}
+	// Time reduction
+	if !entry.T.IsZero() {
+		entry.S = entry.T.Unix()
+		entry.T = time.Time{}
+	}
 	err := enc.Encode(entry)
 	if err != nil {
 		logger.Fatalln("Failed to write to new DB:", err)
diff --git a/app/pipe.go b/app/pipe.go
index 637ca86..c834bb6 100644
--- a/app/pipe.go
+++ b/app/pipe.go
@@ -99,7 +99,7 @@ func SocketManager(streams map[string]*Stream) {
 			case Show:
 				response.ClientStatus = genClientStatus(actions, matches, &actionsLock, &matchesLock)
 			case Flush:
-				le := LogEntry{time.Now(), request.Pattern, "", "", 0, false}
+				le := LogEntry{time.Now(), 0, request.Pattern, "", "", 0, false}
 				matchesC := FlushMatchOrder{request.Pattern, make(chan MatchesMap)}
 				actionsC := FlushActionOrder{request.Pattern, make(chan ActionsMap)}
 				flushToMatchesC <- matchesC
diff --git a/app/types.go b/app/types.go
index b8e67b7..0825878 100644
--- a/app/types.go
+++ b/app/types.go
@@ -62,6 +62,7 @@ type Action struct {
 
 type LogEntry struct {
 	T              time.Time
+	S              int64
 	Pattern        string
 	Stream, Filter string
 	SF             int