ADD and DEL operations now supported

openldap-log-parser.go

@@ -43,10 +43,14 @@ const (
 	UnbindRE          = `(UNBIND)?`
 	// group[40]
 	ConnClosedRE      = `(closed)?(?: \(connection lost\))?`
+	// group[41]
+	AddDnRE           = `(?:ADD dn="(.*)")?`
+	// group[42]
+	DelDnRE           = `(?:DEL dn="(.*)")?`
 	
 	LogLineRE = SyslogPri + TimeRE + ` ` + HostRE + ` ` + ProcessRE + ` ` + ConnIdRE + ` ` + ConnFdRE + OperationIdRE + ` ` +
 				AcceptRE + STARTTLSRE + BindMethodRE + BindMechRE + ResultRE + SearchBaseRE + SearchAttrRE + SearchResultRE + ModDnRE + ModAttrRE + 
-				PassModRE + UnbindRE + ConnClosedRE
+				PassModRE + UnbindRE + ConnClosedRE + AddDnRE + DelDnRE
 )
 
 type (
@@ -74,6 +78,8 @@ type (
 		SSF            string     `json:"ssf"`
 		ModDN          string     `json:"mod_dn"`
 		ModAttr        string     `json:"mod_attr"`
+		AddDN          string     `json:"add_dn"`
+		DelDN          string     `json:"del_dn"`
 		PassModDN      string     `json:"passmod_dn"`
 		Result         bool
 		ResTag         string     `json:"result_tag"`
@@ -184,6 +190,8 @@ func (o *OpenldapLog) Parse(text []byte) (LogFormat, error) {
 		ModDN:             string(group[36]),
 		ModAttr:           string(group[37]),
 		PassModDN:         string(group[38]),
+		AddDN:             string(group[41]),
+		DelDN:             string(group[42]),
 	}
 
 	// Now handle Operation Type
@@ -203,6 +211,10 @@ func (o *OpenldapLog) Parse(text []byte) (LogFormat, error) {
 		logFormat.OpType = "unbind"
 	} else if len(group[40]) > 0 {
 		logFormat.OpType = "close"
+	} else if len(group[41]) > 0 {
+		logFormat.OpType = "add"
+	} else if len(group[42]) > 0 {
+		logFormat.OpType = "del"
 	}
 
 	return logFormat, nil

openldap-log-parser/cmd/root.go

@@ -57,6 +57,8 @@ type (
 		SSF            string     `json:"ssf,omitempty"`
 		ModDN          string     `json:"mod_dn,omitempty"`
 		ModAttr        string     `json:"mod_attr,omitempty"`
+		AddDN          string     `json:"add_dn,omitempty"`
+		DelDN          string     `json:"del_dn,omitempty"`
 		PassModDN      string     `json:"passmod_dn,omitempty"`
 		ResTag         string     `json:"result_tag,omitempty"`
 		ResOid         string     `json:"result_oid,omitempty"`
@@ -100,6 +102,8 @@ type (
 		StartTLS       bool       `json:"starttls,omitempty"`
 		ModDN          string     `json:"mod_dn,omitempty"`
 		ModAttr        string     `json:"mod_attr,omitempty"`
+		AddDN          string     `json:"add_dn,omitempty"`
+		DelDN          string     `json:"del_dn,omitempty"`
 		PassModDN      string     `json:"passmod_dn,omitempty"`
 		ResTag         string     `json:"result_tag,omitempty"`
 		ResOid         string     `json:"result_oid,omitempty"`
@@ -125,7 +129,7 @@ var (
 	File   os.File
 	Writer *bufio.Writer
 
-	Version = "0.6.6"
+	Version = "0.6.11"
 
 	BuildInfo = promauto.NewGaugeVec(prometheus.GaugeOpts{
 		Name: "openldaplogparser_build_info",
@@ -151,35 +155,43 @@ var (
 		Name: "openldaplogparser_client_count",
 		Help: "Number of connected clients",
 	})
-  AcceptCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+	AcceptCnt = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "openldaplogparser_accept_count",
 		Help: "Number of ACCEPT commands executed",
 	}, []string{"host"})
-  BindCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+	BindCnt = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "openldaplogparser_bind_count",
 		Help: "Number of BIND commands executed",
 	}, []string{"host"})
-  SearchCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+	SearchCnt = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "openldaplogparser_search_count",
 		Help: "Number of SRCH commands executed",
 	}, []string{"host"})
-  ModCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+	ModCnt = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "openldaplogparser_mod_count",
 		Help: "Number of MOD commands executed",
 	}, []string{"host"})
-  PassModCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+	AddCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+	  Name: "openldaplogparser_add_count",
+	  Help: "Number of ADD commands executed",
+	}, []string{"host"})
+	DelCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+		Name: "openldaplogparser_del_count",
+		Help: "Number of DEL commands executed",
+	}, []string{"host"})
+	PassModCnt = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "openldaplogparser_passmod_count",
 		Help: "Number of PASSMOD commands executed",
 	}, []string{"host"})
-  UnbindCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+	UnbindCnt = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "openldaplogparser_unbind_count",
 		Help: "Number of UNBIND commands executed",
 	}, []string{"host"})
-  CloseCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+	CloseCnt = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "openldaplogparser_close_count",
 		Help: "Number of closed connections",
 	}, []string{"host"})
-  StartTLSCnt = promauto.NewCounterVec(prometheus.CounterOpts{
+	StartTLSCnt = promauto.NewCounterVec(prometheus.CounterOpts{
 		Name: "openldaplogparser_starttlscount",
 		Help: "Number of STARTTLS commands executed",
 	}, []string{"host"})
@@ -217,7 +229,7 @@ func OlcToFlat(olc *OpenLdapConnection) []OpenLdapConnectionFlat {
 
 	for i := range olc.Operations {
 		olcf[i] = OpenLdapConnectionFlat{
-			Time:           olc.Time,
+			Time:           olc.Operations[i].Time,
 			Hostname:       olc.Hostname,
 			Process:        olc.Process,
 			ClientIp:       olc.ClientIp,
@@ -273,6 +285,22 @@ func OlcToFlat(olc *OpenLdapConnection) []OpenLdapConnectionFlat {
 				olcf[i].ResQTime = olc.Operations[i].ResQTime
 				olcf[i].ResETime = olc.Operations[i].ResETime
 				olcf[i].ResText = olc.Operations[i].ResText
+			case "add":
+				olcf[i].AddDN = olc.Operations[i].AddDN
+				olcf[i].ResTag = olc.Operations[i].ResTag
+				olcf[i].ResOid = olc.Operations[i].ResOid
+				olcf[i].ResErr = olc.Operations[i].ResErr
+				olcf[i].ResQTime = olc.Operations[i].ResQTime
+				olcf[i].ResETime = olc.Operations[i].ResETime
+				olcf[i].ResText = olc.Operations[i].ResText
+			case "del":
+				olcf[i].DelDN = olc.Operations[i].DelDN
+				olcf[i].ResTag = olc.Operations[i].ResTag
+				olcf[i].ResOid = olc.Operations[i].ResOid
+				olcf[i].ResErr = olc.Operations[i].ResErr
+				olcf[i].ResQTime = olc.Operations[i].ResQTime
+				olcf[i].ResETime = olc.Operations[i].ResETime
+				olcf[i].ResText = olc.Operations[i].ResText
 			case "passmod":
 				olcf[i].PassModDN = olc.Operations[i].PassModDN
 				olcf[i].ResTag = olc.Operations[i].ResTag
@@ -321,35 +349,39 @@ func writeOut(msg string, filename string) error {
 	return nil
 }
 
-// Every 24H, remove sent, milter-rejected and deferred that entered queue more than 5 days ago
-/*
-func periodicallyCleanMQueue(mqueue map[int]*PostfixLogParser, mqMtx *sync.Mutex) {
-	var ok int
+func cleanMQueue(mqueue map[string]*OpenLdapConnection, mqMtx *sync.Mutex, age time.Duration) {
+    var ok bool
 
-	for range time.Tick(time.Hour * 24) {
-		// Do we need read lock?
-		for _, inmail := range mqueue {
-			ok = 0
-			// Check all mails were sent (multiple destinations mails)
-			//  or rejected
-			for _, outmail := range inmail.Messages {
-				if outmail.Status == "sent" || outmail.Status == "milter-reject" {
-					ok += 1
-				} else if outmail.Status == "deferred" {
-					if inmail.Time.Add(time.Hour * 5 * 24).Before(time.Now()) {
-						ok += 1
-					}
+	log.Printf("Start cleaning queue task: %d items in queue", len(mqueue))
+
+	// Do we need read lock?
+	for uid, ldcon := range mqueue {
+		ok = false
+		// Check if a close operation exist
+		for _, op := range ldcon.Operations {
+			if op.OpType == "close" {
+				if op.Time.Add(age).Before(time.Now()) {
+					ok = true
 				}
 			}
-			if ok == len(inmail.Messages) {
-				mqMtx.Lock()
-				delete(mqueue, inmail.MessageId)
-				mqMtx.Unlock()
-			}
+		}
+		if ok == true {
+			mqMtx.Lock()
+			delete(mqueue, uid)
+			mqMtx.Unlock()
 		}
 	}
+	log.Printf("Finished cleaning queue task: %d items in queue", len(mqueue))
+}
+
+
+
+// Every 24H, remove connections closed more than 24 hours ago
+func periodicallyCleanMQueue(mqueue map[string]*OpenLdapConnection, mqMtx *sync.Mutex) {
+	for range time.Tick(time.Hour * 24) {
+		cleanMQueue(mqueue, mqMtx, 24 * time.Hour)
+	}
 }
-*/
 
 func initConfig() {}
 
@@ -731,6 +763,82 @@ func parseStoreAndWrite(input []byte, mq map[string]*OpenLdapConnection, mqMtx *
 		mqMtx.Unlock()
 		return nil
 	}
+	
+	/*
+	 * 2022-07-18T14:35:17.381223+02:00 ldap.domain.org slapd slapd[82581] conn=16113 op=3 ADD dn="cn=coincoin,dc=domain,dc=org"
+	 */
+	if logFormat.AddDN != "" {
+		op := &Operation{
+			Time:           logFormat.Time,
+			OpType:         logFormat.OpType,
+			OpId:           &logFormat.OpId,
+			AddDN:          logFormat.AddDN,
+		}
+		mqMtx.Lock()
+		olc, ok := mq[fmt.Sprintf("%s:%d", logFormat.Hostname, logFormat.ConnId)]
+		if false == ok {
+			if false == gDispUnkConn {
+				mqMtx.Unlock()
+				return nil
+			} else {
+				// Create connection
+				olc = &OpenLdapConnection{
+					Time:           logFormat.Time,
+					Hostname:       logFormat.Hostname,
+					Process:        logFormat.Process,
+					ConnId:         logFormat.ConnId,
+					ConnFd:         logFormat.ConnFd,
+					ClientIp:       logFormat.ClientIp,
+					ClientPort:     logFormat.ClientPort,
+					ServerIp:       logFormat.ServerIp,
+					ServerPort:     logFormat.ServerPort,
+				}
+				mq[fmt.Sprintf("%s:%d", logFormat.Hostname, logFormat.ConnId)] = olc
+			}
+		}
+		olc.Operations = append(olc.Operations, op)
+		mqMtx.Unlock()
+		AddCnt.WithLabelValues(olc.Hostname).Inc()
+		return nil
+	}
+	
+	/*
+	 * 2022-07-18T14:35:17.381223+02:00 ldap.domain.org slapd slapd[82581] conn=16113 op=3 DEL dn="cn=coincoin,dc=domain,dc=org"
+	 */
+	if logFormat.DelDN != "" {
+		op := &Operation{
+			Time:           logFormat.Time,
+			OpType:         logFormat.OpType,
+			OpId:           &logFormat.OpId,
+			DelDN:          logFormat.DelDN,
+		}
+		mqMtx.Lock()
+		olc, ok := mq[fmt.Sprintf("%s:%d", logFormat.Hostname, logFormat.ConnId)]
+		if false == ok {
+			if false == gDispUnkConn {
+				mqMtx.Unlock()
+				return nil
+			} else {
+				// Create connection
+				olc = &OpenLdapConnection{
+					Time:           logFormat.Time,
+					Hostname:       logFormat.Hostname,
+					Process:        logFormat.Process,
+					ConnId:         logFormat.ConnId,
+					ConnFd:         logFormat.ConnFd,
+					ClientIp:       logFormat.ClientIp,
+					ClientPort:     logFormat.ClientPort,
+					ServerIp:       logFormat.ServerIp,
+					ServerPort:     logFormat.ServerPort,
+				}
+				mq[fmt.Sprintf("%s:%d", logFormat.Hostname, logFormat.ConnId)] = olc
+			}
+		}
+		olc.Operations = append(olc.Operations, op)
+		mqMtx.Unlock()
+		DelCnt.WithLabelValues(olc.Hostname).Inc()
+		return nil
+	}
   
 	/*
 		2022-07-18T11:13:17.521717+02:00 ldap.domain.org slapd[82581] conn=16113 op=4 PASSMOD id="cn=pika,ou=users,dc=domain,dc=org" new
@@ -975,7 +1083,18 @@ func processLogs(cmd *cobra.Command, args []string) {
 	}
 
 	// Cleaner thread
-	//go periodicallyCleanMQueue(mQueue, &mqMtx)
+	go periodicallyCleanMQueue(mQueue, &mqMtx)
+
+    // On demand Mqueue cleaning... For debug, dont try this at home, kids!
+/*	sig2 := make(chan os.Signal)
+	signal.Notify(sig2, syscall.SIGUSR2)
+	go func() {
+		for {
+			<-sig2
+			cleanMQueue(mQueue, &mqMtx, 5 * time.Minute)
+		}
+	}()
+*/
 
 	// Initialize Stdin input...
 	if true == strings.EqualFold(gSyslogListenAddress, "do-not-listen") {