`3c407d8`

fix race conditions in ML engines causing data corruption

Add sync.RWMutex protection to prevent concurrent map access:
- tfidf_engine.go: protect vocabulary and idf maps
- bm25_engine.go: protect vocabulary and idf maps
- markov_generator.go: protect chains and starters maps
- ensemble_system.go: prevent concurrent Train() calls

This fixes the bug where dates appeared in command_frequency
instead of actual commands, and eliminates 'concurrent map
read and map write' panics.

Authored by mfwolffe <wolffemf@dukes.jmu.edu> 4 months ago

SHA: 3c407d83a24359d0715da2f19ae0328ea0b1cdc5
Parents: 08b88f7
Tree: 38c0833

4 changed files

Status	File	+	-
M	`internal/llm/bm25_engine.go`	8	0
M	`internal/llm/ensemble_system.go`	10	3
M	`internal/llm/markov_generator.go`	14	3
M	`internal/llm/tfidf_engine.go`	8	0

internal/llm/bm25_engine.gomodified

  import (
  	"math"
 +	"sync"
+ )
  // BM25Engine implements BM25 ranking algorithm (superior to basic TF-IDF)
  // BM25 is the industry standard for text search and ranking
  type BM25Engine struct {
 +	mu            sync.RWMutex
  	vocabulary    map[string]int      // word -> index
  	idf           map[string]float64  // word -> inverse document frequency
  	docLengths    []int               // document lengths
  // BuildCorpus builds the BM25 corpus from documents
  func (engine *BM25Engine) BuildCorpus(documents []string) {
 +	engine.mu.Lock()
 +	defer engine.mu.Unlock()
++
  	// First pass: extract terms and calculate document frequencies
  	documentFreq := make(map[string]int)
  	engine.docLengths = make([]int, len(documents))
  // Score calculates BM25 score for a query against a document
  func (engine *BM25Engine) Score(query string, document string) float64 {
 +	engine.mu.RLock()
 +	defer engine.mu.RUnlock()
++
  	queryTerms := engine.extractNGrams(query)
  	docTerms := engine.extractNGrams(document)

internal/llm/ensemble_system.gomodified

  import (
  	"math"
  	"sort"
 +	"sync"
+ )
  // EnsembleSystem combines multiple ML techniques for optimal insult selection
  type EnsembleSystem struct {
 +	mu               sync.RWMutex
  	tfidfEngine      *TFIDFEngine
  	bm25Engine       *BM25Engine  // NEW: Industry-standard BM25 ranking
  	markovGen        *MarkovGenerator
  // Train trains all ML components on the insult database
  func (es *EnsembleSystem) Train() {
 +	es.mu.Lock()
  	if es.trained {
 +		es.mu.Unlock()
  		return // Already trained
+ 	}
 +	es.trained = true // Mark as training to prevent concurrent attempts
 +	es.mu.Unlock()
  	// Collect all insult texts
  	insults := make([]string, 0, len(es.database.Insults))
  	// Train Markov generator
  	es.markovGen.Train(insults)
+-
 -	es.trained = true
+ }
  // GenerateInsult generates the best possible insult using ensemble methods
  	personality string,
  ) string {
  	// Ensure training is done
 -	if !es.trained {
 +	es.mu.RLock()
 +	trained := es.trained
 +	es.mu.RUnlock()
 +	if !trained {
  		es.Train()
+ 	}

internal/llm/markov_generator.gomodified

  import (
  	"math/rand"
  	"strings"
 +	"sync"
  	"time"
+ )
  // MarkovGenerator generates novel insults using Markov chains
  type MarkovGenerator struct {
 +	mu          sync.RWMutex
  	chains      map[string]map[string]int // state -> next_word -> count
  	starters    []string                   // possible starting words
  	order       int                        // n-gram order (2 = bigram)
  // Train trains the Markov chain on a corpus of insults
  func (mg *MarkovGenerator) Train(insults []string) {
 +	mg.mu.Lock()
 +	defer mg.mu.Unlock()
++
  	for _, insult := range insults {
 -		mg.trainOnText(insult)
 +		mg.trainOnTextUnlocked(insult)
+ 	}
+ }
 -// trainOnText trains on a single text
 -func (mg *MarkovGenerator) trainOnText(text string) {
 +// trainOnTextUnlocked trains on a single text (caller must hold lock)
 +func (mg *MarkovGenerator) trainOnTextUnlocked(text string) {
  	words := mg.tokenize(text)
  	if len(words) < mg.order+1 {
  		return
  // Generate generates a novel insult
  func (mg *MarkovGenerator) Generate() string {
 +	mg.mu.RLock()
 +	defer mg.mu.RUnlock()
++
  	if len(mg.starters) == 0 || len(mg.chains) == 0 {
  		return "" // Not trained yet
+ 	}
  // GenerateContextual generates an insult with context hints
  func (mg *MarkovGenerator) GenerateContextual(seedWords []string) string {
 +	mg.mu.RLock()
 +	defer mg.mu.RUnlock()
++
  	if len(mg.chains) == 0 {
  		return ""
+ 	}

internal/llm/tfidf_engine.gomodified

  import (
  	"math"
  	"strings"
 +	"sync"
  	"unicode"
+ )
  // TFIDFEngine implements semantic similarity using TF-IDF vectors
  type TFIDFEngine struct {
 +	mu             sync.RWMutex
  	vocabulary     map[string]int    // word -> index
  	idf            map[string]float64 // word -> inverse document frequency
  	documentCount  int
  // BuildCorpus builds the TF-IDF corpus from a collection of documents
  func (engine *TFIDFEngine) BuildCorpus(documents []string) {
 +	engine.mu.Lock()
 +	defer engine.mu.Unlock()
++
  	// First pass: build vocabulary and count document frequencies
  	documentFreq := make(map[string]int)
  // Vectorize converts text to TF-IDF vector
  func (engine *TFIDFEngine) Vectorize(text string) map[string]float64 {
 +	engine.mu.RLock()
 +	defer engine.mu.RUnlock()
++
  	vector := make(map[string]float64)
  	tokens := engine.extractNGrams(text)