Create mail2news.go

1 files changed, 1652 insertions, 0 deletions

diff --git a/mail2news.go b/mail2news.go
new file mode 100644
index 0000000..cb5df5b
--- /dev/null
+++ b/mail2news.go
@@ -0,0 +1,1652 @@
+package main
+
+import (
+	"bufio"
+	"crypto/sha256"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math/rand"
+	"net"
+	"net/mail"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/spf13/viper"
+	"golang.org/x/net/proxy"
+	// Imports for encoding handling
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/charmap"
+	"golang.org/x/text/encoding/htmlindex"
+	"golang.org/x/text/transform"
+	// Imports for transfer encoding
+	"mime/quotedprintable"
+	"encoding/base64"
+)
+
+// Config structure to hold all configuration
+type Config struct {
+	Paths      PathsConfig      `mapstructure:"paths"`
+	NNTP       NNTPConfig       `mapstructure:"nntp"`
+	Thresholds ThresholdsConfig `mapstructure:"thresholds"`
+	Logging    LoggingConfig    `mapstructure:"logging"`
+	Encoding   EncodingConfig   `mapstructure:"encoding"`
+}
+
+type PathsConfig struct {
+	Log     string `mapstructure:"log"`
+	Etc     string `mapstructure:"etc"`
+	Lib     string `mapstructure:"lib"`
+	History string `mapstructure:"history"`
+}
+
+type NNTPConfig struct {
+	Path           string `mapstructure:"path_header"`
+	InjectionHost  string `mapstructure:"injection_host"`
+	Contact        string `mapstructure:"contact"`
+	MessageID      string `mapstructure:"messageid"`
+	DefaultFrom    string `mapstructure:"default_from"`
+	PrimaryOnion   string `mapstructure:"primary_onion"`
+	FallbackServer string `mapstructure:"fallback_server"`
+	TorProxy       string `mapstructure:"tor_proxy"`
+	AlwaysUseTor   bool   `mapstructure:"always_use_tor"`
+}
+
+type ThresholdsConfig struct {
+	MaxBytes      int `mapstructure:"max_bytes"`
+	MaxCrossposts int `mapstructure:"max_crossposts"`
+	HoursPast     int `mapstructure:"hours_past"`
+	HoursFuture   int `mapstructure:"hours_future"`
+	SocketTimeout int `mapstructure:"socket_timeout"`
+}
+
+type LoggingConfig struct {
+	Level   string `mapstructure:"level"`
+	Format  string `mapstructure:"format"`
+	DateFmt string `mapstructure:"datefmt"`
+	Retain  int    `mapstructure:"retain"`
+}
+
+type EncodingConfig struct {
+	ForceUtf8       bool   `mapstructure:"force_utf8"`
+	FallbackCharset string `mapstructure:"fallback_charset"`
+}
+
+// Global configuration
+var config Config
+
+// Version constant
+const VERSION = "0.9.4"
+
+// Global lock file path and handle
+var lockFilePath string
+var lockFile *os.File
+
+// MessageIDCache stores recently processed message IDs to avoid duplicates
+type MessageIDCache struct {
+	cache    map[string]time.Time // Maps message ID to time it was processed
+	mutex    sync.RWMutex
+	maxAge   time.Duration // Maximum age of cache entries
+	cacheDir string        // Directory for persistent cache
+}
+
+// Global message ID cache
+var messageCache *MessageIDCache
+
+// NewMessageIDCache creates a new message ID cache with the specified max age
+func NewMessageIDCache(maxAge time.Duration, cacheDir string) *MessageIDCache {
+	// Create cache directory if it doesn't exist
+	if cacheDir != "" {
+		os.MkdirAll(cacheDir, 0755)
+	}
+
+	cache := &MessageIDCache{
+		cache:    make(map[string]time.Time),
+		maxAge:   maxAge,
+		cacheDir: cacheDir,
+	}
+
+	// Load persisted cache (if exists)
+	cache.loadCache()
+
+	// Start background cleanup goroutine
+	go cache.startCleanupLoop(maxAge / 2)
+
+	return cache
+}
+
+// loadCache loads the persisted cache from disk
+func (c *MessageIDCache) loadCache() {
+	if c.cacheDir == "" {
+		return
+	}
+
+	cacheFile := filepath.Join(c.cacheDir, "message_cache.txt")
+	data, err := ioutil.ReadFile(cacheFile)
+	if err != nil {
+		// Cache file may not exist yet, that's ok
+		return
+	}
+
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+
+	// Clear existing cache
+	c.cache = make(map[string]time.Time)
+
+	// Parse each line as "messageID timestamp"
+	lines := strings.Split(string(data), "\n")
+	now := time.Now()
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+
+		parts := strings.Split(line, " ")
+		if len(parts) != 2 {
+			continue
+		}
+
+		messageID := parts[0]
+		timestampStr := parts[1]
+		timestamp, err := strconv.ParseInt(timestampStr, 10, 64)
+		if err != nil {
+			continue
+		}
+
+		processTime := time.Unix(timestamp, 0)
+		// Only load entries that haven't expired
+		if now.Sub(processTime) < c.maxAge {
+			c.cache[messageID] = processTime
+		}
+	}
+
+	logMessage(fmt.Sprintf("Loaded %d recent message IDs from cache", len(c.cache)), "DEBUG")
+}
+
+// saveCache persists the cache to disk
+func (c *MessageIDCache) saveCache() {
+	if c.cacheDir == "" {
+		return
+	}
+
+	c.mutex.RLock()
+	defer c.mutex.RUnlock()
+
+	// Build cache file content
+	var cacheContent strings.Builder
+	for messageID, timestamp := range c.cache {
+		cacheContent.WriteString(fmt.Sprintf("%s %d\n", messageID, timestamp.Unix()))
+	}
+
+	// Write to file
+	cacheFile := filepath.Join(c.cacheDir, "message_cache.txt")
+	err := ioutil.WriteFile(cacheFile, []byte(cacheContent.String()), 0644)
+	if err != nil {
+		logMessage(fmt.Sprintf("Error saving message cache: %v", err), "WARNING")
+	}
+}
+
+// startCleanupLoop periodically cleans up expired cache entries
+func (c *MessageIDCache) startCleanupLoop(interval time.Duration) {
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+
+	for range ticker.C {
+		c.cleanup()
+		c.saveCache()
+	}
+}
+
+// cleanup removes expired entries from the cache
+func (c *MessageIDCache) cleanup() {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+
+	now := time.Now()
+	for id, timestamp := range c.cache {
+		if now.Sub(timestamp) > c.maxAge {
+			delete(c.cache, id)
+		}
+	}
+}
+
+// IsProcessed checks if a message ID has been processed recently
+func (c *MessageIDCache) IsProcessed(messageID string) bool {
+	// Strip any angle brackets from message ID
+	messageID = strings.Trim(messageID, "<>")
+
+	c.mutex.RLock()
+	defer c.mutex.RUnlock()
+
+	_, exists := c.cache[messageID]
+	return exists
+}
+
+// MarkProcessed marks a message ID as processed
+func (c *MessageIDCache) MarkProcessed(messageID string) {
+	// Strip any angle brackets from message ID
+	messageID = strings.Trim(messageID, "<>")
+
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+
+	c.cache[messageID] = time.Now()
+
+	// If cache is getting large, trigger a save
+	if len(c.cache)%100 == 0 {
+		go c.saveCache()
+	}
+}
+
+// acquireLock creates a lock file to ensure only one instance is running
+// Using both atomic file creation with O_EXCL and system-level flock
+func acquireLock() bool {
+	// Use a fixed location for better lock handling
+	lockDir := "/var/lock/mail2news"
+	err := os.MkdirAll(lockDir, 0755)
+	if err != nil {
+		log.Printf("Error creating lock directory %s: %v", lockDir, err)
+		// Fall back to temp directory if can't create lock dir
+		lockDir = os.TempDir()
+	}
+	
+	lockFilePath = filepath.Join(lockDir, "mail2news.lock")
+	
+	// Attempt to create the lock file with O_EXCL flag
+	file, err := os.OpenFile(lockFilePath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
+	if err != nil {
+		// If file exists, check if it's stale
+		if os.IsExist(err) {
+			log.Printf("Lock file exists at %s. Checking if it's stale...", lockFilePath)
+			
+			// Try to acquire the existing lock for reading
+			existingFile, err := os.OpenFile(lockFilePath, os.O_RDWR, 0600)
+			if err != nil {
+				log.Printf("Error opening existing lock file: %v", err)
+				return false
+			}
+			
+			// Try to apply a non-blocking flock - if successful, the lock is stale
+			err = syscall.Flock(int(existingFile.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
+			if err != nil {
+				// Cannot lock - another process has it
+				existingFile.Close()
+				log.Printf("Another mail2news instance is holding the lock. Exiting.")
+				return false
+			}
+			
+			// We got the lock, so it was stale. Read PID from file for logging
+			content, _ := ioutil.ReadAll(existingFile)
+			lines := strings.Split(string(content), "\n")
+			if len(lines) > 0 {
+				pidStr := strings.TrimSpace(lines[0])
+				pid, _ := strconv.Atoi(pidStr)
+				log.Printf("Found stale lock from PID %d. Removing it.", pid)
+			} else {
+				log.Printf("Found stale lock file with invalid format. Removing it.")
+			}
+			
+			// Keep this file and handle since we've already flocked it
+			lockFile = existingFile
+			
+			// Truncate and rewrite with our info
+			lockFile.Truncate(0)
+			lockFile.Seek(0, 0)
+			
+			// Write our PID and info to the lock file
+			pid := os.Getpid()
+			timestamp := time.Now().Format(time.RFC3339)
+			lockContent := fmt.Sprintf("%d\n%s\nmail2news v%s", pid, timestamp, VERSION)
+			lockFile.WriteString(lockContent)
+			
+			log.Printf("Lock acquired with flock by PID %d (recycled stale lock)", pid)
+			return true
+		}
+		
+		// Any other error creating the lock file
+		log.Printf("Failed to create lock file: %v", err)
+		return false
+	}
+	
+	// Keep reference to the file so we can flock it and close it later
+	lockFile = file
+	
+	// Apply a system-level flock to the file - this is the key to robust locking
+	if err := syscall.Flock(int(lockFile.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil {
+		log.Printf("Could not acquire system lock: %v", err)
+		lockFile.Close()
+		os.Remove(lockFilePath)
+		return false
+	}
+	
+	// Successfully created and locked the file
+	// Write PID, timestamp, and version to lock file
+	pid := os.Getpid()
+	timestamp := time.Now().Format(time.RFC3339)
+	lockContent := fmt.Sprintf("%d\n%s\nmail2news v%s", pid, timestamp, VERSION)
+	
+	if _, err := lockFile.WriteString(lockContent); err != nil {
+		log.Printf("Failed to write to lock file: %v", err)
+		releaseLock() // Will release flock and remove file
+		return false
+	}
+	
+	log.Printf("Lock acquired with flock by PID %d", pid)
+	return true
+}
+
+// releaseLock releases the process lock
+func releaseLock() {
+	if lockFile != nil {
+		// Release the system-level lock
+		syscall.Flock(int(lockFile.Fd()), syscall.LOCK_UN)
+		lockFile.Close()
+		lockFile = nil
+		
+		// Remove the lock file
+		log.Printf("Releasing lock (PID %d)", os.Getpid())
+		err := os.Remove(lockFilePath)
+		if err != nil && !os.IsNotExist(err) {
+			log.Printf("Warning: Failed to remove lock file: %v", err)
+		}
+	}
+}
+
+// Initialize logging
+func initLogging() {
+	// Make sure path is set
+	if config.Paths.Log == "" {
+		config.Paths.Log = "/var/log/mail2news/mail2news.log"
+	}
+
+	// Create directory if it doesn't exist
+	logDir := filepath.Dir(config.Paths.Log)
+	err := os.MkdirAll(logDir, 0755)
+	if err != nil {
+		log.Printf("Error creating log directory %s: %v", logDir, err)
+		log.SetOutput(os.Stdout)
+		log.SetFlags(log.Ldate | log.Ltime)
+		log.Println("Logging initialized to stdout due to directory error")
+		return
+	}
+
+	// Open or create log file
+	logFile, err := os.OpenFile(config.Paths.Log, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		log.Printf("Error opening log file %s: %v", config.Paths.Log, err)
+		log.SetOutput(os.Stdout)
+		log.SetFlags(log.Ldate | log.Ltime)
+		log.Println("Logging initialized to stdout due to file error")
+		return
+	}
+
+	// Set output to file
+	log.SetOutput(logFile)
+	log.SetFlags(log.Ldate | log.Ltime)
+	log.Printf("Logging initialized to %s", config.Paths.Log)
+}
+
+// logMessage - wrapper for logging with levels
+func logMessage(message string, level string) {
+	// Default to INFO if level not specified
+	if level == "" {
+		level = "INFO"
+	}
+
+	// Only log if level is sufficient based on config
+	configLevel := strings.ToUpper(config.Logging.Level)
+	if configLevel == "" {
+		configLevel = "INFO" // Default to INFO
+	}
+	
+	messageLevel := strings.ToUpper(level)
+	
+	// Determine if message should be logged based on level
+	shouldLog := false
+	
+	switch configLevel {
+	case "DEBUG":
+		shouldLog = true // Log everything
+	case "INFO":
+		shouldLog = messageLevel != "DEBUG" // Log INFO, WARNING, ERROR
+	case "WARNING":
+		shouldLog = messageLevel == "WARNING" || messageLevel == "ERROR" // Log WARNING, ERROR
+	case "ERROR":
+		shouldLog = messageLevel == "ERROR" // Log only ERROR
+	}
+	
+	if shouldLog {
+		log.Printf("[%s] %s", messageLevel, message)
+	}
+}
+
+// Function to convert string encoding with enhanced charset support
+func convertEncoding(input string, sourceEnc string, targetEnc string) (string, error) {
+	if strings.EqualFold(sourceEnc, targetEnc) {
+		return input, nil
+	}
+	
+	var enc encoding.Encoding
+	var err error
+	
+	// Enhanced encoding handling with more European charsets
+	switch strings.ToLower(sourceEnc) {
+	case "utf-8", "us-ascii":
+		return input, nil
+	case "iso-8859-1", "latin1":
+		enc = charmap.ISO8859_1
+	case "iso-8859-2", "latin2":
+		enc = charmap.ISO8859_2
+	case "iso-8859-3", "latin3":
+		enc = charmap.ISO8859_3
+	case "iso-8859-4", "latin4":
+		enc = charmap.ISO8859_4
+	case "iso-8859-5":
+		enc = charmap.ISO8859_5
+	case "iso-8859-9", "latin5":
+		enc = charmap.ISO8859_9
+	case "iso-8859-10", "latin6":
+		enc = charmap.ISO8859_10
+	case "iso-8859-13", "latin7":
+		enc = charmap.ISO8859_13
+	case "iso-8859-14", "latin8":
+		enc = charmap.ISO8859_14
+	case "iso-8859-15", "latin9":
+		enc = charmap.ISO8859_15
+	case "iso-8859-16":
+		enc = charmap.ISO8859_16
+	case "windows-1250":
+		enc = charmap.Windows1250
+	case "windows-1251":
+		enc = charmap.Windows1251
+	case "windows-1252":
+		enc = charmap.Windows1252
+	case "windows-1253":
+		enc = charmap.Windows1253
+	case "windows-1254":
+		enc = charmap.Windows1254
+	case "windows-1255":
+		enc = charmap.Windows1255
+	case "windows-1256":
+		enc = charmap.Windows1256
+	case "windows-1257":
+		enc = charmap.Windows1257
+	case "windows-1258":
+		enc = charmap.Windows1258
+	case "koi8-r":
+		enc = charmap.KOI8R
+	case "koi8-u":
+		enc = charmap.KOI8U
+	default:
+		// Try to get from htmlindex
+		enc, err = htmlindex.Get(sourceEnc)
+		if err != nil {
+			fallbackCharset := "iso-8859-1" // Default fallback
+			if config.Encoding.FallbackCharset != "" {
+				fallbackCharset = config.Encoding.FallbackCharset
+			}
+			logMessage(fmt.Sprintf("Encoding %s not recognized, using fallback %s", 
+				sourceEnc, fallbackCharset), "WARNING")
+			// Use fallback charset
+			enc, _ = htmlindex.Get(fallbackCharset)
+		}
+	}
+	
+	// Convert to UTF-8
+	reader := transform.NewReader(strings.NewReader(input), enc.NewDecoder())
+	result, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return input, err
+	}
+	
+	return string(result), nil
+}
+
+// Function to decode content based on Content-Transfer-Encoding
+func decodeTransferEncoding(content string, encoding string) (string, error) {
+    encoding = strings.ToLower(encoding)
+    
+    switch encoding {
+    case "quoted-printable":
+        reader := quotedprintable.NewReader(strings.NewReader(content))
+        decoded, err := ioutil.ReadAll(reader)
+        if err != nil {
+            return content, err
+        }
+        logMessage(fmt.Sprintf("Successfully decoded quoted-printable content"), "DEBUG")
+        return string(decoded), nil
+        
+    case "base64":
+        // Remove any whitespace from base64 content which can cause decoding errors
+        content = regexp.MustCompile(`\s+`).ReplaceAllString(content, "")
+        decoded, err := base64.StdEncoding.DecodeString(content)
+        if err != nil {
+            return content, err
+        }
+        logMessage(fmt.Sprintf("Successfully decoded base64 content"), "DEBUG")
+        return string(decoded), nil
+        
+    case "7bit", "8bit", "binary", "":
+        // No decoding needed
+        return content, nil
+        
+    default:
+        logMessage(fmt.Sprintf("Unknown Content-Transfer-Encoding: %s", encoding), "WARNING")
+        return content, nil
+    }
+}
+
+// Determine appropriate Content-Transfer-Encoding for a message
+func determineContentTransferEncoding(content string) string {
+    // Check if content has any non-ASCII characters
+    has8bitChars := false
+    hasBinaryChars := false
+    maxLineLength := 0
+    
+    lines := strings.Split(content, "\r\n")
+    for _, line := range lines {
+        // Track maximum line length
+        if len(line) > maxLineLength {
+            maxLineLength = len(line)
+        }
+        
+        for _, c := range line {
+            // Check for 8-bit characters (non-ASCII)
+            if c > 127 {
+                has8bitChars = true
+            }
+            
+            // Check for control characters except tab, CR, LF
+            if c < 32 && c != 9 && c != 10 && c != 13 {
+                hasBinaryChars = true
+            }
+        }
+    }
+    
+    // Determine encoding based on content characteristics
+    if hasBinaryChars || maxLineLength > 998 {
+        return "base64"  // Binary content or very long lines need base64
+    } else if has8bitChars {
+        return "8bit"    // Non-ASCII content uses 8bit
+    } else {
+        return "7bit"    // Plain ASCII with reasonable line length
+    }
+}
+
+// Normalize email format to ensure NNTP compatibility with enhanced UTF-8 support
+func normalizeEmailFormat(message string) string {
+	// Detect original encoding
+	contentTypeRegex := regexp.MustCompile(`(?i)Content-Type:[^\n]*charset=["']?([^"'\r\n;]+)`)
+	matches := contentTypeRegex.FindStringSubmatch(message)
+	
+	sourceEncoding := "utf-8" // Default to UTF-8
+	if len(matches) > 1 {
+		sourceEncoding = strings.ToLower(matches[1])
+		logMessage(fmt.Sprintf("Detected charset in Content-Type: %s", sourceEncoding), "DEBUG")
+	}
+	
+	// Find Content-Transfer-Encoding
+	transferEncodingRegex := regexp.MustCompile(`(?i)Content-Transfer-Encoding:\s*([^\r\n]+)`)
+	transferEncoding := "7bit" // Default
+	matches = transferEncodingRegex.FindStringSubmatch(message)
+	if len(matches) > 1 {
+		transferEncoding = strings.TrimSpace(matches[1])
+		logMessage(fmt.Sprintf("Found Content-Transfer-Encoding: %s", transferEncoding), "DEBUG")
+	}
+
+	// Ensure proper header-body separation (exactly one blank line)
+	headerEnd := strings.Index(message, "\r\n\r\n")
+	if headerEnd == -1 {
+		headerEnd = strings.Index(message, "\n\n")
+	}
+
+	if headerEnd != -1 {
+		headers := message[:headerEnd]
+		body := message[headerEnd:]
+		
+		// Normalize header-body separation to exactly one blank line
+		body = regexp.MustCompile(`^\r?\n\r?\n+`).ReplaceAllString(body, "\r\n\r\n")
+		
+		// Decode body according to transfer encoding
+		if len(body) > 4 { // Ensure there's content after \r\n\r\n
+			decodedBody, err := decodeTransferEncoding(body[4:], transferEncoding) // Skip \r\n\r\n
+			if err != nil {
+				logMessage(fmt.Sprintf("Error decoding body with %s encoding: %v", transferEncoding, err), "WARNING")
+			} else {
+				// Ensure no leading blank lines in body content
+				decodedBody = strings.TrimLeft(decodedBody, "\r\n \t")
+				body = "\r\n\r\n" + decodedBody
+				logMessage(fmt.Sprintf("Successfully decoded message body using %s encoding", transferEncoding), "DEBUG")
+			}
+		}
+		
+		message = headers + body
+	}
+	
+	// Convert to UTF-8 if needed or force_utf8 is true
+	forceUtf8 := config.Encoding.ForceUtf8
+	if sourceEncoding != "utf-8" && (forceUtf8 || sourceEncoding != "us-ascii") {
+		converted, err := convertEncoding(message, sourceEncoding, "utf-8")
+		if err != nil {
+			logMessage(fmt.Sprintf("Error during encoding conversion: %v", err), "WARNING")
+		} else {
+			message = converted
+			logMessage(fmt.Sprintf("Message converted from %s to UTF-8", sourceEncoding), "DEBUG")
+		}
+	}
+	
+	// Replace problematic characters
+	message = strings.ReplaceAll(message, "\u2018", "'") // Left single quote
+	message = strings.ReplaceAll(message, "\u2019", "'") // Right single quote
+	message = strings.ReplaceAll(message, "\u201C", "\"") // Left double quote
+	message = strings.ReplaceAll(message, "\u201D", "\"") // Right double quote
+	message = strings.ReplaceAll(message, "\u2026", "...") // Ellipsis
+	message = strings.ReplaceAll(message, "\u2013", "-") // En dash
+	message = strings.ReplaceAll(message, "\u2014", "-") // Em dash
+	
+	// Process HTML content if present
+	if strings.Contains(message, "<pre") || strings.Contains(message, "<html") {
+		logMessage("HTML content detected, converting to plain text", "INFO")
+		
+		// Remove common HTML tags
+		htmlTags := []string{"<pre[^>]*>", "</pre>", "<html[^>]*>", "</html>", 
+			"<body[^>]*>", "</body>", "<div[^>]*>", "</div>", "<p[^>]*>", "</p>",
+			"<br[^>]*>", "<span[^>]*>", "</span>"}
+		for _, tag := range htmlTags {
+			re := regexp.MustCompile(tag)
+			message = re.ReplaceAllString(message, "")
+		}
+		
+		// Convert common HTML entities
+		message = strings.ReplaceAll(message, "&quot;", "\"")
+		message = strings.ReplaceAll(message, "&apos;", "'")
+		message = strings.ReplaceAll(message, "&lt;", "<")
+		message = strings.ReplaceAll(message, "&gt;", ">")
+		message = strings.ReplaceAll(message, "&amp;", "&")
+		message = strings.ReplaceAll(message, "&nbsp;", " ")
+	}
+	
+	// Handle MIME-Version header
+	// Remove existing MIME-Version headers to prevent malformed ones
+	mimeVersionRegex := regexp.MustCompile(`(?i)MIME-Version:.*\r?\n`)
+	message = mimeVersionRegex.ReplaceAllString(message, "")
+	
+	// Add our own correctly formatted MIME-Version header
+	headerEnd = strings.Index(message, "\r\n\r\n")
+	if headerEnd == -1 {
+		headerEnd = strings.Index(message, "\n\n")
+	}
+	
+	if headerEnd != -1 {
+		message = message[:headerEnd] + 
+				"\r\nMIME-Version: 1.0\r\n" + 
+				message[headerEnd:]
+		logMessage("Added standardized MIME-Version header", "DEBUG")
+	}
+	
+	// Ensure Content-Type exists with UTF-8 charset
+	if !regexp.MustCompile(`(?i)Content-Type:`).MatchString(message) {
+		// Find the end of headers
+		headerEnd := strings.Index(message, "\r\n\r\n")
+		if headerEnd == -1 {
+			headerEnd = strings.Index(message, "\n\n")
+			if headerEnd != -1 {
+				// Insert before empty line
+				message = message[:headerEnd] + 
+						"\r\nContent-Type: text/plain; charset=utf-8\r\n" + 
+						message[headerEnd:]
+			}
+		} else {
+			// Insert before empty line
+			message = message[:headerEnd] + 
+					"\r\nContent-Type: text/plain; charset=utf-8\r\n" + 
+					message[headerEnd:]
+		}
+	} else {
+		// Replace existing encoding with UTF-8
+		message = regexp.MustCompile(`(?i)(Content-Type:[^\n]*charset=)["']?[^"'\r\n;]+`).
+			ReplaceAllString(message, "${1}utf-8")
+	}
+	
+	// Ensure there's a blank line between headers and body (critical for NNTP)
+	headerBodySeparator := "\r\n\r\n"
+	if !strings.Contains(message, headerBodySeparator) {
+		// Find where headers end (first blank line)
+		parts := strings.SplitN(message, "\n\n", 2)
+		if len(parts) == 2 {
+			// Rebuild with correct \r\n
+			message = strings.ReplaceAll(parts[0], "\n", "\r\n") + 
+					"\r\n\r\n" + 
+					strings.ReplaceAll(parts[1], "\n", "\r\n")
+			logMessage("Normalized email format with correct header-body separator", "DEBUG")
+		}
+	}
+	
+	// Ensure all lines end with \r\n (standard NNTP format)
+	message = strings.ReplaceAll(message, "\r\n", "\n")  // First normalize to \n
+	message = strings.ReplaceAll(message, "\n", "\r\n")  // Then convert all \n to \r\n
+	
+	// Check for lines starting with "." (in NNTP protocol, a single "." indicates end of message)
+	lines := strings.Split(message, "\r\n")
+	for i, line := range lines {
+		if line == "." {
+			lines[i] = ".."
+			logMessage("Found single dot line, replaced with '..'", "DEBUG")
+		} else if strings.HasPrefix(line, ".") && line != ".." {
+			lines[i] = "." + line
+			logMessage("Found line starting with '.', doubled it", "DEBUG")
+		}
+	}
+	
+	return strings.Join(lines, "\r\n")
+}
+
+// Parse recipient similar to the Python version
+func parseRecipient(user string) (string, string, bool) {
+	// Extract domain part if present
+	if idx := strings.Index(user, "@"); idx != -1 {
+		user = user[:idx]
+	}
+
+	// Regular expression to match mail2news format
+	re := regexp.MustCompile(`(mail2news|mail2news_nospam)-([0-9]{8})-(.*)`)
+	matches := re.FindStringSubmatch(user)
+	
+	if matches == nil {
+		logMessage("Badly formatted recipient. Rejecting message.", "ERROR")
+		os.Exit(0)
+	}
+
+	recipient := matches[1]
+	timestamp := matches[2]
+	newsgroups := matches[3]
+	
+	// Replace = separator with commas
+	newsgroups = strings.ReplaceAll(newsgroups, "=", ",")
+	
+	// Check for nospam directive
+	nospam := false
+	if recipient == "mail2news_nospam" {
+		logMessage("Message includes a nospam directive. Will munge headers accordingly.", "INFO")
+		nospam = true
+	}
+	
+	return timestamp, newsgroups, nospam
+}
+
+// Validate timestamp
+func validateStamp(stamp string) bool {
+	// Parse the stamp into a time.Time
+	layout := "20060102"
+	parsedTime, err := time.Parse(layout, stamp)
+	if err != nil {
+		logMessage(fmt.Sprintf("Malformed date element: %v. Rejecting message.", err), "ERROR")
+		os.Exit(0)
+	}
+
+	// Get current time and calculate boundaries
+	now := time.Now().UTC()
+	beforeTime := now.Add(-time.Duration(config.Thresholds.HoursPast) * time.Hour)
+	afterTime := now.Add(time.Duration(config.Thresholds.HoursFuture) * time.Hour)
+
+	// Check if within bounds
+	if parsedTime.After(beforeTime) && parsedTime.Before(afterTime) {
+		logMessage(fmt.Sprintf("Timestamp (%s) is valid and within bounds.", stamp), "DEBUG")
+		return true
+	}
+
+	logMessage(fmt.Sprintf("Timestamp (%s) is out of bounds. Rejecting message.", stamp), "ERROR")
+	os.Exit(0)
+	return false
+}
+
+// Validate newsgroups
+func ngvalidate(newsgroups string) string {
+	newsgroups = strings.TrimRight(newsgroups, ",")
+	groups := strings.Split(newsgroups, ",")
+	
+	var goodng []string
+	
+	modfile := filepath.Join(config.Paths.Lib, "moderated.db")
+	// Check if moderation file exists
+	if _, err := os.Stat(modfile); err == nil {
+		logMessage(fmt.Sprintf("Moderated groups file found at %s", modfile), "DEBUG")
+	}
+	
+	// Check each group format
+	re := regexp.MustCompile(`[a-z][a-z0-9]+(\.[0-9a-z-+_]+)+$`)
+	for _, ng := range groups {
+		ng = strings.TrimSpace(ng)
+		
+		if re.MatchString(ng) {
+			// Check for duplicates
+			isDuplicate := false
+			for _, existingNg := range goodng {
+				if existingNg == ng {
+					logMessage(fmt.Sprintf("%s is duplicated in Newsgroups header. Dropping one instance of it.", ng), "INFO")
+					isDuplicate = true
+					break
+				}
+			}
+			
+			if !isDuplicate {
+				// Add to good newsgroups
+				goodng = append(goodng, ng)
+			}
+		} else {
+			logMessage(fmt.Sprintf("%s is not a validated newsgroup, ignoring.", ng), "WARNING")
+		}
+	}
+	
+	// No valid newsgroups
+	if len(goodng) < 1 {
+		logMessage("Message has no valid newsgroups. Rejecting it.", "ERROR")
+		os.Exit(0)
+	}
+	
+	// Check crosspost limit
+	if len(goodng) > config.Thresholds.MaxCrossposts {
+		logMessage(fmt.Sprintf("Message contains %d newsgroups, exceeding crosspost limit of %d. Rejecting.",
+			len(goodng), config.Thresholds.MaxCrossposts), "ERROR")
+		os.Exit(0)
+	}
+	
+	header := strings.Join(goodng, ",")
+	logMessage(fmt.Sprintf("Validated Newsgroups header is: %s", header), "INFO")
+	return header
+}
+
+// Generate message ID with guaranteed uniqueness to prevent duplicates
+func messageID(rightPart string) string {
+	// Override the domain with our fixed privacy-enhanced domain if needed
+	if rightPart == "" {
+		rightPart = "tcpreset-nospam"
+	}
+
+	// Create unique ID using multiple uniqueness factors:
+	// 1. Current timestamp with microsecond precision
+	// 2. Random string
+	// 3. Hash of message content (first 8 chars)
+	now := time.Now().UTC()
+	
+	// Format: YYYYMMDDHHMMSS.microseconds.randomstring.contentHash@domain
+	randomStr := generateRandomString(8)
+	uniqHash := fmt.Sprintf("%d", rand.Int63()) // Additional randomness
+	
+	leftPart := fmt.Sprintf("%s.%d.%s.%s", 
+		now.Format("20060102150405"),     // Standard timestamp
+		now.UnixNano() % 1000000,         // Microseconds for uniqueness 
+		randomStr,                        // Random string
+		uniqHash[:8])                     // Random hash prefix
+		
+	return "<" + leftPart + "@" + rightPart + ">"
+}
+
+// Generate random string for message ID
+func generateRandomString(length int) string {
+	const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
+	b := make([]byte, length)
+	for i := range b {
+		b[i] = charset[rand.Intn(len(charset))]
+	}
+	return string(b)
+}
+
+// Generate content hash for message content
+func generateContentHash(content string) string {
+	hash := sha256.Sum256([]byte(content))
+	return fmt.Sprintf("%x", hash)
+}
+
+// Check blacklists
+func blacklistCheck(badFile string, text string) string {
+	filename := filepath.Join(config.Paths.Etc, badFile)
+	badList := file2list(filename)
+	
+	if len(badList) > 0 {
+		pattern := strings.Join(badList, "|")
+		re, err := regexp.Compile(pattern)
+		if err != nil {
+			logMessage(fmt.Sprintf("Error compiling regex from %s: %v", badFile, err), "ERROR")
+			return ""
+		}
+		
+		if match := re.FindString(text); match != "" {
+			return match
+		}
+	}
+	
+	return ""
+}
+
+// Convert file to list
+func file2list(filename string) []string {
+	var items []string
+	
+	if _, err := os.Stat(filename); os.IsNotExist(err) {
+		return items
+	}
+	
+	file, err := os.Open(filename)
+	if err != nil {
+		return items
+	}
+	defer file.Close()
+	
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		line := scanner.Text()
+		if idx := strings.Index(line, "#"); idx != -1 {
+			line = line[:idx]
+		}
+		line = strings.TrimSpace(line)
+		if line != "" {
+			items = append(items, line)
+		}
+	}
+	
+	return items
+}
+
+// min function for Go versions < 1.21
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}
+
+// Parse From header
+func fromParse(fromHdr string) (string, string) {
+	var name, addy string
+	matched := false
+	
+	// Pattern 1: "Name <user@example.com>"
+	re1 := regexp.MustCompile(`([^<>]*)<([^<>\s]+@[^<>\s]+)>$`)
+	if matches := re1.FindStringSubmatch(fromHdr); matches != nil {
+		name = matches[1]
+		addy = matches[2]
+		matched = true
+	}
+	
+	// Pattern 2: "user@example.com (Name)"
+	re2 := regexp.MustCompile(`([^<>\s]+@[^<>\s]+)\s+\(([^\(\)]*)\)$`)
+	if !matched {
+		if matches := re2.FindStringSubmatch(fromHdr); matches != nil {
+			name = matches[2]
+			addy = matches[1]
+			matched = true
+		}
+	}
+	
+	// Pattern 3: "user@example.com"
+	re3 := regexp.MustCompile(`([^<>\s]+@[^<>\s]+)$`)
+	if !matched {
+		if matches := re3.FindStringSubmatch(fromHdr); matches != nil {
+			name = ""
+			addy = matches[1]
+		}
+	}
+	
+	if addy != "" {
+		addy = strings.ReplaceAll(addy, ".", "<DOT>")
+		addy = strings.ReplaceAll(addy, "@", "<AT>")
+	}
+	
+	return name, addy
+}
+
+// Parse message
+func msgParse(message string) (string, string) {
+	// Display first characters of received message for debug (reduced logging)
+	firstChars := min(200, len(message))
+	logMessage(fmt.Sprintf("First %d characters of received message: %s", firstChars, message[:firstChars]), "DEBUG")
+	
+	// Remove ALL Unix mailbox style "From " lines
+	lines := strings.Split(message, "\n")
+	var cleanedLines []string
+	fromLinesRemoved := 0
+	
+	for _, line := range lines {
+		// Look for "From " lines that don't contain ":" (characteristic of mailbox lines)
+		if strings.HasPrefix(line, "From ") && !strings.Contains(line, ":") {
+			fromLinesRemoved++
+			continue // Skip this line
+		}
+		cleanedLines = append(cleanedLines, line)
+	}
+	
+	if fromLinesRemoved > 0 {
+		logMessage(fmt.Sprintf("Removed %d mailbox-style 'From ' lines", fromLinesRemoved), "DEBUG")
+		message = strings.Join(cleanedLines, "\n")
+	} else {
+		logMessage("No 'From ' lines found to remove", "DEBUG")
+	}
+	
+	logMessage("Skipping history file write due to permission issues", "DEBUG")
+	
+	// Before parsing the email, ensure it has no other format anomalies
+	message = normalizeEmailFormat(message)
+	
+	// Parse the email
+	msg, err := mail.ReadMessage(strings.NewReader(message))
+	if err != nil {
+		logMessage(fmt.Sprintf("Error parsing message: %v", err), "ERROR")
+		os.Exit(1)
+	}
+	
+	// Get message ID first to check if we've already processed it
+	messageId := msg.Header.Get("Message-ID")
+	// If message ID is present, check cache to see if we've already processed it
+	if messageId != "" && messageCache != nil {
+		if messageCache.IsProcessed(messageId) {
+			logMessage(fmt.Sprintf("Message %s has already been processed recently. Skipping.", messageId), "INFO")
+			os.Exit(0) // Exit with success status
+		}
+	}
+	
+	// Only log headers at DEBUG level
+	if config.Logging.Level == "DEBUG" {
+		logMessage("=== Headers received in email ===", "DEBUG")
+		for k, v := range msg.Header {
+			logMessage(fmt.Sprintf("Header: %s = %v", k, v), "DEBUG")
+		}
+	}
+	
+	// Check for authentication headers
+	if xHashcash := msg.Header.Get("X-Hashcash"); xHashcash == "" {
+		logMessage("WARNING: X-Hashcash not found in email!", "WARNING")
+	}
+	
+	if xEd25519Pub := msg.Header.Get("X-Ed25519-Pub"); xEd25519Pub == "" {
+		logMessage("WARNING: X-Ed25519-Pub not found in email!", "WARNING")
+	}
+	
+	if xEd25519Sig := msg.Header.Get("X-Ed25519-Sig"); xEd25519Sig == "" {
+		logMessage("WARNING: X-Ed25519-Sig not found in email!", "WARNING")
+	}
+	
+	// Get message body for Content-Transfer-Encoding analysis
+	body, err := ioutil.ReadAll(msg.Body)
+	if err != nil {
+		logMessage(fmt.Sprintf("Error reading message body: %v", err), "ERROR")
+		os.Exit(1)
+	}
+	
+	// Ensure there's a body, even minimal
+	bodyContent := string(body)
+	if len(bodyContent) == 0 || len(strings.TrimSpace(bodyContent)) == 0 {
+		logMessage("WARNING: Empty message body detected! Adding placeholder text.", "WARNING")
+		bodyContent = "This message had no content."
+		body = []byte(bodyContent)
+	}
+	
+	// Analyze body for Content-Transfer-Encoding - but only if not already set
+	if msg.Header.Get("Content-Transfer-Encoding") == "" {
+		transferEncoding := determineContentTransferEncoding(bodyContent)
+		msg.Header["Content-Transfer-Encoding"] = []string{transferEncoding}
+		logMessage(fmt.Sprintf("Analyzed and set Content-Transfer-Encoding to %s", transferEncoding), "INFO")
+	}
+		
+	// Process Message-ID if not already present
+	if messageId == "" {
+		// Create a content hash to help make a unique Message ID
+		contentHash := generateContentHash(bodyContent)
+		
+		// Call messageID function passing configuration parameter and content hash
+		messageId = messageID(config.NNTP.MessageID + "." + contentHash[:8])
+		
+		msg.Header["Message-ID"] = []string{messageId}
+		logMessage(fmt.Sprintf("Processing message with no Message-ID. Assigned: %s", messageId), "INFO")
+	} else {
+		logMessage(fmt.Sprintf("Processing message: %s", messageId), "INFO")
+	}
+	
+	// Process Date header
+	if msg.Header.Get("Date") == "" {
+		logMessage("Message has no Date header. Inserting current timestamp.", "INFO")
+		msg.Header["Date"] = []string{time.Now().Format(time.RFC1123Z)}
+	}
+	
+	// Process From header
+	fromHeader := msg.Header.Get("From")
+	if fromHeader != "" {
+		if match := blacklistCheck("bad_from", fromHeader); match != "" {
+			logMessage(fmt.Sprintf("From header matches '%s'. Rejecting.", match), "ERROR")
+			os.Exit(1)
+		}
+	} else {
+		logMessage("Message has no From header. Inserting a null one.", "INFO")
+		msg.Header["From"] = []string{config.NNTP.DefaultFrom}
+	}
+	
+	// Minimal References handling
+	if refs := msg.Header.Get("References"); refs != "" {
+		logMessage(fmt.Sprintf("THREADING - Original References: %s", refs), "INFO")
+		
+		// Only add angle brackets if completely missing, otherwise keep as is
+		if !strings.Contains(refs, "<") {
+			refs = "<" + refs
+		}
+		if !strings.Contains(refs, ">") {
+			refs = refs + ">"
+		}
+		
+		// Keep References as is (no splitting/rejoining)
+		msg.Header["References"] = []string{refs}
+		logMessage(fmt.Sprintf("THREADING - Using References with minimal changes: %s", refs), "INFO")
+		
+		// Set In-Reply-To to match References exactly
+		msg.Header["In-Reply-To"] = []string{refs}
+		logMessage("THREADING - Set In-Reply-To to match References exactly", "INFO")
+	} else if strings.HasPrefix(strings.ToLower(msg.Header.Get("Subject")), "re:") {
+		logMessage("WARNING - Message has Re: in subject but no References header!", "WARNING")
+	}
+
+	// Check for poison headers
+	poisonFile := filepath.Join(config.Paths.Etc, "headers_poison")
+	poisonHeaders := file2list(poisonFile)
+	for _, header := range poisonHeaders {
+		if msg.Header.Get(header) != "" {
+			logMessage(fmt.Sprintf("Message contains a blacklisted %s header. Rejecting it.", header), "ERROR")
+			os.Exit(0)
+		}
+	}
+	
+	// Get recipient info
+	var recipient string
+	if to := msg.Header.Get("X-Original-To"); to != "" {
+		recipient = to
+	} else if to := msg.Header.Get("To"); to != "" {
+		recipient = to
+	} else {
+		recipient = "mail2news@m2n.mixmin.net"
+		logMessage(fmt.Sprintf("Could not find recipient info. Guessing %s.", recipient), "WARNING")
+	}
+	
+	if !strings.HasPrefix(recipient, "mail2news") {
+		logMessage(fmt.Sprintf("Recipient %s is not us.", recipient), "ERROR")
+		os.Exit(2)
+	}
+	
+	// Process newsgroups
+	nospam := false
+	var dest string
+	
+	if ng := msg.Header.Get("Newsgroups"); ng != "" {
+		dest = ng
+		delete(msg.Header, "Newsgroups")
+		logMessage(fmt.Sprintf("Message has a Newsgroups header of %s", dest), "INFO")
+		
+		if strings.HasPrefix(recipient, "mail2news_nospam") {
+			nospam = true
+			logMessage("Message includes a nospam directive. Will munge From headers accordingly.", "INFO")
+		}
+	} else {
+		logMessage("No Newsgroups header, trying to parse recipient information", "INFO")
+		var stamp string
+		stamp, dest, nospam = parseRecipient(recipient)
+		
+		if !validateStamp(stamp) {
+			logMessage("No Newsgroups header or valid recipient. Rejecting message.", "ERROR")
+			os.Exit(0)
+		}
+	}
+	
+	// Validate newsgroups
+	validatedGroups := ngvalidate(dest)
+	msg.Header["Newsgroups"] = []string{validatedGroups}
+	
+	// Check for blacklisted newsgroups
+	if match := blacklistCheck("bad_groups", validatedGroups); match != "" {
+		logMessage(fmt.Sprintf("Newsgroups header matches '%s'. Rejecting.", match), "ERROR")
+		os.Exit(1)
+	}
+	
+	// Handle nospam mode
+	if nospam {
+		name, addy := fromParse(msg.Header.Get("From"))
+		if addy != "" {
+			delete(msg.Header, "Author-Supplied-Address")
+			delete(msg.Header, "From")
+			msg.Header["Author-Supplied-Address"] = []string{addy}
+			msg.Header["From"] = []string{name + "<Use-Author-Supplied-Address-Header@[127.1]>"}
+		}
+	}
+	
+	// Process Subject header
+	if subj := msg.Header.Get("Subject"); subj != "" {
+		logMessage(fmt.Sprintf("Subject: %s", subj), "INFO")
+	} else {
+		logMessage("Message has no Subject header. Inserting a null one.", "INFO")
+		msg.Header["Subject"] = []string{"None"}
+	}
+	
+	// Check for Path header
+	if path := msg.Header.Get("Path"); path != "" {
+		logMessage(fmt.Sprintf("Message has a preloaded path header of %s", path), "DEBUG")
+	}
+	
+	// Strip headers
+	stripFile := filepath.Join(config.Paths.Etc, "headers_strip")
+	stripHeaders := file2list(stripFile)
+	
+	// Headers to always preserve
+	preserveHeaders := map[string]bool{
+		"X-Hashcash":     true,
+		"X-Ed25519-Pub":  true,
+		"X-Ed25519-Sig":  true,
+	}
+	
+	for _, header := range stripHeaders {
+		// Don't remove headers we want to preserve
+		if _, shouldPreserve := preserveHeaders[header]; !shouldPreserve {
+			if msg.Header.Get(header) != "" {
+				logMessage(fmt.Sprintf("Stripping header: %s", header), "DEBUG")
+				delete(msg.Header, header)
+			}
+		} else {
+			logMessage(fmt.Sprintf("Preserving authentication header: %s", header), "DEBUG")
+		}
+	}
+	
+	// Add gateway headers
+	msg.Header["Path"] = []string{config.NNTP.Path}
+	msg.Header["Organization"] = []string{"Tcpreset M2N Gateway"}
+	msg.Header["X-Gateway-Info"] = []string{
+		config.NNTP.InjectionHost + "; mail-complaints-to=" + config.NNTP.Contact,
+	}
+	
+	// Update User-Agent
+	delete(msg.Header, "User-Agent")
+	msg.Header["User-Agent"] = []string{fmt.Sprintf("mail2news-go v%s", VERSION)}
+	
+	// Fix MIME Version - ensure correctly formatted
+	delete(msg.Header, "MIME-Version")
+	delete(msg.Header, "Mime-Version")
+	msg.Header["MIME-Version"] = []string{"1.0"}
+	
+	// Log important threading headers only at INFO level
+	logMessage(fmt.Sprintf("THREADING CHECK - Message-ID: %s", msg.Header.Get("Message-ID")), "INFO")
+	logMessage(fmt.Sprintf("THREADING CHECK - References: %s", msg.Header.Get("References")), "INFO")
+	logMessage(fmt.Sprintf("THREADING CHECK - In-Reply-To: %s", msg.Header.Get("In-Reply-To")), "INFO")
+	
+	// Build the complete message
+	txtMsg := ""
+	
+	// Build headers - ensure all headers are correctly formatted
+	for k, vv := range msg.Header {
+		for _, v := range vv {
+			// Ensure header value is properly trimmed
+			v = strings.TrimSpace(v)
+			txtMsg += k + ": " + v + "\r\n"
+		}
+	}
+	
+	// Blank line between headers and body - CRITICAL for NNTP format
+	txtMsg += "\r\n"
+	
+	// Format the body with proper line endings and dot-stuffing
+	bodyStr := string(body)
+	
+	// Make sure body doesn't start with extra blank lines
+    bodyStr = strings.TrimLeft(bodyStr, "\r\n \t")
+	
+	// Normalize line endings in body
+	bodyStr = strings.ReplaceAll(bodyStr, "\r\n", "\n")
+	bodyStr = strings.ReplaceAll(bodyStr, "\n", "\r\n")
+	
+	// Handle lines starting with "."
+	bodyLines := strings.Split(bodyStr, "\r\n")
+	for i, line := range bodyLines {
+		if line == "." {
+			bodyLines[i] = ".."
+		} else if strings.HasPrefix(line, ".") && line != ".." {
+			bodyLines[i] = "." + line
+		}
+	}
+	bodyStr = strings.Join(bodyLines, "\r\n")
+	
+	txtMsg += bodyStr
+	
+	// Ensure message ends with CRLF
+	if !strings.HasSuffix(txtMsg, "\r\n") {
+		txtMsg += "\r\n"
+		logMessage("Added missing final CRLF to message", "DEBUG")
+	}
+	
+	size := len(txtMsg)
+	if size > config.Thresholds.MaxBytes {
+		logMessage(fmt.Sprintf("Message exceeds %d size limit. Rejecting.", config.Thresholds.MaxBytes), "ERROR")
+		os.Exit(1)
+	}
+	logMessage(fmt.Sprintf("Message is %d bytes", size), "INFO")
+
+	// Only log first 100 chars of message at INFO level
+	logPreview := min(100, len(txtMsg))
+	logMessage(fmt.Sprintf("Message prepared for delivery: %s...", txtMsg[:logPreview]), "INFO")
+	
+	// Mark this message as processed in our cache
+	if messageCache != nil && messageId != "" {
+		messageCache.MarkProcessed(messageId)
+	}
+	
+	return messageId, txtMsg
+}
+
+// Send news - improved to avoid duplicates
+func newsSend(mid string, content string) {
+	// Generate a content hash to help track server attempts
+	contentHash := generateContentHash(content)
+	logMessage(fmt.Sprintf("Generated content hash for message tracking: %s", contentHash[:16]), "DEBUG")
+	
+	// Implementation for NNTP sending
+	// Using Tor for all connections
+	hostFile := filepath.Join(config.Paths.Etc, "nntphosts")
+	hosts := file2list(hostFile)
+	
+	// Default Tor proxy address if not specified in config
+	torProxyAddr := "127.0.0.1:9050"
+	if config.NNTP.TorProxy != "" {
+		torProxyAddr = config.NNTP.TorProxy
+	}
+	
+	// Create a SOCKS5 dialer for Tor
+	torDialer, err := proxy.SOCKS5("tcp", torProxyAddr, nil, proxy.Direct)
+	if err != nil {
+		logMessage(fmt.Sprintf("Error creating Tor dialer: %v", err), "ERROR")
+		return
+	}
+	
+	// Track successful delivery and attempted hosts to prevent duplicates
+	successfulDelivery := false
+	attemptedHosts := make(map[string]bool)
+	
+	// Build the ordered server list
+	var serverList []string
+	
+	// Try primary onion server first if configured
+	if config.NNTP.PrimaryOnion != "" {
+		primaryServer := config.NNTP.PrimaryOnion
+		if !strings.Contains(primaryServer, ":") {
+			primaryServer += ":119" // Default NNTP port
+		}
+		serverList = append(serverList, primaryServer)
+	}
+	
+	// Add fallback server if configured and different from primary
+	if config.NNTP.FallbackServer != "" && config.NNTP.FallbackServer != config.NNTP.PrimaryOnion {
+		fallbackServer := config.NNTP.FallbackServer
+		if !strings.Contains(fallbackServer, ":") {
+			fallbackServer += ":119" // Default NNTP port
+		}
+		serverList = append(serverList, fallbackServer)
+	}
+	
+	// Add remaining servers from nntphosts file, avoiding duplicates
+	for _, host := range hosts {
+		if !strings.Contains(host, ":") {
+			host += ":119" // Default NNTP port
+		}
+		
+		// Skip if already in our list
+		alreadyAdded := false
+		for _, existingServer := range serverList {
+			if existingServer == host {
+				alreadyAdded = true
+				break
+			}
+		}
+		
+		if !alreadyAdded {
+			serverList = append(serverList, host)
+		}
+	}
+	
+	// Try each server in order until successful or all failed
+	for _, host := range serverList {
+		// Skip if already attempted
+		if attemptedHosts[host] {
+			logMessage(fmt.Sprintf("Skipping already attempted server: %s", host), "DEBUG")
+			continue
+		}
+		
+		attemptedHosts[host] = true
+		logMessage(fmt.Sprintf("Attempting delivery to %s", host), "INFO")
+		
+		// Attempt delivery with specific error handling for duplicates
+		err, isDuplicate := deliverViaTor(torDialer, host, mid, content)
+		
+		if err == nil {
+			logMessage(fmt.Sprintf("✓ %s successfully delivered to %s", mid, host), "INFO")
+			successfulDelivery = true
+			// Exit early if delivery succeeded
+			break
+		} else if isDuplicate {
+			// If message is a duplicate, count it as successful
+			logMessage(fmt.Sprintf("✓ %s already exists on %s (duplicate)", mid, host), "INFO")
+			successfulDelivery = true
+			// But continue to other servers to check storage status
+		} else {
+			logMessage(fmt.Sprintf("✗ Delivery to %s failed: %v", host, err), "WARNING")
+		}
+	}
+
+	if successfulDelivery {
+		logMessage("Mail2news message successfully delivered or already exists", "INFO")
+	} else {
+		logMessage("Mail2news message delivery FAILED to all available servers", "WARNING")
+	}
+}
+
+// Deliver message via Tor to a specific NNTP server
+// Returns (error, isDuplicate) - where isDuplicate indicates if the message was rejected as a duplicate
+func deliverViaTor(torDialer proxy.Dialer, host string, messageID string, content string) (error, bool) {
+	// Create a dialer with timeout
+	timeoutDialer := &net.Dialer{
+		Timeout: time.Duration(config.Thresholds.SocketTimeout) * time.Second,
+	}
+	
+	// Use Tor for .onion addresses or if always_use_tor is enabled
+	var conn net.Conn
+	var err error
+	
+	if strings.Contains(host, ".onion") || config.NNTP.AlwaysUseTor {
+		// Connect through Tor
+		logMessage(fmt.Sprintf("Connecting to %s via Tor proxy", host), "INFO")
+		conn, err = torDialer.Dial("tcp", host)
+	} else {
+		// Direct connection
+		logMessage(fmt.Sprintf("Connecting directly to %s", host), "INFO")
+		conn, err = timeoutDialer.Dial("tcp", host)
+	}
+	
+	if err != nil {
+		return fmt.Errorf("connection error: %v", err), false
+	}
+	defer conn.Close()
+	
+	// Set deadline for the entire conversation
+	deadline := time.Now().Add(time.Duration(config.Thresholds.SocketTimeout) * time.Second)
+	conn.SetDeadline(deadline)
+	
+	// Set up buffered reader/writer
+	reader := bufio.NewReader(conn)
+	writer := bufio.NewWriter(conn)
+	
+	// Read initial greeting
+	resp, err := reader.ReadString('\n')
+	if err != nil {
+		return fmt.Errorf("error reading greeting: %v", err), false
+	}
+	if !strings.HasPrefix(resp, "200 ") && !strings.HasPrefix(resp, "201 ") {
+		return fmt.Errorf("unexpected greeting: %s", resp), false
+	}
+	
+	respTrimmed := strings.TrimSpace(resp)
+	logMessage(fmt.Sprintf("Connected to %s, greeting: %s", host, respTrimmed), "INFO")
+	
+	// Send MODE READER for onion servers or if greeting indicates transit mode
+	if strings.Contains(host, ".onion") || strings.Contains(respTrimmed, "transit mode") {
+		logMessage(fmt.Sprintf("Sending MODE READER to %s", host), "INFO")
+		_, err = writer.WriteString("MODE READER\r\n")
+		if err != nil {
+			return fmt.Errorf("error sending MODE READER: %v", err), false
+		}
+		err = writer.Flush()
+		if err != nil {
+			return fmt.Errorf("error flushing MODE READER: %v", err), false
+		}
+		
+		// Read response to MODE READER
+		resp, err = reader.ReadString('\n')
+		if err != nil {
+			return fmt.Errorf("error reading MODE READER response: %v", err), false
+		}
+		
+		respTrimmed = strings.TrimSpace(resp)
+		logMessage(fmt.Sprintf("MODE READER response from %s: %s", host, respTrimmed), "INFO")
+		
+		// Check if the server supports READER mode
+		if !strings.HasPrefix(resp, "200") && !strings.HasPrefix(resp, "201") {
+			// If MODE READER failed, try to continue anyway
+			logMessage(fmt.Sprintf("MODE READER not supported on %s, continuing anyway", host), "WARNING")
+		}
+	}
+	
+	// Use POST protocol (more widely supported)
+	logMessage(fmt.Sprintf("Sending POST command to %s", host), "INFO")
+	_, err = writer.WriteString("POST\r\n")
+	if err != nil {
+		return fmt.Errorf("error sending POST: %v", err), false
+	}
+	err = writer.Flush()
+	if err != nil {
+		return fmt.Errorf("error flushing POST: %v", err), false
+	}
+	
+	// Read response to POST
+	resp, err = reader.ReadString('\n')
+	if err != nil {
+		return fmt.Errorf("error reading POST response: %v", err), false
+	}
+	
+	respTrimmed = strings.TrimSpace(resp)
+	logMessage(fmt.Sprintf("POST response from %s: %s", host, respTrimmed), "INFO")
+	
+	// Check if server accepted POST (code 340)
+	if strings.HasPrefix(resp, "340 ") {
+		// Send article content
+		logMessage(fmt.Sprintf("Server %s accepted POST, sending article", host), "INFO")
+		logMessage(fmt.Sprintf("Article sent to %s via POST, waiting for response...", host), "INFO")
+		_, err = writer.WriteString(content + "\r\n.\r\n")
+		if err != nil {
+			return fmt.Errorf("error sending article via POST: %v", err), false
+		}
+		err = writer.Flush()
+		if err != nil {
+			return fmt.Errorf("error flushing article via POST: %v", err), false
+		}
+		
+		// Read final response
+		resp, err = reader.ReadString('\n')
+		if err != nil {
+			return fmt.Errorf("error reading POST final response: %v", err), false
+		}
+		
+		respTrimmed = strings.TrimSpace(resp)
+		logMessage(fmt.Sprintf("POST final response from %s: %s", host, respTrimmed), "INFO")
+		
+		// Check if response indicates a duplicate (435 or 435 Duplicate or similar)
+		isDuplicate := strings.Contains(respTrimmed, "435") || strings.Contains(strings.ToLower(respTrimmed), "duplicate")
+		
+		// Check if article was accepted (code 240)
+		if strings.HasPrefix(resp, "240 ") {
+			// Send QUIT
+			writer.WriteString("QUIT\r\n")
+			writer.Flush()
+			return nil, false
+		}
+		
+		// Send QUIT
+		writer.WriteString("QUIT\r\n")
+		writer.Flush()
+		
+		return fmt.Errorf("article rejected via POST: %s", respTrimmed), isDuplicate
+	} else {
+		return fmt.Errorf("POST not accepted: %s", respTrimmed), false
+	}
+}
+
+func main() {
+	// Initial log to signal startup
+	fmt.Printf("CANARY: Mail2News version %s with enhanced UTF-8 support is running!\n", VERSION)
+	
+	// Acquire lock to prevent multiple instances
+	if !acquireLock() {
+		fmt.Println("Another instance of mail2news is already running. Exiting.")
+		os.Exit(0)
+	}
+	defer releaseLock() // Release lock when program exits
+	
+	// Seed random number generator
+	rand.Seed(time.Now().UnixNano())
+	
+	// Initialize config
+	viper.SetConfigName("config")
+	viper.SetConfigType("yaml")
+	viper.AddConfigPath("/etc/mail2news")
+	viper.AddConfigPath("$HOME/.mail2news")
+	viper.AddConfigPath(".")
+	
+	if err := viper.ReadInConfig(); err != nil {
+		log.Fatalf("Fatal error config file: %s", err)
+	}
+	
+	if err := viper.Unmarshal(&config); err != nil {
+		log.Fatalf("Unable to decode config: %s", err)
+	}
+	
+	// Set defaults for config if not specified
+	if config.Paths.Log == "" {
+		config.Paths.Log = "/var/log/mail2news/mail2news.log"
+	}
+	if config.Logging.Level == "" {
+		config.Logging.Level = "WARNING" // Default to WARNING level
+	}
+	if config.Encoding.FallbackCharset == "" {
+		config.Encoding.FallbackCharset = "iso-8859-1"
+	}
+	
+	// Initialize logging
+	initLogging()
+	
+	// Create message ID cache
+	messageCache = NewMessageIDCache(5*time.Minute, "/var/lib/mail2news/cache")
+	
+	logMessage(fmt.Sprintf("Mail2news version %s initialized with UTF-8 support", VERSION), "INFO")
+	fmt.Println("Type message here. Finish with Ctrl-D.")
+	
+	// Read message from stdin
+	message, err := ioutil.ReadAll(os.Stdin)
+	if err != nil {
+		logMessage(fmt.Sprintf("Error reading from stdin: %v", err), "ERROR")
+		os.Exit(1)
+	}
+	
+	// Parse and process message
+	mid, payload := msgParse(string(message))
+	
+	// Send to news servers
+	newsSend(mid, payload)
+	
+	logMessage("Mail2news program completed successfully", "INFO")
+}