diff --git a/backend/internal/service/content_moderation.go b/backend/internal/service/content_moderation.go index ee1fca41..42b909c9 100644 --- a/backend/internal/service/content_moderation.go +++ b/backend/internal/service/content_moderation.go @@ -1656,6 +1656,11 @@ func (s *ContentModerationService) applyFlaggedAccountSideEffects(ctx context.Co slog.Warn("content_moderation.ban_get_user_failed", "user_id", *log.UserID, "error", err) return false } + if user.IsAdmin() { + slog.Warn("content_moderation.autoban_skipped_admin", "user_id", *log.UserID, "role", user.Role, "count", count, "threshold", cfg.BanThreshold) + // TODO: Disable the triggering API key instead when API key mutation is available here. + return false + } if user.Status != StatusDisabled { user.Status = StatusDisabled if err := s.userRepo.Update(ctx, user); err != nil { diff --git a/backend/internal/service/content_moderation_test.go b/backend/internal/service/content_moderation_test.go index 6c6fef44..9cfdc1e4 100644 --- a/backend/internal/service/content_moderation_test.go +++ b/backend/internal/service/content_moderation_test.go @@ -1,9 +1,11 @@ package service import ( + "bytes" "context" "encoding/json" "fmt" + "log/slog" "net/http" "net/http/httptest" "strings" @@ -1484,6 +1486,94 @@ func TestContentModerationCheck_HashBlockLogsDoNotIncreaseNextViolationCount(t * require.Equal(t, 1, logs[1].ViolationCount) } +func TestContentModerationAutoBanSkipsAdminAccount(t *testing.T) { + var slogOutput bytes.Buffer + previousLogger := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&slogOutput, nil))) + t.Cleanup(func() { + slog.SetDefault(previousLogger) + }) + + cfg := defaultContentModerationConfig() + cfg.BanThreshold = 2 + cfg.ViolationWindowHours = 24 + + userID := int64(1001) + repo := &contentModerationTestRepo{} + require.NoError(t, repo.CreateLog(context.Background(), newContentModerationFlaggedLog(userID))) + userRepo := &contentModerationTestUserRepo{user: &User{ID: userID, Role: RoleAdmin, Status: StatusActive}} + invalidator := &contentModerationTestAuthCacheInvalidator{} + svc := NewContentModerationService(nil, repo, nil, nil, userRepo, invalidator, nil) + + svc.persistContentModerationLog(context.Background(), cfg, newContentModerationFlaggedLog(userID), "", false, true) + + logs := requireContentModerationLogCount(t, repo, 2) + require.Equal(t, 2, logs[1].ViolationCount) + require.False(t, logs[1].AutoBanned) + require.Equal(t, StatusActive, userRepo.user.Status) + require.Empty(t, userRepo.updated) + require.Empty(t, invalidator.userIDs) + require.Contains(t, slogOutput.String(), "content_moderation.autoban_skipped_admin") + require.Contains(t, slogOutput.String(), "user_id=1001") + require.Contains(t, slogOutput.String(), "role=admin") + require.Contains(t, slogOutput.String(), "count=2") + require.Contains(t, slogOutput.String(), "threshold=2") +} + +func TestContentModerationAutoBanDisablesRegularUserAtThreshold(t *testing.T) { + cfg := defaultContentModerationConfig() + cfg.BanThreshold = 2 + cfg.ViolationWindowHours = 24 + + userID := int64(1001) + repo := &contentModerationTestRepo{} + require.NoError(t, repo.CreateLog(context.Background(), newContentModerationFlaggedLog(userID))) + userRepo := &contentModerationTestUserRepo{user: &User{ID: userID, Role: RoleUser, Status: StatusActive}} + invalidator := &contentModerationTestAuthCacheInvalidator{} + svc := NewContentModerationService(nil, repo, nil, nil, userRepo, invalidator, nil) + + svc.persistContentModerationLog(context.Background(), cfg, newContentModerationFlaggedLog(userID), "", false, true) + + logs := requireContentModerationLogCount(t, repo, 2) + require.Equal(t, 2, logs[1].ViolationCount) + require.True(t, logs[1].AutoBanned) + require.Len(t, userRepo.updated, 1) + require.Equal(t, StatusDisabled, userRepo.user.Status) + require.Equal(t, []int64{userID}, invalidator.userIDs) +} + +func TestContentModerationAdminBelowBanThresholdRecordsViolationOnly(t *testing.T) { + cfg := defaultContentModerationConfig() + cfg.BanThreshold = 2 + cfg.ViolationWindowHours = 24 + + userID := int64(1001) + repo := &contentModerationTestRepo{} + userRepo := &contentModerationTestUserRepo{user: &User{ID: userID, Role: RoleAdmin, Status: StatusActive}} + invalidator := &contentModerationTestAuthCacheInvalidator{} + svc := NewContentModerationService(nil, repo, nil, nil, userRepo, invalidator, nil) + + svc.persistContentModerationLog(context.Background(), cfg, newContentModerationFlaggedLog(userID), "", false, true) + + logs := requireContentModerationLogCount(t, repo, 1) + require.Equal(t, 1, logs[0].ViolationCount) + require.False(t, logs[0].AutoBanned) + require.Equal(t, StatusActive, userRepo.user.Status) + require.Empty(t, userRepo.updated) + require.Empty(t, invalidator.userIDs) +} + +func newContentModerationFlaggedLog(userID int64) *ContentModerationLog { + return &ContentModerationLog{ + UserID: &userID, + Action: ContentModerationActionBlock, + Flagged: true, + HighestCategory: "sexual", + HighestScore: 0.9, + CreatedAt: time.Now(), + } +} + func TestContentModerationCheck_PreBlockFlaggedWritesRedisHashCache(t *testing.T) { requestCount := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {