zizi-al/sub2api
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(risk-control): exempt admins from moderation auto-ban

Browse Source
This commit is contained in:
wucm667 2026-06-03 09:33:37 +08:00
parent aa69e3947d
commit c40a74d983
2 changed files with 95 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
backend/internal/service/content_moderation.go
View File

@ -1656,6 +1656,11 @@ func (s *ContentModerationService) applyFlaggedAccountSideEffects(ctx context.Co
slog.Warn("content_moderation.ban_get_user_failed", "user_id", *log.UserID, "error", err)
return false
}
if user.IsAdmin() {
slog.Warn("content_moderation.autoban_skipped_admin", "user_id", *log.UserID, "role", user.Role, "count", count, "threshold", cfg.BanThreshold)
// TODO: Disable the triggering API key instead when API key mutation is available here.
return false
}
if user.Status != StatusDisabled {
user.Status = StatusDisabled
if err := s.userRepo.Update(ctx, user); err != nil {

90
backend/internal/service/content_moderation_test.go
View File

@ -1,9 +1,11 @@
package service
import (
"bytes"
"context"
"encoding/json"
"fmt"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
@ -1484,6 +1486,94 @@ func TestContentModerationCheck_HashBlockLogsDoNotIncreaseNextViolationCount(t *
require.Equal(t, 1, logs[1].ViolationCount)
}
func TestContentModerationAutoBanSkipsAdminAccount(t *testing.T) {
var slogOutput bytes.Buffer
previousLogger := slog.Default()
slog.SetDefault(slog.New(slog.NewTextHandler(&slogOutput, nil)))
t.Cleanup(func() {
slog.SetDefault(previousLogger)
})
cfg := defaultContentModerationConfig()
cfg.BanThreshold = 2
cfg.ViolationWindowHours = 24
userID := int64(1001)
repo := &contentModerationTestRepo{}
require.NoError(t, repo.CreateLog(context.Background(), newContentModerationFlaggedLog(userID)))
userRepo := &contentModerationTestUserRepo{user: &User{ID: userID, Role: RoleAdmin, Status: StatusActive}}
invalidator := &contentModerationTestAuthCacheInvalidator{}
svc := NewContentModerationService(nil, repo, nil, nil, userRepo, invalidator, nil)
svc.persistContentModerationLog(context.Background(), cfg, newContentModerationFlaggedLog(userID), "", false, true)
logs := requireContentModerationLogCount(t, repo, 2)
require.Equal(t, 2, logs[1].ViolationCount)
require.False(t, logs[1].AutoBanned)
require.Equal(t, StatusActive, userRepo.user.Status)
require.Empty(t, userRepo.updated)
require.Empty(t, invalidator.userIDs)
require.Contains(t, slogOutput.String(), "content_moderation.autoban_skipped_admin")
require.Contains(t, slogOutput.String(), "user_id=1001")
require.Contains(t, slogOutput.String(), "role=admin")
require.Contains(t, slogOutput.String(), "count=2")
require.Contains(t, slogOutput.String(), "threshold=2")
}
func TestContentModerationAutoBanDisablesRegularUserAtThreshold(t *testing.T) {
cfg := defaultContentModerationConfig()
cfg.BanThreshold = 2
cfg.ViolationWindowHours = 24
userID := int64(1001)
repo := &contentModerationTestRepo{}
require.NoError(t, repo.CreateLog(context.Background(), newContentModerationFlaggedLog(userID)))
userRepo := &contentModerationTestUserRepo{user: &User{ID: userID, Role: RoleUser, Status: StatusActive}}
invalidator := &contentModerationTestAuthCacheInvalidator{}
svc := NewContentModerationService(nil, repo, nil, nil, userRepo, invalidator, nil)
svc.persistContentModerationLog(context.Background(), cfg, newContentModerationFlaggedLog(userID), "", false, true)
logs := requireContentModerationLogCount(t, repo, 2)
require.Equal(t, 2, logs[1].ViolationCount)
require.True(t, logs[1].AutoBanned)
require.Len(t, userRepo.updated, 1)
require.Equal(t, StatusDisabled, userRepo.user.Status)
require.Equal(t, []int64{userID}, invalidator.userIDs)
}
func TestContentModerationAdminBelowBanThresholdRecordsViolationOnly(t *testing.T) {
cfg := defaultContentModerationConfig()
cfg.BanThreshold = 2
cfg.ViolationWindowHours = 24
userID := int64(1001)
repo := &contentModerationTestRepo{}
userRepo := &contentModerationTestUserRepo{user: &User{ID: userID, Role: RoleAdmin, Status: StatusActive}}
invalidator := &contentModerationTestAuthCacheInvalidator{}
svc := NewContentModerationService(nil, repo, nil, nil, userRepo, invalidator, nil)
svc.persistContentModerationLog(context.Background(), cfg, newContentModerationFlaggedLog(userID), "", false, true)
logs := requireContentModerationLogCount(t, repo, 1)
require.Equal(t, 1, logs[0].ViolationCount)
require.False(t, logs[0].AutoBanned)
require.Equal(t, StatusActive, userRepo.user.Status)
require.Empty(t, userRepo.updated)
require.Empty(t, invalidator.userIDs)
}
func newContentModerationFlaggedLog(userID int64) *ContentModerationLog {
return &ContentModerationLog{
UserID: &userID,
Action: ContentModerationActionBlock,
Flagged: true,
HighestCategory: "sexual",
HighestScore: 0.9,
CreatedAt: time.Now(),
}
}
func TestContentModerationCheck_PreBlockFlaggedWritesRedisHashCache(t *testing.T) {
requestCount := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {