diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fa7a51e6b..01af04b55 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -31,6 +31,10 @@ jobs: api/go.sum tests/go.sum + - name: Run Rate Limit Regression Tests + working-directory: ./api + run: go test -race ./pkg/services ./pkg/middlewares ./pkg/di -run TestRateLimit -count=1 + - name: Generate Firebase Credentials run: | bash tests/generate-firebase-credentials.sh tests/firebase-credentials.json diff --git a/api/.env.docker b/api/.env.docker index 235972f4d..928a3fce0 100644 --- a/api/.env.docker +++ b/api/.env.docker @@ -52,6 +52,9 @@ DATABASE_URL_DEDICATED=postgresql://dbusername:dbpassword@postgres:5432/httpsms # Redis connection string REDIS_URL=redis://@redis:6379 +# Set to "true" to enforce subscription message limits as hourly API budgets. +RATE_LIMIT_ENABLED=false + # Google Cloud Storage bucket for MMS attachments. Leave empty to use in-memory storage. GCS_BUCKET_NAME= diff --git a/api/go.mod b/api/go.mod index a18ad5049..c2a22647e 100644 --- a/api/go.mod +++ b/api/go.mod @@ -13,6 +13,7 @@ require ( github.com/NdoleStudio/lemonsqueezy-go v1.3.2 github.com/NdoleStudio/plunk-go v0.0.2 github.com/NdoleStudio/stacktrace v1.1.0 + github.com/alicebob/miniredis/v2 v2.35.0 github.com/avast/retry-go/v5 v5.0.0 github.com/axiomhq/axiom-go v0.38.0 github.com/carlmjohnson/requests v0.26.2 @@ -179,6 +180,7 @@ require ( github.com/xuri/efp v0.0.3-0.20260924042702-217b81952dec // indirect github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib v1.47.0 // indirect go.opentelemetry.io/contrib/detectors/gcp v1.45.0 // indirect diff --git a/api/go.sum b/api/go.sum index 3cb684e53..57af120a2 100644 --- a/api/go.sum +++ b/api/go.sum @@ -64,6 +64,8 @@ github.com/NdoleStudio/stacktrace v1.1.0 h1:Ai0j//4tDo/So3nR8bZgS3pQ4NYmViHlarpI github.com/NdoleStudio/stacktrace v1.1.0/go.mod h1:WOBcbXShYu+v8JtjFEM+rRV9GJW50m+dwIvp4sLuVTg= github.com/PuerkitoBio/goquery v1.12.0 h1:pAcL4g3WRXekcB9AU/y1mbKez2dbY2AajVhtkO8RIBo= github.com/PuerkitoBio/goquery v1.12.0/go.mod h1:802ej+gV2y7bbIhOIoPY5sT183ZW0YFofScC4q/hIpQ= +github.com/alicebob/miniredis/v2 v2.35.0 h1:QwLphYqCEAo1eu1TqPRN2jgVMPBweeQcR21jeqDCONI= +github.com/alicebob/miniredis/v2 v2.35.0/go.mod h1:TcL7YfarKPGDAthEtl5NBeHZfeUQj6OXMm/+iu5cLMM= github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/andybalholm/cascadia v1.3.4 h1:vM2lgh0Vru9Vwyfm4cQqWP2HHMW0u0+2PAW7Q38Qufg= @@ -381,6 +383,8 @@ github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78/go.mod h1:aL8wCCfTfS github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.8.2 h1:kEGpgqJXdgbkhcOgBxkC0X0PmoPG1ZyoZ117rDVp4zE= github.com/yuin/goldmark v1.8.2/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= go.mongodb.org/mongo-driver/v2 v2.9.2 h1:OCAdjI37pC/SVxucz/9CM8zD4jCdegtsY5QcKzYFCgE= diff --git a/api/main.go b/api/main.go index 5b7539c9c..1cc8fa483 100644 --- a/api/main.go +++ b/api/main.go @@ -1,12 +1,18 @@ package main import ( + "context" "fmt" "os" + "os/signal" "strings" + "syscall" + "time" "github.com/NdoleStudio/httpsms/docs" "github.com/NdoleStudio/httpsms/pkg/di" + "github.com/NdoleStudio/stacktrace" + "github.com/gofiber/fiber/v3" ) // Version is injected at runtime @@ -42,5 +48,13 @@ func main() { } container := di.NewContainer(os.Getenv("GCP_PROJECT_ID"), Version) - container.Logger().Info(container.App().Listen(fmt.Sprintf("%s:%s", os.Getenv("APP_HOST"), os.Getenv("APP_PORT"))).Error()) + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + if err := container.App().Listen( + fmt.Sprintf("%s:%s", os.Getenv("APP_HOST"), os.Getenv("APP_PORT")), + fiber.ListenConfig{GracefulContext: ctx, ShutdownTimeout: 5 * time.Second}, + ); err != nil { + container.Logger().Fatal(stacktrace.Propagatef(err, "cannot listen for HTTP requests")) + } } diff --git a/api/pkg/di/container.go b/api/pkg/di/container.go index e7ca20d28..95b3a78ed 100644 --- a/api/pkg/di/container.go +++ b/api/pkg/di/container.go @@ -9,6 +9,7 @@ import ( "os" "strconv" "strings" + "sync" "time" "cloud.google.com/go/compute/metadata" @@ -98,6 +99,11 @@ type Container struct { phoneRistrettoCache *ristretto.Cache[string, *entities.Phone] contactRistrettoCache *ristretto.Cache[string, services.ContactCacheEntry] inMemoryCache cache.Cache + rateLimitService *services.RateLimitService + rateLimitMiddleware fiber.Handler + redisClient *redis.Client + rateLimitRedisClient *redis.Client + closeOnce sync.Once } // NewLiteContainer creates a Container without any routes or listeners @@ -107,6 +113,20 @@ func NewLiteContainer() (container *Container) { } } +// Close gracefully shuts down container resources +func (container *Container) Close() { + container.closeOnce.Do(func() { + if container.rateLimitRedisClient != nil { + container.logger.Info("closing rate limit Redis client after HTTP shutdown") + if err := container.rateLimitRedisClient.Close(); err != nil { + container.logger.Error(stacktrace.Propagatef(err, "cannot close rate limit Redis client")) + return + } + container.logger.Info("rate limit Redis client shutdown complete") + } + }) +} + // NewContainer creates a new dependency injection container func NewContainer(projectID string, version string) (container *Container) { container = &Container{ @@ -209,6 +229,20 @@ func (container *Container) App() (app *fiber.App) { app.Use(middlewares.BearerAuth(container.Logger(), container.Tracer(), container.FirebaseAuthClient())) app.Use(middlewares.APIKeyAuth(container.Logger(), container.Tracer(), container.UserRepository())) + if os.Getenv("RATE_LIMIT_ENABLED") == "true" { + app.Hooks().OnPreShutdown(func() error { + container.logger.Info("rate limit shutdown waiting for HTTP requests to drain") + return nil + }) + app.Hooks().OnPostShutdown(func(err error) error { + if err != nil { + container.logger.Error(stacktrace.Propagatef(err, "HTTP shutdown did not finish draining requests")) + } + container.Close() + return nil + }) + } + container.app = app return app } @@ -228,7 +262,17 @@ func (container *Container) PhoneAPIKeyMiddleware() fiber.Handler { // AuthenticatedMiddleware creates a new instance of middlewares.Authenticated func (container *Container) AuthenticatedMiddleware() fiber.Handler { container.logger.Debug("creating middlewares.Authenticated") - return middlewares.Authenticated(container.Tracer()) + var limiter fiber.Handler + if os.Getenv("RATE_LIMIT_ENABLED") == "true" { + if container.rateLimitMiddleware == nil { + container.rateLimitMiddleware = middlewares.RateLimit( + container.Tracer(), container.Logger(), container.RateLimitService(), + container.UserRepository(), []string{"/v1/events"}, + ) + } + limiter = container.rateLimitMiddleware + } + return middlewares.Authenticated(container.Tracer(), limiter) } // Logger creates a new instance of telemetry.Logger @@ -451,6 +495,16 @@ func (container *Container) InMemoryCache() cache.Cache { // Cache creates a new instance of cache.Cache func (container *Container) Cache() cache.Cache { container.logger.Debug("creating cache.Cache") + return cache.NewRedisCache(container.Tracer(), container.RedisClient()) +} + +// RedisClient creates or returns the shared *redis.Client +func (container *Container) RedisClient() *redis.Client { + if container.redisClient != nil { + return container.redisClient + } + + container.logger.Debug("creating *redis.Client") opt, err := redis.ParseURL(os.Getenv("REDIS_URL")) if err != nil { container.logger.Fatal(stacktrace.Propagatef(err, "cannot parse redis url [%s]", os.Getenv("REDIS_URL"))) @@ -461,19 +515,48 @@ func (container *Container) Cache() cache.Cache { } } - redisClient := redis.NewClient(opt) + container.redisClient = redis.NewClient(opt) // Enable tracing instrumentation. - if err = redisotel.InstrumentTracing(redisClient); err != nil { + if err = redisotel.InstrumentTracing(container.redisClient); err != nil { container.logger.Error(stacktrace.Propagatef(err, "cannot instrument redis tracing")) } // Enable metrics instrumentation. - if err = redisotel.InstrumentMetrics(redisClient); err != nil { + if err = redisotel.InstrumentMetrics(container.redisClient); err != nil { container.logger.Fatal(stacktrace.Propagatef(err, "cannot instrument redis metrics")) } - return cache.NewRedisCache(container.Tracer(), redisClient) + return container.redisClient +} + +// RateLimitService creates or returns the shared *services.RateLimitService +func (container *Container) RateLimitService() *services.RateLimitService { + if container.rateLimitService != nil { + return container.rateLimitService + } + + container.logger.Debug("creating services.RateLimitService") + options := *container.RedisClient().Options() + options.ContextTimeoutEnabled = true + options.DialTimeout = time.Second + options.ReadTimeout = time.Second + options.WriteTimeout = time.Second + options.PoolTimeout = time.Second + options.MaxRetries = -1 + container.rateLimitRedisClient = redis.NewClient(&options) + if err := redisotel.InstrumentTracing(container.rateLimitRedisClient); err != nil { + container.logger.Error(stacktrace.Propagatef(err, "cannot instrument rate limit Redis tracing")) + } + if err := redisotel.InstrumentMetrics(container.rateLimitRedisClient); err != nil { + container.logger.Error(stacktrace.Propagatef(err, "cannot instrument rate limit Redis metrics")) + } + container.rateLimitService = services.NewRateLimitService( + container.Tracer(), + container.Logger(), + container.rateLimitRedisClient, + ) + return container.rateLimitService } // FirebaseAuthClient creates a new instance of auth.Client diff --git a/api/pkg/di/container_test.go b/api/pkg/di/container_test.go index 611708d31..2ff3a8eb5 100644 --- a/api/pkg/di/container_test.go +++ b/api/pkg/di/container_test.go @@ -1,15 +1,75 @@ package di import ( + "context" + "net" + "net/http" + "net/http/httptest" "reflect" "testing" + "time" "github.com/NdoleStudio/httpsms/pkg/entities" + "github.com/NdoleStudio/httpsms/pkg/middlewares" "github.com/NdoleStudio/httpsms/pkg/services" + "github.com/gofiber/fiber/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +func TestRateLimitDisabledDoesNotInitializeResources(t *testing.T) { + t.Setenv("ENV", "local") + t.Setenv("RATE_LIMIT_ENABLED", "false") + container := NewLiteContainer() + app := fiber.New() + app.Use(func(c fiber.Ctx) error { + c.Locals(middlewares.ContextKeyAuthUserID, entities.AuthContext{ID: "user", Email: "user@example.com"}) + return c.Next() + }) + app.Use(container.AuthenticatedMiddleware()) + app.Get("/", func(c fiber.Ctx) error { return c.SendStatus(http.StatusOK) }) + response, err := app.Test(httptest.NewRequest(http.MethodGet, "/", nil)) + require.NoError(t, err) + defer response.Body.Close() + require.Equal(t, http.StatusOK, response.StatusCode) + require.Nil(t, container.rateLimitService) + require.Nil(t, container.rateLimitMiddleware) + require.Nil(t, container.rateLimitRedisClient) +} + +func TestRateLimitRedisClientBoundsStalledConnections(t *testing.T) { + t.Setenv("ENV", "local") + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { listener.Close() }) + accepted := make(chan net.Conn, 1) + go func() { + connection, err := listener.Accept() + if err == nil { + accepted <- connection + } + }() + t.Cleanup(func() { + select { + case connection := <-accepted: + connection.Close() + case <-time.After(time.Second): + } + }) + t.Setenv("REDIS_URL", "redis://"+listener.Addr().String()) + container := NewLiteContainer() + container.RateLimitService() + t.Cleanup(container.Close) + t.Cleanup(func() { container.redisClient.Close() }) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + start := time.Now() + err = container.rateLimitRedisClient.Ping(ctx).Err() + require.Error(t, err) + require.Less(t, time.Since(start), 1500*time.Millisecond) +} + func TestNotificationHTTPClientUsesOTelRoundTripperWithoutRetries(t *testing.T) { t.Setenv("ENV", "local") client := NewLiteContainer().NotificationHTTPClient() diff --git a/api/pkg/middlewares/authenticated_middlesare.go b/api/pkg/middlewares/authenticated_middlesare.go index d7e6221d4..a4f5b4f5e 100644 --- a/api/pkg/middlewares/authenticated_middlesare.go +++ b/api/pkg/middlewares/authenticated_middlesare.go @@ -18,7 +18,7 @@ const ( ) // Authenticated checks if the request is authenticated -func Authenticated(tracer telemetry.Tracer) fiber.Handler { +func Authenticated(tracer telemetry.Tracer, afterAuth fiber.Handler) fiber.Handler { return func(c fiber.Ctx) error { _, span := tracer.StartFromFiberCtx(c, "middlewares.Authenticated") defer span.End() @@ -31,6 +31,9 @@ func Authenticated(tracer telemetry.Tracer) fiber.Handler { }) } + if afterAuth != nil { + return afterAuth(c) + } return c.Next() } } diff --git a/api/pkg/middlewares/rate_limit_middleware.go b/api/pkg/middlewares/rate_limit_middleware.go new file mode 100644 index 000000000..433ea7fad --- /dev/null +++ b/api/pkg/middlewares/rate_limit_middleware.go @@ -0,0 +1,97 @@ +package middlewares + +import ( + "context" + "strconv" + "strings" + "time" + + "github.com/NdoleStudio/httpsms/pkg/entities" + "github.com/NdoleStudio/httpsms/pkg/repositories" + "github.com/NdoleStudio/httpsms/pkg/telemetry" + "github.com/gofiber/fiber/v3" + ttlCache "github.com/patrickmn/go-cache" + "golang.org/x/sync/singleflight" +) + +const rateLimitCostCap = 200 + +type rateLimitCounter interface { + Allow(context.Context, entities.UserID, entities.SubscriptionName, int) (bool, time.Duration, error) +} + +// RateLimit enforces hourly per-user budgets after authentication and before handlers. +func RateLimit( + tracer telemetry.Tracer, + logger telemetry.Logger, + service rateLimitCounter, + userRepository repositories.UserRepository, + excludePaths []string, +) fiber.Handler { + logger = logger.WithService("middlewares.RateLimit") + plans := ttlCache.New(time.Minute, time.Minute) + var loads singleflight.Group + + return func(c fiber.Ctx) error { + path := c.Path() + for _, excluded := range excludePaths { + if path == excluded || strings.HasPrefix(path, excluded+"/") { + return c.Next() + } + } + + ctx, span, ctxLogger := tracer.StartFromFiberCtxWithLogger(c, logger, "middlewares.RateLimit") + defer span.End() + + authUser, ok := c.Locals(ContextKeyAuthUserID).(entities.AuthContext) + if !ok || authUser.IsNoop() { + return c.Next() + } + // Cache keys outlive Fiber's reusable request buffers. + userID := entities.UserID(strings.Clone(string(authUser.ID))) + + cost := 1 + if c.Method() == fiber.MethodGet { + if limitParam := c.Query("limit"); limitParam != "" { + if parsed, err := strconv.Atoi(limitParam); err == nil && parsed > 0 { + cost = min(parsed, rateLimitCostCap) + } + } + } + + plan, err, _ := loads.Do(string(userID), func() (interface{}, error) { + if cached, found := plans.Get(string(userID)); found { + return cached, nil + } + loadCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + user, err := userRepository.Load(loadCtx, userID) + if err != nil { + return nil, err + } + plans.SetDefault(string(userID), user.SubscriptionName) + return user.SubscriptionName, nil + }) + if err != nil { + ctxLogger.Error(err) + return c.Next() + } + + allowed, retryAfter, err := service.Allow(ctx, userID, plan.(entities.SubscriptionName), cost) + if err != nil { + ctxLogger.Error(err) + return c.Next() + } + if !allowed { + seconds := max(1, int((retryAfter+time.Second-1)/time.Second)) + c.Set(fiber.HeaderRetryAfter, strconv.Itoa(seconds)) + ctxLogger.WithString("user_id", string(userID)).WithString("plan", string(plan.(entities.SubscriptionName))).Info("hourly API rate limit exceeded") + return c.Status(fiber.StatusTooManyRequests).JSON(fiber.Map{ + "status": "error", + "message": "Hourly API rate limit exceeded. Try again after the time indicated by Retry-After.", + }) + } + + return c.Next() + } +} diff --git a/api/pkg/middlewares/rate_limit_middleware_test.go b/api/pkg/middlewares/rate_limit_middleware_test.go new file mode 100644 index 000000000..b5a492c2e --- /dev/null +++ b/api/pkg/middlewares/rate_limit_middleware_test.go @@ -0,0 +1,258 @@ +package middlewares + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/NdoleStudio/httpsms/pkg/entities" + "github.com/NdoleStudio/httpsms/pkg/repositories" + "github.com/NdoleStudio/httpsms/pkg/services" + "github.com/NdoleStudio/httpsms/pkg/telemetry" + "github.com/alicebob/miniredis/v2" + "github.com/gofiber/fiber/v3" + "github.com/hirosassa/zerodriver" + "github.com/redis/go-redis/v9" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +func TestRateLimitMiddleware_WeightedCost(t *testing.T) { + for _, test := range []struct { + name string + method string + query string + cost int + }{ + {"requested page", http.MethodGet, "?limit=10", 10}, + {"search maximum", http.MethodGet, "?limit=200", 200}, + {"capped", http.MethodGet, "?limit=300", 200}, + {"missing limit", http.MethodGet, "", 1}, + {"invalid limit", http.MethodGet, "?limit=invalid", 1}, + {"negative limit", http.MethodGet, "?limit=-10", 1}, + {"write request", http.MethodPost, "?limit=10", 1}, + } { + t.Run(test.name, func(t *testing.T) { + repo := &rateLimitUserRepository{} + counter := &recordingRateLimitCounter{} + app := rateLimitTestApp(repo, counter, rateLimitTestAuth, nil) + response, err := app.Test(httptest.NewRequest(test.method, "/v1/messages/search"+test.query, nil)) + require.NoError(t, err) + defer response.Body.Close() + require.Equal(t, http.StatusOK, response.StatusCode) + require.Equal(t, []int{test.cost}, counter.costs) + }) + } +} + +func TestRateLimitMiddleware_RouteAuthentication(t *testing.T) { + for _, phoneKey := range []bool{false, true} { + t.Run(map[bool]string{false: "3cx bearer key", true: "phone key"}[phoneKey], func(t *testing.T) { + repo := &rateLimitUserRepository{} + counter := &recordingRateLimitCounter{} + logger, tracer := rateLimitTestTelemetry() + auth := BearerAPIKeyAuth(logger, tracer, repo) + request := httptest.NewRequest(http.MethodGet, "/v1/messages/outstanding", nil) + request.Header.Set("Authorization", "Bearer test-key") + if phoneKey { + auth = PhoneAPIKeyAuth(logger, tracer, &rateLimitPhoneRepository{}) + request.Header.Set("x-api-key", "pk_test-key") + } + app := rateLimitTestApp(repo, counter, auth, nil) + response, err := app.Test(request) + require.NoError(t, err) + defer response.Body.Close() + require.Equal(t, http.StatusOK, response.StatusCode) + require.Equal(t, []entities.UserID{"user"}, counter.users) + }) + } +} + +func TestRateLimitMiddleware_CachesPlansPerUser(t *testing.T) { + repo := &rateLimitUserRepository{} + counter := &recordingRateLimitCounter{} + app := rateLimitTestApp(repo, counter, rateLimitTestAuth, nil) + for _, user := range []string{"user", "user", "other", "other", "user"} { + request := httptest.NewRequest(http.MethodGet, "/v1/messages/search", nil) + request.Header.Set("test-user", user) + response, err := app.Test(request) + require.NoError(t, err) + response.Body.Close() + } + require.Equal(t, 2, repo.loads) + require.Len(t, counter.costs, 5) +} + +func TestRateLimitMiddleware_Exclusions(t *testing.T) { + repo := &rateLimitUserRepository{} + counter := &recordingRateLimitCounter{} + app := rateLimitTestApp(repo, counter, rateLimitTestAuth, nil) + for _, path := range []string{"/v1/events", "/v1/events/dispatch", "/v1/events-other"} { + response, err := app.Test(httptest.NewRequest(http.MethodGet, path, nil)) + require.NoError(t, err) + response.Body.Close() + } + require.Equal(t, []int{1}, counter.costs) + + anonymous := rateLimitTestApp(repo, counter, func(c fiber.Ctx) error { return c.Next() }, nil) + response, err := anonymous.Test(httptest.NewRequest(http.MethodGet, "/v1/messages/search", nil)) + require.NoError(t, err) + response.Body.Close() + require.Len(t, counter.costs, 1) +} + +func TestRateLimitMiddleware_FailsOpenAndPreservesHandlerErrors(t *testing.T) { + for _, test := range []struct { + name string + repoError error + countError error + nextError error + status int + }{ + {"plan lookup fails", errors.New("database unavailable"), nil, nil, http.StatusOK}, + {"tracking fails", nil, errors.New("redis unavailable"), nil, http.StatusOK}, + {"handler fails", nil, nil, fiber.ErrTeapot, http.StatusTeapot}, + } { + t.Run(test.name, func(t *testing.T) { + app := rateLimitTestApp( + &rateLimitUserRepository{err: test.repoError}, + &recordingRateLimitCounter{err: test.countError}, + rateLimitTestAuth, test.nextError, + ) + response, err := app.Test(httptest.NewRequest(http.MethodGet, "/v1/messages/search", nil)) + require.NoError(t, err) + defer response.Body.Close() + require.Equal(t, test.status, response.StatusCode) + }) + } +} + +func TestRateLimitMiddleware_RejectsBeforeHandler(t *testing.T) { + counter := &recordingRateLimitCounter{denied: true, retryAfter: 1500 * time.Millisecond} + app := rateLimitTestApp(&rateLimitUserRepository{}, counter, rateLimitTestAuth, nil) + response, err := app.Test(httptest.NewRequest(http.MethodPost, "/v1/messages/send", nil)) + require.NoError(t, err) + defer response.Body.Close() + require.Equal(t, http.StatusTooManyRequests, response.StatusCode) + require.Equal(t, "2", response.Header.Get("Retry-After")) + require.Empty(t, response.Header.Get("X-Handler-Ran")) +} + +func TestRateLimitMiddleware_HourlyEnforcementAcrossRequests(t *testing.T) { + server := miniredis.RunT(t) + client := redis.NewClient(&redis.Options{Addr: server.Addr(), ContextTimeoutEnabled: true, MaxRetries: -1}) + t.Cleanup(func() { client.Close() }) + logger, tracer := rateLimitTestTelemetry() + limiter := services.NewRateLimitService(tracer, logger, client) + app := fiber.New() + app.Use(rateLimitTestAuth) + app.Use(Authenticated(tracer, RateLimit(tracer, logger, limiter, &rateLimitUserRepository{}, []string{"/v1/events"}))) + app.Use(func(c fiber.Ctx) error { + c.Set("X-Handler-Ran", "true") + return c.SendStatus(http.StatusOK) + }) + for _, test := range []struct { + path string + status int + }{ + {"/v1/messages/search?limit=199", http.StatusOK}, + {"/v1/messages/search?limit=2", http.StatusTooManyRequests}, + {"/v1/messages/search?limit=1", http.StatusOK}, + {"/v1/messages/search?limit=1", http.StatusTooManyRequests}, + {"/v1/events", http.StatusOK}, + } { + response, err := app.Test(httptest.NewRequest(http.MethodGet, test.path, nil)) + require.NoError(t, err) + response.Body.Close() + require.Equal(t, test.status, response.StatusCode) + if test.status == http.StatusTooManyRequests { + require.Equal(t, "3600", response.Header.Get("Retry-After")) + require.Empty(t, response.Header.Get("X-Handler-Ran")) + } + } + server.FastForward(time.Hour) + response, err := app.Test(httptest.NewRequest(http.MethodGet, "/v1/messages/search?limit=200", nil)) + require.NoError(t, err) + defer response.Body.Close() + require.Equal(t, http.StatusOK, response.StatusCode) +} + +func TestRateLimitMiddleware_AuthenticationPrecedesEnforcement(t *testing.T) { + logger, tracer := rateLimitTestTelemetry() + counter := &recordingRateLimitCounter{denied: true, retryAfter: time.Hour} + app := fiber.New() + app.Use(Authenticated(tracer, RateLimit(tracer, logger, counter, &rateLimitUserRepository{}, nil))) + app.Get("/", func(c fiber.Ctx) error { return c.SendStatus(http.StatusOK) }) + response, err := app.Test(httptest.NewRequest(http.MethodGet, "/", nil)) + require.NoError(t, err) + defer response.Body.Close() + require.Equal(t, http.StatusUnauthorized, response.StatusCode) + require.Empty(t, counter.costs) +} + +type rateLimitUserRepository struct { + repositories.UserRepository + loads int + err error +} + +func (repo *rateLimitUserRepository) Load(context.Context, entities.UserID) (*entities.User, error) { + repo.loads++ + return &entities.User{SubscriptionName: entities.SubscriptionNameFree}, repo.err +} + +func (*rateLimitUserRepository) LoadAuthContext(context.Context, string) (entities.AuthContext, error) { + return entities.AuthContext{ID: "user", Email: "user@example.com"}, nil +} + +type rateLimitPhoneRepository struct { + repositories.PhoneAPIKeyRepository +} + +func (*rateLimitPhoneRepository) LoadAuthContext(context.Context, string) (entities.AuthContext, error) { + return entities.AuthContext{ID: "user", Email: "user@example.com"}, nil +} + +type recordingRateLimitCounter struct { + costs []int + users []entities.UserID + err error + denied bool + retryAfter time.Duration +} + +func (counter *recordingRateLimitCounter) Allow(_ context.Context, user entities.UserID, _ entities.SubscriptionName, cost int) (bool, time.Duration, error) { + counter.costs = append(counter.costs, cost) + counter.users = append(counter.users, user) + return !counter.denied, counter.retryAfter, counter.err +} + +func rateLimitTestTelemetry() (telemetry.Logger, telemetry.Tracer) { + driver := zerolog.Nop() + logger := telemetry.NewZerologLogger("", nil, &zerodriver.Logger{Logger: &driver}, nil) + return logger, telemetry.NewOtelLogger("", logger) +} + +func rateLimitTestAuth(c fiber.Ctx) error { + user := c.Get("test-user", "user") + c.Locals(ContextKeyAuthUserID, entities.AuthContext{ID: entities.UserID(user), Email: "user@example.com"}) + return c.Next() +} + +func rateLimitTestApp(repo repositories.UserRepository, counter rateLimitCounter, auth fiber.Handler, handlerError error) *fiber.App { + logger, tracer := rateLimitTestTelemetry() + app := fiber.New() + app.Use(auth) + app.Use(RateLimit(tracer, logger, counter, repo, []string{"/v1/events"})) + app.Use(func(c fiber.Ctx) error { + c.Set("X-Handler-Ran", "true") + if handlerError != nil { + return handlerError + } + return c.SendStatus(http.StatusOK) + }) + return app +} diff --git a/api/pkg/services/rate_limit_allow.lua b/api/pkg/services/rate_limit_allow.lua new file mode 100644 index 000000000..188b91b1e --- /dev/null +++ b/api/pkg/services/rate_limit_allow.lua @@ -0,0 +1,27 @@ +local key = KEYS[1] +local limit, cost, window = tonumber(ARGV[1]), tonumber(ARGV[2]), tonumber(ARGV[3]) +local stored = redis.call('GET', key) +local count, ttl = 0, 0 + +if stored then + count = tonumber(stored) + if not string.match(stored, '^%d+$') or not count or count > 9007199254740991 + or (#stored > 1 and string.sub(stored, 1, 1) == '0') then + return redis.error_reply('invalid hourly rate limit counter') + end + ttl = redis.call('PTTL', key) + if ttl < 0 or ttl > window then + return redis.error_reply('invalid hourly rate limit expiry') + end +end + +if cost > limit - count then + return {0, ttl} +end + +if stored then + redis.call('INCRBY', key, cost) +else + redis.call('SET', key, cost, 'PX', window) +end +return {1, 0} diff --git a/api/pkg/services/rate_limit_service.go b/api/pkg/services/rate_limit_service.go new file mode 100644 index 000000000..886f9d5fd --- /dev/null +++ b/api/pkg/services/rate_limit_service.go @@ -0,0 +1,63 @@ +package services + +import ( + "context" + _ "embed" + "fmt" + "time" + + "github.com/NdoleStudio/httpsms/pkg/entities" + "github.com/NdoleStudio/httpsms/pkg/telemetry" + "github.com/NdoleStudio/stacktrace" + "github.com/redis/go-redis/v9" +) + +const ( + rateLimitKeyPrefix = "user_rate_limit:hourly:" + rateLimitWindow = time.Hour + rateLimitRedisTimeout = time.Second +) + +//go:embed rate_limit_allow.lua +var rateLimitAllowScript string + +// RateLimitService admits requests against shared fixed hourly user budgets. +type RateLimitService struct { + tracer telemetry.Tracer + logger telemetry.Logger + client *redis.Client +} + +// NewRateLimitService creates a synchronous Redis-backed admission service. +func NewRateLimitService(tracer telemetry.Tracer, logger telemetry.Logger, client *redis.Client) (rateLimiter *RateLimitService) { + return &RateLimitService{ + tracer: tracer, + logger: logger.WithService(fmt.Sprintf("%T", rateLimiter)), + client: client, + } +} + +// Allow atomically admits and charges cost, or returns the remaining window TTL. +// Failures are returned with allowed=true so callers can log and fail open. +// An oversized request without an active window is denied with a zero TTL. +func (service *RateLimitService) Allow(ctx context.Context, userID entities.UserID, plan entities.SubscriptionName, cost int) (allowed bool, retryAfter time.Duration, err error) { + ctx, span, _ := service.tracer.StartWithLogger(ctx, service.logger) + defer span.End() + if cost <= 0 { + return true, 0, service.tracer.WrapErrorSpan(span, stacktrace.NewErrorf("invalid rate limit cost [%d] for user [%s]", cost, userID)) + } + if service.client == nil { + return true, 0, service.tracer.WrapErrorSpan(span, stacktrace.NewErrorf("rate limit Redis client is not configured for user [%s]", userID)) + } + ctx, cancel := context.WithTimeout(ctx, rateLimitRedisTimeout) + defer cancel() + result, err := service.client.Eval(ctx, rateLimitAllowScript, []string{rateLimitKeyPrefix + string(userID)}, + plan.Limit(), cost, rateLimitWindow.Milliseconds()).Int64Slice() + if err != nil { + return true, 0, service.tracer.WrapErrorSpan(span, stacktrace.Propagatef(err, "cannot check hourly rate limit for user [%s]", userID)) + } + if len(result) != 2 || (result[0] != 0 && result[0] != 1) || result[1] < 0 || result[1] > rateLimitWindow.Milliseconds() { + return true, 0, service.tracer.WrapErrorSpan(span, stacktrace.NewErrorf("invalid hourly rate limit result for user [%s]: %v", userID, result)) + } + return result[0] == 1, time.Duration(result[1]) * time.Millisecond, nil +} diff --git a/api/pkg/services/rate_limit_service_test.go b/api/pkg/services/rate_limit_service_test.go new file mode 100644 index 000000000..4b6060cdb --- /dev/null +++ b/api/pkg/services/rate_limit_service_test.go @@ -0,0 +1,352 @@ +package services + +import ( + "context" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/NdoleStudio/httpsms/pkg/entities" + "github.com/NdoleStudio/httpsms/pkg/telemetry" + "github.com/alicebob/miniredis/v2" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/trace" +) + +func newTestRateLimitService(t *testing.T, server *miniredis.Miniredis) *RateLimitService { + t.Helper() + client := redis.NewClient(&redis.Options{ + Addr: server.Addr(), ContextTimeoutEnabled: true, MaxRetries: -1, + DialTimeout: time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, PoolTimeout: time.Second, + }) + t.Cleanup(func() { require.NoError(t, client.Close()) }) + logger := &testLogger{} + return NewRateLimitService(telemetry.NewOtelLogger("test", logger), logger, client) +} + +func TestRateLimitService_FreeHourlyBudget(t *testing.T) { + server := miniredis.RunT(t) + service := newTestRateLimitService(t, server) + for range 200 { + allowed, retryAfter, err := service.Allow(context.Background(), "user", entities.SubscriptionNameFree, 1) + require.NoError(t, err) + require.True(t, allowed) + require.Zero(t, retryAfter) + } + for range 2 { + allowed, retryAfter, err := service.Allow(context.Background(), "user", entities.SubscriptionNameFree, 1) + require.NoError(t, err) + assert.False(t, allowed) + assert.InDelta(t, time.Hour.Milliseconds(), retryAfter.Milliseconds(), 1000) + } + count, err := server.Get("user_rate_limit:hourly:user") + require.NoError(t, err) + assert.Equal(t, "200", count) +} + +func TestRateLimitService_WeightedDenialDoesNotSpendRemainingBudget(t *testing.T) { + server := miniredis.RunT(t) + service := newTestRateLimitService(t, server) + for _, test := range []struct { + cost int + allowed bool + total string + }{ + {190, true, "190"}, + {11, false, "190"}, + {10, true, "200"}, + {1, false, "200"}, + } { + allowed, _, err := service.Allow(context.Background(), "user", entities.SubscriptionNameFree, test.cost) + require.NoError(t, err) + assert.Equal(t, test.allowed, allowed) + count, err := server.Get("user_rate_limit:hourly:user") + require.NoError(t, err) + assert.Equal(t, test.total, count) + } +} + +func TestRateLimitService_FirstDeniedRequestDoesNotStartWindow(t *testing.T) { + server := miniredis.RunT(t) + service := newTestRateLimitService(t, server) + allowed, retryAfter, err := service.Allow(context.Background(), "user", entities.SubscriptionNameFree, 201) + require.NoError(t, err) + assert.False(t, allowed) + assert.Zero(t, retryAfter) + assert.False(t, server.Exists("user_rate_limit:hourly:user")) + allowed, _, err = service.Allow(context.Background(), "user", entities.SubscriptionNameFree, 200) + require.NoError(t, err) + assert.True(t, allowed) + assert.Equal(t, time.Hour, server.TTL("user_rate_limit:hourly:user")) +} + +func TestRateLimitService_ConcurrentInstancesNeverOversubscribe(t *testing.T) { + server := miniredis.RunT(t) + var admitted atomic.Int64 + failures := make(chan error, 400) + var workers sync.WaitGroup + for range 4 { + service := newTestRateLimitService(t, server) + workers.Go(func() { + for range 100 { + allowed, _, err := service.Allow(context.Background(), "user", entities.SubscriptionNameFree, 1) + if err != nil { + failures <- err + } else if allowed { + admitted.Add(1) + } + } + }) + } + workers.Wait() + close(failures) + for err := range failures { + assert.NoError(t, err) + } + assert.Equal(t, int64(200), admitted.Load()) + count, err := server.Get("user_rate_limit:hourly:user") + require.NoError(t, err) + assert.Equal(t, "200", count) +} + +func TestRateLimitService_RestartSeesSharedBudget(t *testing.T) { + server := miniredis.RunT(t) + first := newTestRateLimitService(t, server) + allowed, _, err := first.Allow(context.Background(), "user", entities.SubscriptionNameFree, 199) + require.NoError(t, err) + require.True(t, allowed) + second := newTestRateLimitService(t, server) + allowed, _, err = second.Allow(context.Background(), "user", entities.SubscriptionNameFree, 1) + require.NoError(t, err) + assert.True(t, allowed) + allowed, _, err = second.Allow(context.Background(), "user", entities.SubscriptionNameFree, 1) + require.NoError(t, err) + assert.False(t, allowed) +} + +func TestRateLimitService_AdmitsAndDenialsNeverExtendWindow(t *testing.T) { + server := miniredis.RunT(t) + service := newTestRateLimitService(t, server) + allowed, _, err := service.Allow(context.Background(), "user", entities.SubscriptionNameFree, 100) + require.NoError(t, err) + require.True(t, allowed) + server.FastForward(30*time.Minute + 500*time.Millisecond) + allowed, _, err = service.Allow(context.Background(), "user", entities.SubscriptionNameFree, 1) + require.NoError(t, err) + assert.True(t, allowed) + assert.Equal(t, 30*time.Minute-500*time.Millisecond, server.TTL("user_rate_limit:hourly:user")) + allowed, retryAfter, err := service.Allow(context.Background(), "user", entities.SubscriptionNameFree, 100) + require.NoError(t, err) + assert.False(t, allowed) + assert.Equal(t, 30*time.Minute-500*time.Millisecond, retryAfter) + assert.Equal(t, retryAfter, server.TTL("user_rate_limit:hourly:user")) +} + +func TestRateLimitService_WindowResetsAfterExactlyOneHour(t *testing.T) { + server := miniredis.RunT(t) + service := newTestRateLimitService(t, server) + allowed, _, err := service.Allow(context.Background(), "user", entities.SubscriptionNameFree, 200) + require.NoError(t, err) + require.True(t, allowed) + server.FastForward(3599 * time.Second) + allowed, retryAfter, err := service.Allow(context.Background(), "user", entities.SubscriptionNameFree, 1) + require.NoError(t, err) + assert.False(t, allowed) + assert.Equal(t, time.Second, retryAfter) + server.FastForward(time.Second) + allowed, _, err = service.Allow(context.Background(), "user", entities.SubscriptionNameFree, 200) + require.NoError(t, err) + assert.True(t, allowed) + assert.Equal(t, time.Hour, server.TTL("user_rate_limit:hourly:user")) +} + +func TestRateLimitService_SubscriptionHourlyBudgets(t *testing.T) { + for _, test := range []struct { + plan entities.SubscriptionName + limit int + }{ + {entities.SubscriptionNameFree, 200}, + {entities.SubscriptionNameProMonthly, 5000}, + {entities.SubscriptionNameProYearly, 5000}, + {entities.SubscriptionNameProLifetime, 5000}, + {entities.SubscriptionNameUltraMonthly, 10000}, + {entities.SubscriptionNameUltraYearly, 10000}, + {entities.SubscriptionName20KMonthly, 20000}, + {entities.SubscriptionName20KYearly, 20000}, + {entities.SubscriptionName50KMonthly, 50000}, + {entities.SubscriptionName100KMonthly, 100000}, + {entities.SubscriptionName200KMonthly, 200000}, + } { + t.Run(string(test.plan), func(t *testing.T) { + server := miniredis.RunT(t) + service := newTestRateLimitService(t, server) + allowed, _, err := service.Allow(context.Background(), "user", test.plan, test.limit) + require.NoError(t, err) + assert.True(t, allowed) + allowed, _, err = service.Allow(context.Background(), "user", test.plan, 1) + require.NoError(t, err) + assert.False(t, allowed) + }) + } +} + +func TestRateLimitService_IgnoresLegacyDailyNamespace(t *testing.T) { + server := miniredis.RunT(t) + require.NoError(t, server.Set("user_rate_limit:user", "999999")) + service := newTestRateLimitService(t, server) + allowed, _, err := service.Allow(context.Background(), "user", entities.SubscriptionNameFree, 200) + require.NoError(t, err) + assert.True(t, allowed) + legacy, err := server.Get("user_rate_limit:user") + require.NoError(t, err) + assert.Equal(t, "999999", legacy) +} + +func TestRateLimitService_MalformedStateFailsOpenWithError(t *testing.T) { + for _, state := range []string{"not-an-integer", "-1", "3.5", "1e2", "0200", "9007199254740992", "no-expiry", "long-expiry", "wrong-type"} { + t.Run(state, func(t *testing.T) { + server := miniredis.RunT(t) + key := "user_rate_limit:hourly:user" + switch state { + case "no-expiry": + require.NoError(t, server.Set(key, "10")) + case "long-expiry": + require.NoError(t, server.Set(key, "10")) + server.SetTTL(key, 2*time.Hour) + case "wrong-type": + server.HSet(key, "count", "10") + server.SetTTL(key, time.Hour) + default: + require.NoError(t, server.Set(key, state)) + server.SetTTL(key, time.Hour) + } + service := newTestRateLimitService(t, server) + allowed, retryAfter, err := service.Allow(context.Background(), "user", entities.SubscriptionNameFree, 1) + require.Error(t, err) + assert.Contains(t, err.Error(), "user") + assert.True(t, allowed) + assert.Zero(t, retryAfter) + }) + } +} + +func TestRateLimitService_RedisOutageFailsOpen(t *testing.T) { + server := miniredis.RunT(t) + service := newTestRateLimitService(t, server) + server.Close() + start := time.Now() + allowed, retryAfter, err := service.Allow(context.Background(), "user", entities.SubscriptionNameFree, 1) + require.Error(t, err) + assert.True(t, allowed) + assert.Zero(t, retryAfter) + assert.Less(t, time.Since(start), 1500*time.Millisecond) +} + +func TestRateLimitService_RedisDeadlineIsOneSecond(t *testing.T) { + server := miniredis.RunT(t) + service := newTestRateLimitService(t, server) + service.client.AddHook(&rateLimitProcessHook{run: func(ctx context.Context, _ redis.Cmder, _ redis.ProcessHook) error { + <-ctx.Done() + return ctx.Err() + }}) + start := time.Now() + allowed, retryAfter, err := service.Allow(context.Background(), "user", entities.SubscriptionNameFree, 1) + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.True(t, allowed) + assert.Zero(t, retryAfter) + assert.Less(t, time.Since(start), 1500*time.Millisecond) +} + +func TestRateLimitService_RequestCancellationFailsOpenWithoutSpending(t *testing.T) { + server := miniredis.RunT(t) + service := newTestRateLimitService(t, server) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + allowed, retryAfter, err := service.Allow(ctx, "user", entities.SubscriptionNameFree, 1) + require.ErrorIs(t, err, context.Canceled) + assert.True(t, allowed) + assert.Zero(t, retryAfter) + assert.False(t, server.Exists("user_rate_limit:hourly:user")) +} + +func TestRateLimitService_OneRedisOperationPreservesTraceCorrelation(t *testing.T) { + server := miniredis.RunT(t) + service := newTestRateLimitService(t, server) + require.NoError(t, service.client.Ping(context.Background()).Err()) + correlation := trace.NewSpanContext(trace.SpanContextConfig{TraceID: trace.TraceID{1}, SpanID: trace.SpanID{2}}) + ctx := trace.ContextWithSpanContext(context.Background(), correlation) + var commands []string + service.client.AddHook(&rateLimitProcessHook{run: func(ctx context.Context, cmd redis.Cmder, next redis.ProcessHook) error { + commands = append(commands, cmd.Name()) + assert.Equal(t, correlation.TraceID(), trace.SpanContextFromContext(ctx).TraceID()) + return next(ctx, cmd) + }}) + allowed, _, err := service.Allow(ctx, "user", entities.SubscriptionNameFree, 1) + require.NoError(t, err) + assert.True(t, allowed) + assert.Equal(t, []string{"eval"}, commands) +} + +func TestRateLimitService_InvalidCostCannotReduceUsage(t *testing.T) { + server := miniredis.RunT(t) + service := newTestRateLimitService(t, server) + allowed, _, err := service.Allow(context.Background(), "user", entities.SubscriptionNameFree, 200) + require.NoError(t, err) + require.True(t, allowed) + for _, cost := range []int{0, -1} { + allowed, _, err = service.Allow(context.Background(), "user", entities.SubscriptionNameFree, cost) + require.Error(t, err) + assert.True(t, allowed) + } + allowed, _, err = service.Allow(context.Background(), "user", entities.SubscriptionNameFree, 1) + require.NoError(t, err) + assert.False(t, allowed) +} + +func TestRateLimitService_UserBudgetsAreIndependent(t *testing.T) { + server := miniredis.RunT(t) + service := newTestRateLimitService(t, server) + for _, user := range []entities.UserID{"first", "second"} { + allowed, _, err := service.Allow(context.Background(), user, entities.SubscriptionNameFree, 200) + require.NoError(t, err) + assert.True(t, allowed) + } +} + +func TestRateLimitService_MissingRedisClientFailsOpenWithError(t *testing.T) { + logger := &testLogger{} + service := NewRateLimitService(telemetry.NewOtelLogger("test", logger), logger, nil) + allowed, retryAfter, err := service.Allow(context.Background(), "user", entities.SubscriptionNameFree, 1) + require.Error(t, err) + assert.True(t, allowed) + assert.Zero(t, retryAfter) +} + +type rateLimitProcessHook struct { + run func(context.Context, redis.Cmder, redis.ProcessHook) error +} + +func (h *rateLimitProcessHook) DialHook(next redis.DialHook) redis.DialHook { return next } +func (h *rateLimitProcessHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { + return next +} + +func (h *rateLimitProcessHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook { + return func(ctx context.Context, cmd redis.Cmder) error { return h.run(ctx, cmd, next) } +} + +type testLogger struct{} + +func (l *testLogger) Info(string) {} +func (l *testLogger) Error(error) {} +func (l *testLogger) Warn(error) {} +func (l *testLogger) Fatal(error) {} +func (l *testLogger) Trace(string) {} +func (l *testLogger) Debug(string) {} +func (l *testLogger) WithService(string) telemetry.Logger { return l } +func (l *testLogger) WithString(string, string) telemetry.Logger { return l } +func (l *testLogger) WithSpan(trace.Span) telemetry.Logger { return l } +func (l *testLogger) Printf(string, ...interface{}) {}