AuthorizationFilter throws exception

Question

Trying to register an AuthorizationFilter against my controllers. I have previously set one up but that is an IActionFilter but when doing the same thing with an AuthorizationFilter instead it throws this exception:

System.InvalidOperationException: No authenticationScheme was specified, and there was no DefaultChallengeScheme found. The default schemes can be set using either AddAuthentication(string defaultScheme) or AddAuthentication(Action<AuthenticationOptions> configureOptions).

My code looks like this: Program.cs

builder.Services.AddControllers(options => {
    options.Filters.Add<SecureByDefaultFilter>();
    options.Filters.Add<AuthorisedFilter>();
});

AuthorizationFilter that doesn't work

internal class AuthorisedFilter : IAuthorizationFilter {
        public void OnAuthorization(AuthorizationFilterContext context) {
            var authorizedAttribute = context.ActionDescriptor.EndpointMetadata.OfType<AuthorisedAttribute>().FirstOrDefault();

            if (authorizedAttribute?.Roles == "Public") {
                return;
            }

            Console.WriteLine(authorizedAttribute?.Roles);
        }
    }

Action filter that works

internal class SecureByDefaultFilter : IActionFilter {
        public void OnActionExecuting(ActionExecutingContext context) {
            var authorizedAttribute = context.ActionDescriptor.EndpointMetadata.OfType<AuthorisedAttribute>().FirstOrDefault();
            
            if (authorizedAttribute == null) {
                throw new MissingAttributeException("Endpoint missing Authorised Attribute");
            }      
        }

        public void OnActionExecuted(ActionExecutedContext context) { }
    }