diff --git a/jwt-authorizer/src/jwks/key_store_manager.rs b/jwt-authorizer/src/jwks/key_store_manager.rs
index 2ff3edf..86eba6a 100644
--- a/jwt-authorizer/src/jwks/key_store_manager.rs
+++ b/jwt-authorizer/src/jwks/key_store_manager.rs
@@ -161,6 +161,11 @@ impl KeyStore {
                 self.fail_time = Some(Instant::now());
                 AuthError::JwksRefreshError(e.to_string())
             })?
+            .error_for_status()
+            .map_err(|e| {
+                self.fail_time = Some(Instant::now());
+                AuthError::JwksRefreshError(e.to_string())
+            })?
             .json::<JwkSet>()
             .await
             .map(|jwks| {