diff --git a/src/NonlinearSolve.jl b/src/NonlinearSolve.jl
index 9b6b9ec04..f6f6aee1d 100644
--- a/src/NonlinearSolve.jl
+++ b/src/NonlinearSolve.jl
@@ -12,8 +12,8 @@ import PrecompileTools: @recompile_invalidations, @compile_workload, @setup_work
           LinearAlgebra, LinearSolve, MaybeInplace, Preferences, Printf, SciMLBase,
           SimpleNonlinearSolve, SparseArrays, SparseDiffTools
 
-    import ArrayInterface: undefmatrix, can_setindex, restructure, fast_scalar_indexing,
-                           ismutable
+    import ArrayInterface: ArrayInterface, undefmatrix, can_setindex, restructure,
+                           fast_scalar_indexing, ismutable
     import DiffEqBase: AbstractNonlinearTerminationMode,
                        AbstractSafeNonlinearTerminationMode,
                        AbstractSafeBestNonlinearTerminationMode,
diff --git a/src/internal/linear_solve.jl b/src/internal/linear_solve.jl
index f2277a165..783bf4956 100644
--- a/src/internal/linear_solve.jl
+++ b/src/internal/linear_solve.jl
@@ -163,9 +163,11 @@ function (cache::LinearSolverCache)(;
     cache.lincache = linres.cache
     # Unfortunately LinearSolve.jl doesn't have the most uniform ReturnCode handling
     if linres.retcode === ReturnCode.Failure
-        # TODO: We need to guard this somehow because this will surely fail if A is on GPU
-        # TODO: or some fancy Matrix type
-        if !(cache.linsolve isa QRFactorization{ColumnNorm})
+        structured_mat = ArrayInterface.isstructured(cache.lincache.A)
+        is_gpuarray = ArrayInterface.device(cache.lincache.A) isa ArrayInterface.GPU
+        if !(cache.linsolve isa QRFactorization{ColumnNorm}) &&
+           !is_gpuarray &&
+           !structured_mat
             if verbose
                 @warn "Potential Rank Deficient Matrix Detected. Attempting to solve using \
                        Pivoted QR Factorization."
@@ -189,6 +191,20 @@ function (cache::LinearSolverCache)(;
             linres.retcode === ReturnCode.Failure &&
                 return LinearSolveResult(; u = linres.u, success = false)
             return LinearSolveResult(; u = linres.u)
+        elseif !(cache.linsolve isa QRFactorization{ColumnNorm})
+            if verbose
+                if structured_mat
+                    @warn "Potential Rank Deficient Matrix Detected. But Matrix is \
+                           Structured. Currently, we don't attempt to solve Rank Deficient \
+                           Structured Matrices. Please open an issue at \
+                           https://github.com/SciML/NonlinearSolve.jl"
+                elseif is_gpuarray
+                    @warn "Potential Rank Deficient Matrix Detected. But Matrix is on GPU. \
+                           Currently, we don't attempt to solve Rank Deficient GPU \
+                           Matrices. Please open an issue at \
+                           https://github.com/SciML/NonlinearSolve.jl"
+                end
+            end
         end
         return LinearSolveResult(; u = linres.u, success = false)
     end
diff --git a/test/misc/polyalg_tests.jl b/test/misc/polyalg_tests.jl
index 666db02ab..c2864759c 100644
--- a/test/misc/polyalg_tests.jl
+++ b/test/misc/polyalg_tests.jl
@@ -250,12 +250,7 @@ end
     u0 = @SVector [0.0, 0.0, 0.0]
     prob = NonlinearProblem(f1_infeasible, u0)
 
-    try
-        sol = solve(prob)
-        @test all(!isnan, sol.u)
-        @test !SciMLBase.successful_retcode(sol.retcode)
-        @inferred solve(prob)
-    catch err
-        @test err isa LinearAlgebra.SingularException
-    end
+    sol = solve(prob)
+    @test all(!isnan, sol.u)
+    @test !SciMLBase.successful_retcode(sol.retcode)
 end