Use net-imap's SASL implementation 🚧[WIP]🚧

This commit converts `#authenticate` to use `net-imap` as a generic
fallback for mechanisms that haven't otherwise been added (as subclasses
of `Authenticator`).  In this commit, the original implementation is
still used by `#authenticate` for the `PLAIN`, `LOGIN`, and `CRAM-MD5`
mechanisms.  Every other mechanism supported by `net-imap` v0.4.0 is
added here:
* `ANONYMOUS`
* `DIGEST-MD5` _(deprecated)_
* `EXTERNAL`
* `OAUTHBEARER`
* `SCRAM-SHA-1` and `SCRAM-SHA-256`
* `XOAUTH`

**TODO:** Ideally, `net-smtp` and `net-imap` should both depend on a
shared `sasl` or `net-sasl` gem, rather than keep the SASL
implementation inside one or the other.  See
ruby/net-imap#23.

**TODO:** since we already know the authenticator arguments up-front, we
can validate authenticator arguments by simply creating the
authenticator object and rely on the its initializer to raise
ArgumentError for missing args.

.github/workflows/test.yml

@@ -19,4 +19,4 @@ jobs:
     - name: Install dependencies
       run: bundle install
     - name: Run test
-      run: rake test
+      run: bundle exec rake test

lib/net/smtp.rb

@@ -171,8 +171,11 @@ class SMTPUnsupportedCommand < ProtocolError
   #
   # === SMTP Authentication
   #
-  # The Net::SMTP class supports three authentication schemes;
-  # PLAIN, LOGIN and CRAM MD5.  (SMTP Authentication: [RFC2554])
+  # The Net::SMTP class supports several authentication schemes;
+  # ({SMTP Authentication: [RFC4956]}[https://www.rfc-editor.org/rfc/rfc4954.html])
+  # +ANONYMOUS+, +EXTERNAL+, +OAUTHBEARER+, +PLAIN+, +SCRAM-SHA-1+,
+  # +SCRAM-SHA-256+, and +XOAUTH2+.
+  #
   # To use SMTP authentication, pass extra arguments to
   # SMTP.start or SMTP#start.
   #
@@ -182,26 +185,43 @@ class SMTPUnsupportedCommand < ProtocolError
   #     Net::SMTP.start("your.smtp.server", 25,
   #                     auth: {type:     :plain,
   #                            username: "authentication identity",
-  #                            password: password})
+  #                            password: password,
+  #                            authzid:  "authorization identity"}) # optional
   #
-  #     # LOGIN
-  #     Net::SMTP.start('your.smtp.server', 25,
-  #                     user: 'Your Account', secret: 'Your Password', authtype: :login)
+  #     # SCRAM-SHA-256
   #     Net::SMTP.start("your.smtp.server", 25,
-  #                     auth: {type:     :login,
+  #                     user: "authentication identity", secret: password,
+  #                     authtype: :scram_sha_256)
+  #     Net::SMTP.start("your.smtp.server", 25,
+  #                     auth: {type:     :scram_sha_256,
   #                            username: "authentication identity",
-  #                            password: password})
+  #                            password: password,
+  #                            authzid:  "authorization identity"}) # optional
   #
-  #     # CRAM MD5
-  #     Net::SMTP.start('your.smtp.server', 25,
-  #                     user: 'Your Account', secret: 'Your Password', authtype: :cram_md5)
+  #     # OAUTHBEARER
+  #     Net::SMTP.start("your.smtp.server", 25,
+  #                     auth: {type:         :oauthbearer,
+  #                            oauth2_token: oauth2_access_token,
+  #                            authzid:      "authorization identity", # optional
+  #                            host:         "your.smtp.server",       # optional
+  #                            port:         25})                      # optional
+  #
+  #     # XOAUTH2
+  #     Net::SMTP.start("your.smtp.server", 25,
+  #                     user: "username", secret: oauth2_access_token, authtype: :xoauth2)
   #     Net::SMTP.start("your.smtp.server", 25,
-  #                     auth: {type:     :cram_md5,
-  #                            username: 'Your Account',
-  #                            password: 'Your Password'})
+  #                     auth: {type:         :xoauth2,
+  #                            username:     "username",
+  #                            oauth2_token: oauth2_token})
   #
-  # +LOGIN+, and +CRAM-MD5+ are still available for backwards compatibility, but
-  # are deprecated and should be avoided.
+  #     # EXTERNAL
+  #     Net::SMTP.start("your.smtp.server", 587,
+  #                     starttls: :always, ssl_context_params: ssl_ctx_params,
+  #                     authtype: "external")
+  #
+  # +DIGEST-MD5+, +LOGIN+, and +CRAM-MD5+ are still available for backwards
+  # compatibility, but are deprecated and should be avoided.  <em>Using a
+  # deprecated authentication mechanisms will print a warning.</em>
   #
   class SMTP < Protocol
     VERSION = "0.4.0"
@@ -501,12 +521,6 @@ def debug_output=(arg)
     # +helo+ is the _HELO_ _domain_ provided by the client to the
     # server (see overview comments); it defaults to 'localhost'.
     #
-    # The remaining arguments are used for SMTP authentication, if required
-    # or desired.  +user+ is the account name; +secret+ is your password
-    # or other authentication token; and +authtype+ is the authentication
-    # type, one of :plain, :login, or :cram_md5.  See the discussion of
-    # SMTP Authentication in the overview notes.
-    #
     # If +tls+ is true, enable TLS. The default is false.
     # If +starttls+ is :always, enable STARTTLS, if +:auto+, use STARTTLS when the server supports it,
     # if false, disable STARTTLS.
@@ -520,6 +534,13 @@ def debug_output=(arg)
     #
     # +tls_verify: true+ is equivalent to +ssl_context_params: { verify_mode: OpenSSL::SSL::VERIFY_PEER }+.
     #
+    # The remaining arguments are used for SMTP authentication, if required or
+    # desired.  +user+ or +username+ is the authentication or authorization
+    # identity (depending on +authtype+); +secret+ or +password+ is your
+    # password or other authentication token; and +authtype+ is the
+    # authentication type.  +auth+ is a hash of arbitrary keyword parameters for
+    # #auth.  See the discussion of SMTP Authentication in the overview notes.
+    #
     # === Errors
     #
     # This method may raise:
@@ -565,10 +586,13 @@ def started?
     # +helo+ is the _HELO_ _domain_ that you'll dispatch mails from; see
     # the discussion in the overview notes.
     #
-    # If either +auth+ or +user+ are given, SMTP authentication will be
-    # attempted using the AUTH command.  +authtype+ specifies the type of
-    # authentication to attempt; it must be one of :login, :plain, and
-    # :cram_md5.  See the notes on SMTP Authentication in the overview.
+    # If +user+, +username+, +secret+, +password+, +authtype+, or +auth+ given,
+    # SMTP authentication will be attempted using the #auth command.  +authtype+
+    # specifies the SASL mechanism to attempt; +user+ or +username+ is the
+    # authentication or authorization identity (depending on +authtype+);
+    # +secret+ or +password+ is your password or other authentication token;
+    # +auth+ is a hash of arbitrary keyword parameters for #auth.  See the
+    # discussion of SMTP Authentication in the overview notes.
     #
     # === Block Usage
     #
@@ -871,15 +895,16 @@ def authenticate(user, secret, authtype = DEFAULT_AUTH_TYPE)
     # include +authcid+ for authentication identity, +authzid+ for authorization
     # identity, +username+ for either "authentication identity" or
     # "authorization identity" depending on the +mechanism+, and +password+.
+    # Keyword arguments that do not apply to the +mechanism+ may be silently
+    # ignored.
     def auth(*args, **kwargs, &blk)
       args, kwargs = backward_compatible_auth_args(*args, **kwargs)
-      authtype, *args = args
-      authenticator = Authenticator.auth_class(authtype).new(self)
-      if kwargs.empty?
-        # TODO: remove this, unless it is needed for 2.6/2.7/3.0 compatibility
-        critical { authenticator.auth(*args, &blk) }
-      else
-        critical { authenticator.auth(*args, **kwargs, &blk) }
+      critical do
+        Authenticator::SASLAdapter.new(self).authenticate(*args, **kwargs, &blk)
+      rescue SMTPServerBusy, SMTPSyntaxError, SMTPFatalError => error
+        raise SMTPAuthenticationError.new(error.response)
+      rescue SASL::AuthenticationIncomplete => error
+        raise error.response.exception_class.new(error.response)
       end
     end
 
@@ -919,21 +944,28 @@ def merge_auth_params(user, secret, authtype, auth)
       auth
     end
 
-    # Convert +type+, +username+, +secret+ (etc) kwargs to positional args, for
-    # compatibility with existing authenticators.
-    def backward_compatible_auth_args(_type = nil, *args, type: nil,
-                                      username: nil, authcid: nil,
-                                      secret: nil, password: nil,
-                                      **kwargs)
-      type && _type and
+    def backward_compatible_auth_args(authtype = nil, *args,
+                                      type: nil, secret: nil, **kwargs)
+      type && authtype and
         raise ArgumentError, 'conflict between "type" keyword argument ' \
                              'and positional argument'
-      type ||= _type || DEFAULT_AUTH_TYPE
+      type ||= authtype || DEFAULT_AUTH_TYPE
       check_auth_method(type)
+      if secret
+        secret_type = type.match?(/\AX?OAUTH/i) ? :oauth2_token : :password
+        kwargs.key?(secret_type) and
+          raise ArgumentError 'conflict between "secret" and %p keyword args' % [
+            secret_type.to_s
+          ]
+        kwargs[secret_type] = secret
+      end
       auth_class = Authenticator.auth_class(type)
-      if auth_class.is_a?(Class) && auth_class <= Authenticator
-        args[0] ||= authcid  || username
-        args[1] ||= password || secret
+      if auth_class.is_a?(Class) && auth_class <= Authenticator ||
+          type.match?(/\A(?:LOGIN|CRAM[-_]MD5)\z/i)
+        usernames = [kwargs.delete(:authcid), kwargs.delete(:username)]
+        secrets   = [kwargs.delete(:password)]
+        args[0] ||= usernames.compact.first
+        args[1] ||= secrets.compact.first
         check_auth_args(args[0], args[1], type)
       end
       [[type, *args], kwargs]
@@ -1047,6 +1079,27 @@ def get_response(reqline)
       recv_response()
     end
 
+    # Returns a successful Response.
+    #
+    # Yields continuation data.
+    #
+    # This method may raise:
+    #
+    # * Net::SMTPAuthenticationError
+    # * Net::SMTPServerBusy
+    # * Net::SMTPSyntaxError
+    # * Net::SMTPFatalError
+    # * Net::SMTPUnknownError
+    def send_command_with_continuations(*args)
+      server_resp = get_response args.join(" ")
+      while server_resp.continue?
+        client_resp = yield server_resp.string.strip.split(nil, 2).last
+        server_resp = get_response client_resp
+      end
+      server_resp.success? or raise server_resp.exception_class.new(server_resp)
+      server_resp
+    end
+
     private
 
     def validate_line(line)

lib/net/smtp/auth_sasl_adapter.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "net/imap"
+
+module Net
+  class SMTP
+    SASL = Net::IMAP::SASL
+
+    class Authenticator
+
+      # Experimental
+      #
+      # Initialize with a block that runs a command, yielding for continuations.
+      class SASLAdapter < SASL::ClientAdapter
+        include SASL::ProtocolAdapters::SMTP
+
+        RESPONSE_ERRORS = [
+          SMTPAuthenticationError,
+          SMTPServerBusy,
+          SMTPSyntaxError,
+          SMTPFatalError,
+        ].freeze
+
+        def initialize(...)
+          super
+          @command_proc ||= client.method(:send_command_with_continuations)
+        end
+
+        def host;                     client.address                  end
+        def response_errors;          RESPONSE_ERRORS                 end
+        def sasl_ir_capable?;         true                            end # TODO
+        def auth_capable?(mechanism); client.auth_capable?(mechanism) end
+        def drop_connection;          client.finish                   end
+        def drop_connection!;         client.finish                   end # TODO
+      end
+
+    end
+  end
+end

lib/net/smtp/auth_sasl_compatibility_adapter.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Net
+  class SMTP
+
+    # Curries arguments to SMTP#auth, using the Authenticator API.
+    #
+    # Net::SMTP#authenticate still supports the v0.4.0 Authenticator API, so
+    # Authenticator subclasses can still be added and used with it.  This class
+    # will be used as the default, when no matching Authenticator subclass
+    # exists.
+    class CompatibilityAdapter
+      def initialize(mechanism) @mechanism = mechanism end
+      def new(smtp) @smtp = smtp; self end
+      def auth(*args, **kwargs, &block)
+        args.pop while args.any? && args.last.nil?
+        @smtp.auth(@mechanism, *args, **kwargs, &block)
+      end
+    end
+
+    Authenticator.auth_classes.default_proc = ->_, mechanism {
+      CompatibilityAdapter.new(mechanism)
+    }
+
+  end
+end

net-smtp.gemspec

@@ -26,4 +26,5 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "net-protocol"
+  spec.add_dependency "net-imap", ">= 0.4.1" # experimental SASL support
 end

test/net/smtp/test_smtp.rb

@@ -530,16 +530,6 @@ def test_start_auth_cram_md5
       assert_raise Net::SMTPAuthenticationError do
         Net::SMTP.start('localhost', port, user: 'account', password: 'password', authtype: :cram_md5){}
       end
-
-      port = fake_server_start(auth: 'CRAM-MD5')
-      smtp = Net::SMTP.new('localhost', port)
-      auth_cram_md5 = Net::SMTP::AuthCramMD5.new(smtp)
-      auth_cram_md5.define_singleton_method(:digest_class) { raise '"openssl" or "digest" library is required' }
-      Net::SMTP::AuthCramMD5.define_singleton_method(:new) { |_| auth_cram_md5 }
-      e = assert_raise RuntimeError do
-        smtp.start(user: 'account', password: 'password', authtype: :cram_md5){}
-      end
-      assert_equal('"openssl" or "digest" library is required', e.message)
     end
 
     def test_start_instance