🔀 Merge pull request #696 from ruby/backport/v0.5-multiple

🍒 Backports #675, #676, #677, #678, #679, #681 to v0.5

Author: nevans

lib/net/imap.rb

@@ -2209,6 +2209,7 @@ def uid_expunge(uid_set)
     # provided as an array or a string.
     # See {"Argument translation"}[rdoc-ref:#search@Argument+translation]
     # and {"Search criteria"}[rdoc-ref:#search@Search+criteria], below.
+    # <em>Please note</em> the warning for when +criteria+ is a String.
     #
     # +return+ options control what kind of information is returned about
     # messages matching the search +criteria+.  Specifying +return+ should force
@@ -2619,7 +2620,8 @@ def search(...)
     # backward compatibility) but adds SearchResult#modseq when the +CONDSTORE+
     # capability has been enabled.
     #
-    # See #search for documentation of parameters.
+    # See #search for documentation of parameters.  <em>Please note</em> the
+    # warning for when +criteria+ is a String.
     #
     # ==== Capabilities
     #
@@ -2705,7 +2707,8 @@ def fetch(...)
     # {SequenceSet[...]}[rdoc-ref:SequenceSet@Creating+sequence+sets].
     # (For message sequence numbers, use #fetch instead.)
     #
-    # +attr+ behaves the same as with #fetch.
+    # +attr+ behaves the same as with #fetch.  <em>Please note</em> the #fetch
+    # warning on the +attr+ argument.
     # >>>
     #   *Note:* Servers _MUST_ implicitly include the +UID+ message data item as
     #   part of any +FETCH+ response caused by a +UID+ command, regardless of
@@ -2917,8 +2920,10 @@ def uid_move(set, mailbox)
 
     # Sends a {SORT command [RFC5256 §3]}[https://www.rfc-editor.org/rfc/rfc5256#section-3]
     # to search a mailbox for messages that match +search_keys+ and return an
-    # array of message sequence numbers, sorted by +sort_keys+.  +search_keys+
-    # are interpreted the same as for #search.
+    # array of message sequence numbers, sorted by +sort_keys+.
+    #
+    # +search_keys+ are interpreted the same as the +criteria+ argument for
+    # #search.  <em>Please note</em> the #search warning for String +criteria+.
     #
     #--
     # TODO: describe +sort_keys+
@@ -2943,8 +2948,10 @@ def sort(sort_keys, search_keys, charset)
 
     # Sends a {UID SORT command [RFC5256 §3]}[https://www.rfc-editor.org/rfc/rfc5256#section-3]
     # to search a mailbox for messages that match +search_keys+ and return an
-    # array of unique identifiers, sorted by +sort_keys+.  +search_keys+ are
-    # interpreted the same as for #search.
+    # array of unique identifiers, sorted by +sort_keys+.
+    #
+    # +search_keys+ are interpreted the same as the +criteria+ argument for
+    # #search.  <em>Please note</em> the #search warning for String +criteria+.
     #
     # Related: #sort, #search, #uid_search, #thread, #uid_thread
     #
@@ -2958,8 +2965,10 @@ def uid_sort(sort_keys, search_keys, charset)
 
     # Sends a {THREAD command [RFC5256 §3]}[https://www.rfc-editor.org/rfc/rfc5256#section-3]
     # to search a mailbox and return message sequence numbers in threaded
-    # format, as a ThreadMember tree.  +search_keys+ are interpreted the same as
-    # for #search.
+    # format, as a ThreadMember tree.
+    #
+    # +search_keys+ are interpreted the same as the +criteria+ argument for
+    # #search.  <em>Please note</em> the #search warning for String +criteria+.
     #
     # The supported algorithms are:
     #
@@ -2985,6 +2994,9 @@ def thread(algorithm, search_keys, charset)
     # Similar to #thread, but returns unique identifiers instead of
     # message sequence numbers.
     #
+    # +search_keys+ are interpreted the same as the +criteria+ argument for
+    # #search.  <em>Please note</em> the #search warning for String +criteria+.
+    #
     # Related: #thread, #search, #uid_search, #sort, #uid_sort
     #
     # ==== Capabilities

lib/net/imap/command_data.rb

@@ -17,14 +17,15 @@ def validate_data(data)
       when nil
       when String
       when Integer
-        NumValidator.ensure_number(data)
+        # Covers modseq-valzer, which is the largest valid IMAP integer
+        if data.negative?
+          raise DataFormatError, "Integer argument must be unsigned: #{data}"
+        elsif 0xffff_ffff_ffff_ffff < data
+          raise DataFormatError, "Integer argument must fit in 64 bits: #{data}"
+        end
       when Array
-        if data[0] == 'CHANGEDSINCE'
-          NumValidator.ensure_mod_sequence_value(data[1])
-        else
-          data.each do |i|
-            validate_data(i)
-          end
+        data.each do |i|
+          validate_data(i)
         end
       when Time, Date, DateTime
       when Symbol
@@ -112,8 +113,9 @@ def send_literal(str, tag = nil, binary: false, non_sync: nil)
       end
     end
 
+    # NOTE: +num+ should already be an Integer
     def send_number_data(num)
-      put_string(num.to_s)
+      put_string(Integer(num).to_s)
     end
 
     def send_list_data(list, tag = nil)
@@ -190,7 +192,12 @@ def send_data(imap, tag) = imap.__send__(:put_string, data)
 
     class RawData < CommandData # :nodoc:
       def initialize(data:)
-        data = split_parts(data)
+        case data
+        in String then data = self.class.split(data)
+        in Array  if   data.all? { _1 in RawText | Literal }
+        else
+          raise TypeError, "expected String or Array[#{RawText} | #{Literal}]"
+        end
         super
         validate
       end
@@ -204,9 +211,11 @@ def validate
         end
       end
 
-      private
-
-      def split_parts(data)
+      # Splits an input +string+ into an array of RawText and Literal/Literal8.
+      #
+      # NOTE: unlike RawData#validate, this does not prevent the final RawText
+      # from ending with a literal prefix.
+      def self.split(data)
         data = data.b # dups and ensures BINARY encoding
         parts = []
         while data.match(/(~)?\{(0|[1-9]\d*)(\+)?\}\r\n/n)
@@ -220,14 +229,15 @@ def split_parts(data)
         parts
       end
 
-      def extract_literal(data, binary:, bytesize:, non_sync:)
+      def self.extract_literal(data, binary:, bytesize:, non_sync:)
         if data.bytesize < bytesize
           raise DataFormatError, "Too few bytes in string for literal, " \
             "expected: %s, remaining: %s" % [bytesize, data.bytesize]
         end
         literal = data.byteslice(0, bytesize)
         (binary ? Literal8 : Literal).new(data: literal, non_sync:)
       end
+      private_class_method :extract_literal
     end
 
     class Atom < CommandData # :nodoc:

lib/net/imap/response_parser.rb

@@ -2189,10 +2189,7 @@ def next_token
             if $1
               return Token.new(T_SPACE, $+)
             elsif $2
-              len = $+.to_i
-              val = @str[@pos, len]
-              @pos += len
-              return Token.new(T_LITERAL8, val)
+              literal_token($+, T_LITERAL8)
             elsif $3 && $7
               # greedily match ATOM, prefixed with NUMBER, NIL, or PLUS.
               return Token.new(T_ATOM, $3)
@@ -2220,10 +2217,7 @@ def next_token
             elsif $15
               return Token.new(T_RBRA, $+)
             elsif $16
-              len = $+.to_i
-              val = @str[@pos, len]
-              @pos += len
-              return Token.new(T_LITERAL, val)
+              literal_token($+)
             elsif $17
               return Token.new(T_PERCENT, $+)
             elsif $18
@@ -2249,10 +2243,7 @@ def next_token
             elsif $4
               return Token.new(T_QUOTED, Patterns.unescape_quoted($+))
             elsif $5
-              len = $+.to_i
-              val = @str[@pos, len]
-              @pos += len
-              return Token.new(T_LITERAL, val)
+              literal_token($+)
             elsif $6
               return Token.new(T_LPAR, $+)
             elsif $7
@@ -2267,6 +2258,23 @@ def next_token
         else
           parse_error("invalid @lex_state - %s", @lex_state.inspect)
         end
+      rescue DataFormatError => error
+        parse_error error.message
+      end
+
+      def literal_token(len, type = T_LITERAL)
+        len = coerce_number64 len.to_i
+        val = @str[@pos, len]
+        @pos += len
+        Token.new(type, val)
+      end
+
+      # copied/adapted from NumValidator in v0.6
+      def coerce_number64(num)
+        int = num.to_i
+        return int if 0 <= int && int <= 0x7fff_ffff_ffff_ffff
+        raise DataFormatError,
+          "number64 must be unsigned 63-bit integer: #{num}"
       end
 
     end

lib/net/imap/response_reader.rb

@@ -4,6 +4,8 @@ module Net
   class IMAP
     # See https://www.rfc-editor.org/rfc/rfc9051#section-2.2.2
     class ResponseReader # :nodoc:
+      include NumValidator
+
       attr_reader :client
 
       def initialize(client, sock)
@@ -35,7 +37,10 @@ def done?               = line_done? && !literal_size
       def line_done?          = buff.end_with?(CRLF)
 
       def get_literal_size(buff)
-        buff.end_with?("}\r\n") && buff.rindex(/\{(\d+)\}\r\n\z/n) && $1.to_i
+        buff.end_with?("}\r\n") && buff.rindex(/\{(\d+)\}\r\n\z/n) &&
+          coerce_number64($1)
+      rescue DataFormatError
+        raise DataFormatError, format("invalid response literal size (%s)", $1)
       end
 
       def read_line
@@ -74,6 +79,14 @@ def max_response_remaining!
         )
       end
 
+      # copied/adapted from NumValidator in v0.6
+      def coerce_number64(num)
+        int = num.to_i
+        return int if 0 <= int && int <= 0x7fff_ffff_ffff_ffff
+        raise DataFormatError,
+          "number64 must be unsigned 63-bit integer: #{num}"
+      end
+
     end
   end
 end

test/net/imap/fake_server/command_reader.rb

@@ -3,13 +3,15 @@
 require "net/imap"
 
 class Net::IMAP::FakeServer
-  CommandParseError = RuntimeError
+  CommandParseError = Class.new(RuntimeError)
 
   class CommandReader
+    attr_reader :config
     attr_reader :last_command
     attr_accessor :literal_acceptor
 
-    def initialize(socket)
+    def initialize(socket, config:)
+      @config = config
       @socket = socket
       @last_command = nil
       @literal_acceptor = proc {|buff, size| true }
@@ -35,8 +37,11 @@ def get_command
       end
       throw :eof if buf.empty?
       @last_command = parse(buf)
-    rescue CommandParseError => err
-      raise IOError, err.message if socket.eof? && !buf.end_with?("\r\n")
+    rescue CommandParseError
+      if config.ignore_abrupt_eof? && socket.eof? && !buf.end_with?("\r\n")
+        throw :eof
+      end
+      raise
     end
 
     private
@@ -46,7 +51,7 @@ def get_command
     # TODO: convert bad command exception to tagged BAD response, when possible
     def parse(buf)
       /\A([^ ]+) ((?:UID )?\w+)(?: (.+))?\r\n\z/min =~ buf or
-        raise CommandParseError, "bad request: %p" [buf]
+        raise CommandParseError, "bad request: %p" % [buf]
       case $2.upcase
       when "LOGIN", "SELECT", "EXAMINE", "ENABLE", "AUTHENTICATE"
         Command.new $1, $2, scan_astrings($3), buf

test/net/imap/fake_server/configuration.rb

@@ -45,6 +45,8 @@ class Configuration
       mailboxes: {
         "INBOX" => { name: "INBOX" }.freeze,
       }.freeze,
+
+      ignore_abrupt_eof: false,
     }
 
     def initialize(with_extensions: [], without_extensions: [], **opts, &block)
@@ -68,6 +70,7 @@ def initialize(with_extensions: [], without_extensions: [], **opts, &block)
     alias greeting_bye?          greeting_bye
     alias greeting_capabilities? greeting_capabilities
     alias sasl_ir?               sasl_ir
+    alias ignore_abrupt_eof?     ignore_abrupt_eof
 
     def on(event, &handler)
       handler or raise ArgumentError

test/net/imap/fake_server/connection.rb

@@ -12,7 +12,7 @@ def initialize(server, tcp_socket:)
       @config = server.config
       @socket = Socket.new tcp_socket, config: config
       @state  = ConnectionState.new socket: socket, config: config
-      @reader = CommandReader.new  socket
+      @reader = CommandReader.new  socket, config: config
       @writer = ResponseWriter.new socket, config: config, state: state
       @router = CommandRouter.new  writer, config: config, state: state
       @mutex  = Thread::Mutex.new

test/net/imap/fake_server/socket.rb

@@ -18,10 +18,10 @@ def initialize(tcp_socket, config:)
     def tls?; !!@tls_socket end
     def closed?; @closed end
 
-    def eof?;      socket.eof?       end
-    def gets(...)  socket.gets(...)  end
-    def read(...)  socket.read(...)  end
-    def print(...) socket.print(...) end
+    def eof?;      ignore_closed?(true) { socket.eof?       } end
+    def gets(...)  ignore_closed?(nil)  { socket.gets(...)  } end
+    def read(...)  ignore_closed?(nil)  { socket.read(...)  } end
+    def print(...) ignore_closed?(nil)  { socket.print(...) } end
 
     def use_tls
       @tls_socket ||= OpenSSL::SSL::SSLSocket.new(tcp_socket, ssl_ctx).tap do |s|
@@ -48,5 +48,13 @@ def ssl_ctx
       end
     end
 
+    def ignore_closed?(fallback)
+      yield
+    rescue IOError => err
+      close if !closed? && (@tcp_socket.closed? || @tls_socket.closed?)
+      return fallback if err.message.match?(/stream closed|closed stream/i)
+      raise
+    end
+
   end
 end

test/net/imap/fixtures/response_parser/quirky_behaviors.yml

@@ -7,6 +7,22 @@
       data:
       raw_data: "* NOOP\r\n"
 
+  "literal numeric formatted with zero-prefix":
+    :response: "* 20367 FETCH (BODY[HEADER.FIELDS (Foo)] {012}\r\nFoo: bar\r\n\r\n)\r\n"
+    :expected: !ruby/struct:Net::IMAP::UntaggedResponse
+      name: FETCH
+      data: !ruby/struct:Net::IMAP::FetchData
+        seqno: 20367
+        attr:
+          BODY[HEADER.FIELDS (Foo)]: "Foo: bar\r\n\r\n"
+      raw_data: "* 20367 FETCH (BODY[HEADER.FIELDS (Foo)] {012}\r\nFoo: bar\r\n\r\n)\r\n"
+
+  "invalid literal numeric format (too large)":
+    :test_type: :assert_parse_failure
+    :message: "number64 must be unsigned 63-bit integer: 99999999999999999999"
+    :response:
+      "* 20367 FETCH (BODY[] {99999999999999999999}\r\nwon't parse this)\r\n"
+
   test_invalid_noop_response_with_unparseable_data:
     :response: "* NOOP froopy snood\r\n"
     :expected: !ruby/struct:Net::IMAP::IgnoredResponse

test/net/imap/test_command_data.rb

@@ -356,6 +356,23 @@ class RawDataTest < CommandDataTest
       raw = RawData.new(data: " {123} ")
       assert_equal [RawText[" {123} "]], raw.data
     end
+
+    data(
+      "simple raw text"     => 'hello "world"',
+      "text, literal, text" => "OK {5}\r\nhello {5}\r\nworld",
+      "empty literals"      => "{0}\r\n{0+}\r\n~{0}\r\n~{0+}\r\n",
+      "binary and regular"  => "foo ~{7}\r\n\0bar\r\nbaz {4}\r\nquux",
+    )
+    test ".split" do |string|
+      assert_equal(RawData[string].data, RawData.split(string))
+    end
+
+    test ".split allows final literal prefix" do
+      assert_equal [RawText["text {123}"]],     RawData.split("text {123}")
+      assert_equal [RawText["text+ {123+}"]],   RawData.split("text+ {123+}")
+      assert_equal [RawText["~text ~{123}"]],   RawData.split("~text ~{123}")
+      assert_equal [RawText["~text+ ~{123+}"]], RawData.split("~text+ ~{123+}")
+    end
   end
 
 end