From e1e4ea8fa91a0c62dea69977d989d0bb2b526b64 Mon Sep 17 00:00:00 2001
From: Jeremy Evans <code@jeremyevans.net>
Date: Fri, 27 Mar 2020 10:29:00 -0700
Subject: [PATCH] Set external encoding correctly for File.open('f',
 FILE::BINARY) on Windows

Previously, the external encoding was only set correctly for
File::BINARY if keyword arguments were provided.  This copies
the logic for the keyword arguments case to the no keyword
arguments case.  Possibly it should be refactored into a
separate function.

Fixes [Bug #16737]
---
 io.c                 | 12 ++++++++++++
 test/ruby/test_io.rb | 12 ++++++++++++
 2 files changed, 24 insertions(+)

diff --git a/io.c b/io.c
index 9634e2c57d..adf5c17f97 100644
--- a/io.c
+++ b/io.c
@@ -5955,6 +5955,18 @@ rb_io_extract_modeenc(VALUE *vmode_p, VALUE *vperm_p, VALUE opthash,
 #endif
 	SET_UNIVERSAL_NEWLINE_DECORATOR_IF_ENC2(enc2, ecflags);
         ecopts = Qnil;
+        if (fmode & FMODE_BINMODE) {
+#ifdef O_BINARY
+            oflags |= O_BINARY;
+#endif
+            if (!has_enc)
+                rb_io_ext_int_to_encs(rb_ascii8bit_encoding(), NULL, &enc, &enc2, fmode);
+        }
+#if DEFAULT_TEXTMODE
+        else if (NIL_P(vmode)) {
+            fmode |= DEFAULT_TEXTMODE;
+        }
+#endif
     }
     else {
 	VALUE v;
diff --git a/test/ruby/test_io.rb b/test/ruby/test_io.rb
index a6df5c8b08..a49fd0130e 100644
--- a/test/ruby/test_io.rb
+++ b/test/ruby/test_io.rb
@@ -3630,15 +3630,27 @@ __END__
   end
 
   def test_open_flag_binary
+    binary_enc = Encoding.find("BINARY")
     make_tempfile do |t|
       open(t.path, File::RDONLY, flags: File::BINARY) do |f|
         assert_equal true, f.binmode?
+        assert_equal binary_enc, f.external_encoding
       end
       open(t.path, 'r', flags: File::BINARY) do |f|
         assert_equal true, f.binmode?
+        assert_equal binary_enc, f.external_encoding
       end
       open(t.path, mode: 'r', flags: File::BINARY) do |f|
         assert_equal true, f.binmode?
+        assert_equal binary_enc, f.external_encoding
+      end
+      open(t.path, File::RDONLY|File::BINARY) do |f|
+        assert_equal true, f.binmode?
+        assert_equal binary_enc, f.external_encoding
+      end
+      open(t.path, File::RDONLY|File::BINARY, autoclose: true) do |f|
+        assert_equal true, f.binmode?
+        assert_equal binary_enc, f.external_encoding
       end
     end
   end if File::BINARY != 0