Specify encoding when reading the user's script. Fixes #399
diff --git a/lib/streamlit/ScriptRunner.py b/lib/streamlit/ScriptRunner.py
index b16d3b689257..f1cb545558b7 100644
--- a/lib/streamlit/ScriptRunner.py
+++ b/lib/streamlit/ScriptRunner.py
@@ -22,6 +22,7 @@
 
 from streamlit import config
 from streamlit import magic
+from streamlit import source_util
 from streamlit.ReportThread import ReportThread
 from streamlit.ScriptRequestQueue import ScriptRequest
 from streamlit.logger import get_logger
@@ -244,7 +245,7 @@ def _run_script(self, rerun_data):
             # Python 3 got rid of the native execfile() command, so we read
             # the file, compile it, and exec() it. This implementation is
             # compatible with both 2 and 3.
-            with open(self._report.script_path) as f:
+            with source_util.open_python_file(self._report.script_path) as f:
                 filebody = f.read()
 
             if config.get_option("runner.magicEnabled"):
diff --git a/lib/streamlit/__init__.py b/lib/streamlit/__init__.py
index 5c16e986ba58..f29586e0562e 100644
--- a/lib/streamlit/__init__.py
+++ b/lib/streamlit/__init__.py
@@ -92,6 +92,7 @@
 
 from streamlit import code_util as _code_util
 from streamlit import util as _util
+from streamlit import source_util as _source_util
 from streamlit.ReportThread import get_report_ctx, add_report_ctx
 from streamlit.DeltaGenerator import DeltaGenerator as _DeltaGenerator
 
@@ -537,7 +538,7 @@ def echo():
         else:
             end_line = frame[1]
         lines_to_display = []
-        with open(filename) as source_file:
+        with source_util.open_python_file(filename) as source_file:
             source_lines = source_file.readlines()
             lines_to_display.extend(source_lines[start_line:end_line])
             initial_spaces = _SPACES_RE.match(lines_to_display[0]).end()
diff --git a/lib/streamlit/caching.py b/lib/streamlit/caching.py
index 2c75c996ff11..60f0b0c1f693 100644
--- a/lib/streamlit/caching.py
+++ b/lib/streamlit/caching.py
@@ -611,8 +611,9 @@ def has_changes(self):
         context_indent = len(code_context) - len(code_context.lstrip())
 
         lines = []
-        # TODO: Memoize open(filename, 'r') in a way that clears the memoized version with each
-        # run of the user's script. Then use the memoized text here, in st.echo, and other places.
+        # TODO: Memoize open(filename, 'r') in a way that clears the memoized
+        # version with each run of the user's script. Then use the memoized
+        # text here, in st.echo, and other places.
         with open(filename, "r") as f:
             for line in f.readlines()[caller_lineno:]:
                 if line.strip() == "":
diff --git a/lib/streamlit/config.py b/lib/streamlit/config.py
index 65a059a89390..36a2bf07070c 100644
--- a/lib/streamlit/config.py
+++ b/lib/streamlit/config.py
@@ -814,7 +814,7 @@ def parse_config_file():
         if not os.path.exists(filename):
             continue
 
-        with open(filename) as input:
+        with open(filename, "r") as input:
             file_contents = input.read()
 
         _update_config_with_toml(file_contents, filename)
diff --git a/lib/streamlit/elements/exception_proto.py b/lib/streamlit/elements/exception_proto.py
index 403f96aaeb61..4bc4a579bbca 100644
--- a/lib/streamlit/elements/exception_proto.py
+++ b/lib/streamlit/elements/exception_proto.py
@@ -39,20 +39,25 @@ def _format_syntax_error_message(exception):
     str
 
     """
-    return (
-        'File "%(filename)s", line %(lineno)d\n'
-        "  %(text)s\n"
-        "  %(caret_indent)s^\n"
-        "%(errname)s: %(msg)s"
-        % {
-            "filename": exception.filename,
-            "lineno": exception.lineno,
-            "text": exception.text.rstrip(),
-            "caret_indent": " " * max(exception.offset - 1, 0),
-            "errname": type(exception).__name__,
-            "msg": exception.msg,
-        }
-    )
+    if exception.text:
+        return (
+            'File "%(filename)s", line %(lineno)d\n'
+            "  %(text)s\n"
+            "  %(caret_indent)s^\n"
+            "%(errname)s: %(msg)s"
+            % {
+                "filename": exception.filename,
+                "lineno": exception.lineno,
+                "text": exception.text.rstrip(),
+                "caret_indent": " " * max(exception.offset - 1, 0),
+                "errname": type(exception).__name__,
+                "msg": exception.msg,
+            }
+        )
+    # If a few edge cases, SyntaxErrors don't have all these nice fields. So we
+    # have a fall back here.
+    # Example edge case error message: encoding declaration in Unicode string
+    return str(exception)
 
 
 def marshall(exception_proto, exception, exception_traceback=None):
diff --git a/lib/streamlit/source_util.py b/lib/streamlit/source_util.py
new file mode 100644
index 000000000000..7fd4adc1d3fa
--- /dev/null
+++ b/lib/streamlit/source_util.py
@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+# Copyright 2018-2019 Streamlit Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def open_python_file(filename):
+    """Open a read-only Python file taking proper care of its encoding.
+
+    In Python 3, we would like all files to be opened with utf-8 encoding.
+    However, some author like to specify PEP263 headers in their source files
+    with their own encodings. In that case, we should respect the author's
+    encoding.
+    """
+    import tokenize
+    if hasattr(tokenize, 'open'):  # Added in Python 3.2
+        # Open file respecting PEP263 encoding. If no encoding header is
+        # found, opens as utf-8.
+        return tokenize.open(filename)
+    else:
+        return open(filename, 'r')
diff --git a/lib/tests/streamlit/scriptrunner/ScriptRunner_test.py b/lib/tests/streamlit/scriptrunner/ScriptRunner_test.py
index f47ae3805648..8f410528b1c9 100644
--- a/lib/tests/streamlit/scriptrunner/ScriptRunner_test.py
+++ b/lib/tests/streamlit/scriptrunner/ScriptRunner_test.py
@@ -20,6 +20,8 @@
 import time
 import unittest
 
+from parameterized import parameterized
+
 from streamlit.DeltaGenerator import DeltaGenerator
 from streamlit.Report import Report
 from streamlit.ReportQueue import ReportQueue
@@ -43,6 +45,18 @@ def _create_widget(id, states):
     return states.widgets[-1]
 
 
+import tokenize
+if hasattr(tokenize, 'open'):
+    text_utf = "complete! 👨‍🎤"
+    text_no_encoding = text_utf
+    text_latin = "complete! ð\x9f\x91¨â\x80\x8dð\x9f\x8e¤"
+else:
+    text_utf = u"complete! 👨‍🎤"
+    text_no_encoding = (
+        u"complete! \xf0\x9f\x91\xa8\xe2\x80\x8d\xf0\x9f\x8e\xa4")
+    text_latin = text_no_encoding
+
+
 class ScriptRunnerTest(unittest.TestCase):
     def test_startup_shutdown(self):
         """Test that we can create and shut down a ScriptRunner."""
@@ -54,9 +68,14 @@ def test_startup_shutdown(self):
         self._assert_events(scriptrunner, [ScriptRunnerEvent.SHUTDOWN])
         self._assert_text_deltas(scriptrunner, [])
 
-    def test_run_script(self):
+    @parameterized.expand([
+        ("good_script.py", text_utf),
+        ("good_script_no_encoding.py", text_no_encoding),
+        ("good_script_latin_encoding.py", text_latin),
+        ])
+    def test_run_script(self, filename, text):
         """Tests that we can run a script to completion."""
-        scriptrunner = TestScriptRunner("good_script.py")
+        scriptrunner = TestScriptRunner(filename)
         scriptrunner.enqueue_rerun()
         scriptrunner.start()
         scriptrunner.join()
@@ -70,7 +89,7 @@ def test_run_script(self):
                 ScriptRunnerEvent.SHUTDOWN,
             ],
         )
-        self._assert_text_deltas(scriptrunner, ["complete!"])
+        self._assert_text_deltas(scriptrunner, [text])
         # The following check is a requirement for the CodeHasher to
         # work correctly. The CodeHasher is scoped to
         # files contained in the directory of __main__.__file__, which we
@@ -244,7 +263,7 @@ def test_coalesce_rerun(self):
                 ScriptRunnerEvent.SHUTDOWN,
             ],
         )
-        self._assert_text_deltas(scriptrunner, ["complete!"])
+        self._assert_text_deltas(scriptrunner, [text_utf])
 
     def test_multiple_scriptrunners(self):
         """Tests that multiple scriptrunners can run simultaneously."""
@@ -352,7 +371,8 @@ def _assert_text_deltas(self, scriptrunner, text_deltas):
             [
                 delta.new_element.text.body
                 for delta in scriptrunner.deltas()
-                if delta.HasField("new_element") and delta.new_element.HasField("text")
+                if delta.HasField("new_element")
+                and delta.new_element.HasField("text")
             ],
         )
 
@@ -376,7 +396,8 @@ def enqueue_fn(msg):
         )
         self.script_request_queue = ScriptRequestQueue()
 
-        script_path = os.path.join(os.path.dirname(__file__), "test_data", script_name)
+        script_path = os.path.join(
+            os.path.dirname(__file__), "test_data", script_name)
 
         super(TestScriptRunner, self).__init__(
             report=Report(script_path, "test command line"),
diff --git a/lib/tests/streamlit/scriptrunner/test_data/good_script.py b/lib/tests/streamlit/scriptrunner/test_data/good_script.py
index e675335e279d..ad5656358f34 100644
--- a/lib/tests/streamlit/scriptrunner/test_data/good_script.py
+++ b/lib/tests/streamlit/scriptrunner/test_data/good_script.py
@@ -17,4 +17,5 @@
 
 import streamlit as st
 
-st.text("complete!")
+st.text(u"complete! 👨‍🎤")
+
diff --git a/lib/tests/streamlit/scriptrunner/test_data/good_script_latin_encoding.py b/lib/tests/streamlit/scriptrunner/test_data/good_script_latin_encoding.py
new file mode 100644
index 000000000000..9beda32f49a0
--- /dev/null
+++ b/lib/tests/streamlit/scriptrunner/test_data/good_script_latin_encoding.py
@@ -0,0 +1,20 @@
+# -*- coding: latin-1 -*-
+# Copyright 2018-2019 Streamlit Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A test script for ScriptRunnerTest that enqueues a delta."""
+
+import streamlit as st
+
+st.text(u"complete! 👨‍🎤")
diff --git a/lib/tests/streamlit/scriptrunner/test_data/good_script_no_encoding.py b/lib/tests/streamlit/scriptrunner/test_data/good_script_no_encoding.py
new file mode 100644
index 000000000000..441713460acf
--- /dev/null
+++ b/lib/tests/streamlit/scriptrunner/test_data/good_script_no_encoding.py
@@ -0,0 +1,19 @@
+# Copyright 2018-2019 Streamlit Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A test script for ScriptRunnerTest that enqueues a delta."""
+
+import streamlit as st
+
+st.text(u"complete! 👨‍🎤")

To solve #399 we need to specify the encoding as UTF8 when reading the user's script. So you'd think this is all we need to do:

```
open(the_file, encoding='utf-8')
```

But this doesn't always work because:
1) The `encoding` kwarg doesn't exist in Python 2
2) If we pass the `encoding` arg and it doesn't match whatever the user's script specifies (via [PEP263](https://www.python.org/dev/peps/pep-0263/) headers), Python raises a SyntaxError

For (2), we have to use the `tokenize` module's `open()` function, which respects PEP263 and defaults to UTF8.

But `tokenize.open()` doesn't exist in Python 2. So we need to provide a fallback in that case. The fallback is Streamlit's old behavior, where bug #399 exists.

---
In the process of writing this PR I noticed that some SyntaxErrors don't show up correctly in Streamlit — so I fixed that too.