From: Yaroslav Halchenko <debian@onerussian.com>
Subject: Disable unicode strings in commands to be executed in tests

As you could see largely it is about executing a command with unicode, or later
logging it.  Whenvever Python2 seems to do it automagical conversions without blowing up,
on Python3 I found no reliable way to achieve desired -- logger would not accept bytes,
but would puke upon attempt to encode unicode into 'ascii', etc

Problems go away if UTF-8 locale is configured and set (instead of C or POSIX)

Last-Update: 2018-06-05

--- a/datalad/tests/test_config.py
+++ b/datalad/tests/test_config.py
@@ -64,7 +64,7 @@ novalue
 empty =
 myint = 3
 
-[onemore "complicated の beast with.dot"]
+[onemore "complicated nounicode beast with.dot"]
 findme = 5.0
 """
 
@@ -139,16 +139,16 @@ def test_something(path=None, new_home=N
     assert_true(cfg.has_section('something'))
     assert_false(cfg.has_section('somethingelse'))
     assert_equal(sorted(cfg.sections()),
-                 [u'onemore.complicated の beast with.dot', 'something'])
+                 [u'onemore.complicated nounicode beast with.dot', 'something'])
     assert_true(cfg.has_option('something', 'user'))
     assert_false(cfg.has_option('something', 'us?er'))
     assert_false(cfg.has_option('some?thing', 'user'))
     assert_equal(sorted(cfg.options('something')), ['empty', 'myint', 'novalue', 'user'])
-    assert_equal(cfg.options(u'onemore.complicated の beast with.dot'), ['findme'])
+    assert_equal(cfg.options(u'onemore.complicated nounicode beast with.dot'), ['findme'])
 
     assert_equal(
         sorted(cfg.items()),
-        [(u'onemore.complicated の beast with.dot.findme', '5.0'),
+        [(u'onemore.complicated nounicode beast with.dot.findme', '5.0'),
          ('something.empty', ''),
          ('something.myint', '3'),
          ('something.novalue', None),
@@ -168,7 +168,7 @@ def test_something(path=None, new_home=N
         cfg.get('something.user', get_all=True),
         ('name=Jane Doe', 'email=jd@example.com'))
     assert_raises(KeyError, cfg.__getitem__, 'somedthing.user')
-    assert_equal(cfg.getfloat(u'onemore.complicated の beast with.dot', 'findme'), 5.0)
+    assert_equal(cfg.getfloat(u'onemore.complicated nounicode beast with.dot', 'findme'), 5.0)
     assert_equal(cfg.getint('something', 'myint'), 3)
     assert_equal(cfg.getbool('something', 'myint'), True)
     # git demands a key without value at all to be used as a flag, thus True
@@ -188,8 +188,8 @@ def test_something(path=None, new_home=N
 
     # modification follows
     if FILESYSTEM_SUPPORTS_UTF8:
-        cfg.add('something.new', 'の')
-        assert_equal(cfg.get('something.new'), u'の')
+        cfg.add('something.new', 'nonunicode')
+        assert_equal(cfg.get('something.new'), u'nonunicode')
     # sections are added on demand
     cfg.add('unheard.of', 'fame')
     assert_true(cfg.has_section('unheard.of'))
--- a/datalad/tests/utils_pytest.py
+++ b/datalad/tests/utils_pytest.py
@@ -1490,7 +1490,7 @@ with_parametric_batch = pytest.mark.para
 OBSCURE_PREFIX = os.getenv('DATALAD_TESTS_OBSCURE_PREFIX', '')
 # Those will be tried to be added to the base name if filesystem allows
 OBSCURE_FILENAME_PARTS = [' ', '/', '|', ';', '&', '%b5', '{}', "'", '"', '<', '>']
-UNICODE_FILENAME = u"ΔЙקم๗あ"
+UNICODE_FILENAME = u""
 
 # OSX is exciting -- some I guess FS might be encoding differently from decoding
 # so Й might get recoded
--- a/datalad/local/tests/test_configuration.py
+++ b/datalad/local/tests/test_configuration.py
@@ -29,7 +29,7 @@ from datalad.tests.utils_pytest import (
 
 # before 0.14 ui.message() (used in dump) was not friendly to unicode
 complicated_str = 'complicated {} beast with.dot'.format(
-    "の" if Version(datalad.__version__) >= Version('0.14.0') else 'blob'
+    "nonunicode" if Version(datalad.__version__) >= Version('0.14.0') else 'blob'
 )
 
 _config_file_content = """\
--- a/datalad/core/local/tests/test_diff.py
+++ b/datalad/core/local/tests/test_diff.py
@@ -501,7 +501,7 @@ def test_diff_nonexistent_ref_unicode(pa
         pytest.skip("requires UTF-8 filesystem encoding for unicode ref args")
     ds = Dataset(path).create()
     assert_result_count(
-        ds.diff(fr="HEAD", to=u"β", on_failure="ignore", result_renderer='disabled'),
+        ds.diff(fr="HEAD", to=u"B", on_failure="ignore", result_renderer='disabled'),
         1,
         path=ds.path,
         status="impossible")
--- a/datalad/core/local/tests/test_run.py
+++ b/datalad/core/local/tests/test_run.py
@@ -193,37 +193,37 @@ def test_py2_unicode_command(path=None):
     # CrippledFS variants like vfat advertise UTF-8 (because the OS-level
     # locale is UTF-8) yet still reject non-ASCII or trailing-space names.
     # Probe the actual tempdir before proceeding.
-    if not fs_supports_filename(path, u"bβ0.dat"):
+    if not fs_supports_filename(path, u"bB0.dat"):
         pytest.skip("filesystem under %s rejects non-ASCII names" % path)
     # Avoid OBSCURE_FILENAME to avoid windows-breakage (gh-2929).
     ds = Dataset(path).create()
     touch_cmd = "import sys; open(sys.argv[1], 'w').write('')"
     cmd_str = u"{} -c \"{}\" {}".format(sys.executable,
                                         touch_cmd,
-                                        u"bβ0.dat")
+                                        u"bB0.dat")
     ds.run(cmd_str)
     assert_repo_status(ds.path)
-    ok_exists(op.join(path, u"bβ0.dat"))
+    ok_exists(op.join(path, u"bB0.dat"))
 
-    # Some CrippledFS mounts (notably vfat) accept "bβ0.dat" but still
-    # reject leading/trailing-space names like " β1 ". Probe for the
+    # Some CrippledFS mounts (notably vfat) accept "bB0.dat" but still
+    # reject leading/trailing-space names like " B1 ". Probe for the
     # worst-case name actually used below to decide whether to run this
     # block. Subsumes the previous hardcoded TMPDIR == '/crippledfs'
     # workaround.
-    if not on_windows and fs_supports_filename(path, u" β1 "):
-        ds.run([sys.executable, "-c", touch_cmd, u"bβ1.dat"])
+    if not on_windows and fs_supports_filename(path, u" B1 "):
+        ds.run([sys.executable, "-c", touch_cmd, u"bB1.dat"])
         assert_repo_status(ds.path)
-        ok_exists(op.join(path, u"bβ1.dat"))
+        ok_exists(op.join(path, u"bB1.dat"))
 
         # Send in a list of byte-strings to mimic a py2 command-line
         # invocation.
         ds.run([s.encode("utf-8")
-                for s in [sys.executable, "-c", touch_cmd, u" β1 "]])
+                for s in [sys.executable, "-c", touch_cmd, u" B1 "]])
         assert_repo_status(ds.path)
-        ok_exists(op.join(path, u" β1 "))
+        ok_exists(op.join(path, u" B1 "))
 
     assert_in_results(
-        ds.run(u"bβ2.dat", result_renderer=None, on_failure="ignore"),
+        ds.run(u"bB2.dat", result_renderer=None, on_failure="ignore"),
         status="error", action="run")
 
 
--- a/datalad/tests/test_utils.py
+++ b/datalad/tests/test_utils.py
@@ -832,7 +832,7 @@ def test_assure_unicode():
 
 def test_pathlib_unicode():
     eq_(str(Path("a")), u"a")
-    eq_(str(Path(u"β")), u"β")
+    eq_(str(Path(u"B")), u"B")
 
 
 @with_tempfile(mkdir=True)
--- a/datalad/runner/tests/test_witless_runner.py
+++ b/datalad/runner/tests/test_witless_runner.py
@@ -67,7 +67,7 @@ result_counter = 0
 @with_tempfile
 def test_runner(tempfile: str = "") -> None:
     runner = Runner()
-    content = 'Testing real run' if on_windows or not FILESYSTEM_SUPPORTS_UTF8 else 'Testing äöü東 real run'
+    content = 'Testing real run'
     cmd = 'echo %s > %s' % (content, tempfile)
     res = runner.run(cmd)
     assert isinstance(res, dict)
