Test tuning.

- Longer timeouts avoid problems with slow interpreters
  (especially Jython on Windows).
- Verbose unit test run to see where tests possibly hang.
- Skip hanging tests with IronPython.
- Support interpreter having spaces like `py -3.8`.
- Cleanup (TRY/EXCEPT and IF/ELSE FTW!).

Author: pekkaklarck

test/atest/resource.robot

@@ -5,7 +5,7 @@ Library              Collections
 
 *** Variables ***
 ${INTERPRETER}       python
-${SERVER TIMEOUT}    10 seconds
+${SERVER TIMEOUT}    30 seconds
 
 *** Keywords ***
 Start And Import Remote Library
@@ -18,23 +18,20 @@ Start And Import Remote Library
 
 Start Remote Library
     [Arguments]    ${library}    ${port}=0    ${args}=@{EMPTY}
-    ${library} =    Normalize Path    ${CURDIR}/../libs/${library}
-    ${port file} =    Normalize Path    ${CURDIR}/../results/server_port.txt
-    ${output} =    Normalize Path    ${CURDIR}/../results/server_output.txt
     @{interpreter} =    Split Command Line    ${INTERPRETER}
+    ${library} =        Normalize Path        ${CURDIR}/../libs/${library}
+    ${port file} =      Normalize Path        ${CURDIR}/../results/server_port.txt
+    ${output} =         Normalize Path        ${CURDIR}/../results/server_output.txt
     Start Process    @{interpreter}    ${library}    ${port}    ${port file}
     ...    @{args}    alias=${library}    stdout=${output}    stderr=STDOUT
-    ${status}    ${result} =    Run Keyword And Ignore Error
-    ...    Read Port File    ${port file}
-    Return From Keyword If    "${status}" == "PASS"    ${result}
-    ${result} =    Wait For Process    timeout=10s    on_timeout=terminate
-    Fail    Starting remote server failed:\n${result.stdout}
-
-Read Port File
-    [Arguments]    ${path}
-    Wait Until Created    ${path}    timeout=${SERVER TIMEOUT}
-    Run Keyword And Return   Get File    ${path}
-    [Teardown]    Remove File    ${path}
+    TRY
+        Wait Until Created    ${port file}    timeout=${SERVER TIMEOUT}
+    EXCEPT
+        ${result} =    Wait For Process    timeout=10s    on_timeout=terminate
+        Fail    Starting remote server failed:\n${result.stdout}
+    END
+    ${port} =    Get File    ${port file}
+    RETURN    ${port}
 
 Set Pythonpath
     ${src} =    Normalize Path    ${CURDIR}/../../src
@@ -53,6 +50,7 @@ Server Should Be Stopped And Correct Messages Logged
     ${expected} =    Catenate    SEPARATOR=\n
     ...    Robot Framework remote server at 127.0.0.1:${ACTIVE PORT} started.
     ...    Robot Framework remote server at 127.0.0.1:${ACTIVE PORT} stopped.
-    Run Keyword If    ${test logging}
-    ...    Should Be Equal    ${result.stdout}    ${expected}
+    IF    ${test logging}
+        Should Be Equal    ${result.stdout}    ${expected}
+    END
     Should Be Equal    ${result.rc}    ${0}

test/libs/KeywordDecorator.py

@@ -7,7 +7,7 @@ class KeywordDecorator(object):
     def _this_name_doesnt_matter(self, arg):
         assert arg == 'arg'
 
-    @keyword('Result of ${expression} should be ${result:\d+}')
+    @keyword(r'Result of ${expression} should be ${result:\d+}')
     def calculate(self, expression, expected):
         assert eval(expression) == int(expected)
 

test/run.py

@@ -43,19 +43,21 @@
 curdir = dirname(abspath(__file__))
 results = join(curdir, 'results')
 output = join(results, 'output.xml')
-interpreter = sys.argv[1] if len(sys.argv) > 1 else 'python'
-version = subprocess.check_output([interpreter, '-V'], encoding='UTF-8',
-                                  stderr=subprocess.STDOUT)
+interpreter = shlex.split(sys.argv[1] if len(sys.argv) > 1 else 'python')
+version = subprocess.check_output(interpreter + ['-V'], encoding='UTF-8',
+                                  stderr=subprocess.STDOUT).strip()
 py2 = version.split()[1][:3] == '2.7'
 arguments = sys.argv[2:]
 
 if exists(results):
     shutil.rmtree(results)
 os.mkdir(results)
 
+print(f'Running tests on {version}.\n')
+
 if not arguments:
-    print(f'Running unit tests with "{interpreter}".')
-    command = shlex.split(interpreter) + [join(curdir, 'utest', 'run.py')]
+    command = interpreter + [join(curdir, 'utest', 'run.py')]
+    print('Running unit tests:\n' + ' '.join(command))
     rc = subprocess.call(command)
     print()
     if rc != 0:
@@ -67,17 +69,16 @@
 excludes = []
 if os.sep == '\\':
     excludes.extend(['--exclude', 'no-windows'])
-if 'ipy' in interpreter:
+if 'ipy' in interpreter[0]:
     excludes.extend(['--exclude', 'no-ipy'])
 command = [
     'python', '-m', 'robot.run',
-    '--variable', f'INTERPRETER:{interpreter}',
+    '--variable', f'INTERPRETER:{subprocess.list2cmdline(interpreter)}',
     '--variable', f'PY2:{py2}',
-    '--doc', f"Remote server tests using '{interpreter}'.",
-    '--metadata', f'Interpreter Version:{version}',
+    '--metadata', f'Interpreter:{version}',
     '--output', output, '--log', 'NONE', '--report', 'NONE'
 ] + excludes + arguments
-print('Running acceptance tests with command:\n' + ' '.join(command))
+print('Running acceptance tests:\n' + ' '.join(command))
 rc = subprocess.call(command)
 print()
 if rc > 250:

test/utest/run.py

@@ -13,7 +13,7 @@
 test_files = [f[:-3] for f in os.listdir(curdir)
               if f.startswith('test_') and f.endswith('.py')]
 suite = unittest.defaultTestLoader.loadTestsFromNames(test_files)
-runner = unittest.TextTestRunner()
+runner = unittest.TextTestRunner(verbosity=2)
 result = runner.run(suite)
 rc = len(result.failures) + len(result.errors)
 sys.exit(rc)

test/utest/test_serve.py

@@ -17,6 +17,7 @@ def kw(self):
         return 42
 
 
+@unittest.skipIf(sys.platform == 'cli', 'Tests hang on IronPython')
 class TestServeAndStop(unittest.TestCase):
 
     def setUp(self):