Replace malformed nvidia-smi XML

Author: jfennick

cwltool/cuda.py

@@ -15,7 +15,14 @@ def cuda_version_and_device_count() -> Tuple[str, int]:
     except Exception as e:
         _logger.warning("Error checking CUDA version with nvidia-smi: %s", e)
         return ("", 0)
-    dm = xml.dom.minidom.parseString(out)  # nosec
+
+    # Apparently nvidia-smi is not safe to call concurrently.
+    # With --parallel, sometimes the returned XML will contain
+    # <process_name>\xff...\xff</process_name>
+    # and xml.dom.minidom.parseString will raise
+    # "xml.parsers.expat.ExpatError: not well-formed (invalid token)"
+    out_no_xff = out.replace(b'\xff', b'')
+    dm = xml.dom.minidom.parseString(out_no_xff)  # nosec
 
     ag = dm.getElementsByTagName("attached_gpus")
     if len(ag) < 1 or ag[0].firstChild is None: