diff --git a/onnxscript/backend/onnx_export.py b/onnxscript/backend/onnx_export.py
index 94e673be69..1c26313c0f 100644
--- a/onnxscript/backend/onnx_export.py
+++ b/onnxscript/backend/onnx_export.py
@@ -152,7 +152,10 @@ def _translate_value_info(value_info: ValueInfoProto) -> str:
 
 def _to_str(s):
     if isinstance(s, bytes):
-        return s.decode("utf-8")
+        try:
+            return s.decode("utf-8")
+        except UnicodeDecodeError:
+            pass
     return s
 
 
@@ -391,8 +394,8 @@ def _translate_attributes(self, node):
                 attributes.append((at.name, at.ref_attr_name))
                 continue
             value = _attribute_value(at)
-            if isinstance(value, str):
-                attributes.append((at.name, f"{value!r}"))
+            if isinstance(value, (str, bytes)):
+                attributes.append((at.name, repr(value)))
                 continue
             if isinstance(value, np.ndarray):
                 onnx_dtype = at.t.data_type