Merge pull request #159 from wiseaidev/fix-uni-dec-enc

handle UnicodeDecode exception
rottingresearch · Oct 2, 2022 · 1926d5c · 1926d5c
2 parents 1616f0f + 7db232d
commit 1926d5c
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/linkrot/backends.py b/linkrot/backends.py
@@ -51,7 +51,10 @@ def make_compat_str(in_str):
     enc = chardet.detect(in_str)
 
     # Decode the object into a unicode object
-    out_str = in_str.decode(enc["encoding"])
+    try:
+        out_str = in_str.decode(enc["encoding"])
+    except UnicodeDecodeError as err:
+        out_str = ""
 
     # Cleanup
     if enc["encoding"] == "UTF-16BE":