Upload New File

e3c524e1 · Sartika Aritonang · c727b534 · e3c524e1
Commit e3c524e1 authored May 29, 2020 by Sartika Aritonang
Hide whitespace changes
Inline Side-by-side

Showing with 11 additions and 0 deletions

TextNormalizer.py ...b/site-packages/Sastrawi/Stemmer/Filter/TextNormalizer.py +11 -0

No files found.
--- a/stbi/Lib/site-packages/Sastrawi/Stemmer/Filter/TextNormalizer.py
+++ b/stbi/Lib/site-packages/Sastrawi/Stemmer/Filter/TextNormalizer.py
+import re
+
+def normalize_text(text):
+    result = str.lower(text)
+    result = re.sub(r'[^a-z0-9 -]', ' ', result, flags = re.IGNORECASE|re.MULTILINE)
+    result = re.sub(r'( +)', ' ', result, flags = re.IGNORECASE|re.MULTILINE)
+
+    return result.strip()
+
+
+