add phrase search

SearchEngine/InvertedIndexSimulator/inverted/main.py

@@ -187,7 +187,7 @@ def stemming(tokens):
     return tokens
     
 
-def searching(dcmnt_xml, query):
+def proximity(dcmnt_xml, query):
 
     all_doc_no = dcmnt_xml.getElementsByTagName('DOCNO')
     all_song = dcmnt_xml.getElementsByTagName('SONG')
@@ -322,3 +322,110 @@ def detail(id):
             
     return lyrics ,judul
 
+def phrase(dcmnt_xml, query):
+
+    all_doc_no = dcmnt_xml.getElementsByTagName('DOCNO')
+    all_song = dcmnt_xml.getElementsByTagName('SONG')
+    all_lyrics = dcmnt_xml.getElementsByTagName('LYRICS')
+    
+    N_DOC = len(all_doc_no)
+    
+    all_sentence_doc = []
+    for i in range(N_DOC):
+        sentence_doc = all_song[i].firstChild.data +' '+ all_lyrics[i].firstChild.data
+        all_sentence_doc.append(sentence_doc)
+    
+    tokens_doc = []
+    for i in range(N_DOC):
+        tokens_doc.append(remove_punc_tokenize(all_sentence_doc[i]))
+    
+    for i in range(N_DOC):
+        tokens_doc[i] = to_lower(tokens_doc[i])
+    
+    for i in range(N_DOC):
+        tokens_doc[i] = stop_word_token(tokens_doc[i])
+    
+    for i in range(N_DOC):
+        tokens_doc[i] = ([w for w in tokens_doc[i] if not any(j.isdigit() for j in w)])
+    
+    for i in range(N_DOC):
+        tokens_doc[i] = stemming(tokens_doc[i])
+    
+    all_tokens =[]
+    for i in range(N_DOC):
+        for j in tokens_doc[i]:
+            all_tokens.append(j)
+            
+    new_sentences = ' '.join([w for w in all_tokens])
+
+    for j in CountVectorizer().build_tokenizer()(new_sentences):
+        all_tokens.append(j)
+    
+    all_tokens = set(all_tokens)
+    
+    ##Phrase Search##
+    bi_gram_tokens = []
+    bi_gram_sentence_doc = []
+
+    for n in range(N_DOC):
+        token = []
+        for i in range(len(tokens_doc[n])):
+            if not(i == len(tokens_doc[n])-1):            
+                token.append(tokens_doc[n][i]+'_'+tokens_doc[n][i+1])
+                bi_gram_tokens.append(tokens_doc[n][i]+'_'+tokens_doc[n][i+1])
+        bi_gram_sentence_doc.append(' '.join(token))
+    
+    bi_gram_index = {}
+    for bigram_token in bi_gram_tokens:
+        doc_no = []
+        for i in range(N_DOC):
+            if(bigram_token in bi_gram_sentence_doc[i]):
+                doc_no.append(all_doc_no[i].firstChild.data)
+        bi_gram_index[bigram_token] = doc_no
+        
+    lst_doc = bi_gram_index[query]
+    for i in range(len(lst_doc)):
+        lst_doc[i] = int(lst_doc[i])
+    
+    for i in range(len(lst_doc)):
+        lst_doc[i] = int(lst_doc[i])
+    
+        xtree = et.parse("InvertedIndexSimulator/data/dataset_STBI.xml")
+        xroot = xtree.getroot() 
+
+        df_cols = ["SONG"]
+        rows = []
+
+        for node in xroot: 
+            lirik = node.find("SONG").text if node is not None else None
+                
+            rows.append({"SONG": lirik})
+
+        df = pd.DataFrame(rows, columns = df_cols)
+        
+        judul = []
+        for i in lst_doc:
+            judul.append(df['SONG'][i-1])
+                
+        hasil = {}
+        for key in lst_doc: 
+            for value in judul: 
+                hasil[key] = value 
+                judul.remove(value) 
+                break
+                
+        numb = []
+        tit = []
+
+        for i, j in hasil.items():
+            numb.append(i)
+            tit.append(j)
+        
+        res = {} 
+        for key in numb: 
+            for value in tit: 
+                res[key] = value 
+                tit.remove(value) 
+                break
+        
+    return res
\ No newline at end of file

SearchEngine/InvertedIndexSimulator/static/assets/css/dataframe.min.css

 @import url('https://fonts.googleapis.com/css?family=Quicksand:400,700&display=swap');
  
  
-body {
-   font-family: sans-serif;
-}
- 
- 
 h2, h3 {
    color: #00a2c6
 }
@@ -55,13 +50,6 @@ footer {
    border-radius: 15px;
    padding: 20px;
    margin-top: 10px;
-   width: 100%;
-}
-
-table{
-    table-layout: fixed;
-	border: 1px solid black;
-    width: 100px;
 }
 
 .jumbotron {
@@ -101,7 +89,6 @@ main {
 }
 
 #content {
-   width: 100%;
    height: 100%;
 }
 

SearchEngine/InvertedIndexSimulator/templates/apps/dataframe.html

@@ -6,6 +6,15 @@
 		<title>Song Lyric Search Engine</title>
 		<link href="../../static/assets/css/dataframe.min.css" rel="stylesheet">
 	</head>
+	<style>
+		table {
+		  border-collapse: collapse;
+		}
+
+		table, td, th {
+		  border: 1px solid black;
+		}
+	</style>
 
 	<body>		
 		<main>
@@ -20,21 +29,15 @@
 							</div>
 						</div>
 					
-						<center><h1>Dataset</h1><br></center>
+						<center><p style="font-size:40px;"><strong>Data</strong></p>
 						<table>									
 							<tr>
-								<th>DOCNO</th>
-								<th>ARTIST</th>
-								<th>SONG</th>
 								<th>LYRICS</th>
 							</tr>
 									
-							{% for i in DOCNO %}
+							{% for i in LYRICS %}
 							<tr>
 								<td>{{ i }}</td>
-								<td>{{ j }}</td>
-								<td>{{ k }}</td>
-								<td>{{ l }}</td>
 							</tr>
 							{% endfor %}
 						</table>

SearchEngine/InvertedIndexSimulator/templates/apps/home.html

@@ -19,11 +19,10 @@
 		<main>
 			<div id="content">
 				<article class="card">
-					<center><h1>Pilih Dataset</h1><br>
+					<center><h1>Pilih Metode Searching</h1><br>
 						<table>
 							<tr>
-								<th><button onclick="pageRedirect()" class="button" style="vertical-align:middle"><span>International Billboard Song </span></button></th>
-								<td><button class="button" style="vertical-align:middle"><span>Indonesian Song </span></button></td>
+								<td><button onclick="data()"class="button" style="vertical-align:middle"><span> Mulai </span></button></td>
 							</tr>
 						</table>
 					</center>
@@ -39,9 +38,9 @@
 	</body>
 	
 	<script>
-		function pageRedirect() {
+		function data() {
 			window.location.href = "/dataframe";
-		}      
+		}  
 	</script>
 
 </html>

SearchEngine/InvertedIndexSimulator/templates/apps/index.html

@@ -16,8 +16,8 @@
 					</div>
 				</div>
 					<div class="row">
-						<center><h1 style="font-size:45px">Searching!<br></h1>
-						<p style="font-size:20px"><strong>Silahkan masukkan lirik dari lagu yang ingin Anda temukan</strong></p>
+						<center><h1 style="font-size:45px">Proximity Search<br></h1>
+						<p style="font-size:20px"><strong>Silahkan masukkan satu kata dalam lirik dari lagu yang ingin Anda temukan</strong></p>
 												
 						<form method="POST" action="/result/">
 							{% csrf_token %}

SearchEngine/InvertedIndexSimulator/templates/apps/indexing.html

@@ -6,6 +6,15 @@
 		<title>Song Lyric Search Engine</title>
 		<link href="../../static/assets/css/trying.min.css" rel="stylesheet">
 	</head>
+	<style>
+		table {
+		  border-collapse: collapse;
+		}
+
+		table, td, th {
+		  border: 1px solid black;
+		}
+	</style>
 
 	<body>		
 		<main>
@@ -21,7 +30,7 @@
 						</div>
 					
 						<center><p style="font-size:40px;"><strong>Indexing</strong></p>
-						<table width="100%"; border="1px solid black">									
+						<table>									
 							<tr>
 								<th>Token</th>
 								<th>Index</th>
@@ -47,7 +56,7 @@
 	
 	<script>
 		function pageRedirect_prev() {
-			window.location.href = "/preprocessing4";
+			window.location.href = "/search";
 		}
 			
 		function pageRedirect_next() {

SearchEngine/InvertedIndexSimulator/templates/apps/phrases.html

+<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<meta name="viewport" content="width=device-width, initial-scale=1">
+		<title>Song Lyric Search Engine</title>
+		<link href="../../static/assets/css/dataframe.min.css" rel="stylesheet">
+	</head>
+
+	<body>
+		<main>
+			<div id="content">				
+				<article class="card">	
+					<div>
+					<div>
+						<button onclick="pageRedirect_prev()" class="button" style="vertical-align:middle"><span>Previous</span></button>					
+					</div>
+				</div>
+					<div class="row">
+						<center><h1 style="font-size:45px">Phrase Search<br></h1>
+						<p style="font-size:20px"><strong>Silahkan masukkan dua kata dalam lirik dari lagu yang ingin Anda temukan (hubungkan dengan <i>underscore</i>)</strong></p>
+						<p>*Misalnya "sugar_pie" </p>					
+						<form method="POST" action="/resultphrase/">
+							{% csrf_token %}
+							<div class="form-row">
+								<input type="text" name="querysearch" placeholder="Masukkan Query Anda...">	<br>
+								<button type="submit">Cari!</button>
+							</div>
+						</form>
+					</div>
+					</center>
+				</article>
+			</div>
+		</main>
+		
+		
+
+	</body>
+	
+	<script>
+		function pageRedirect_prev() {
+			window.location.href = "/search";
+		}
+	</script>
+
+</html>

SearchEngine/InvertedIndexSimulator/templates/apps/preprocessing.html

@@ -6,6 +6,15 @@
 		<title>Song Lyric Search Engine</title>
 		<link href="../../static/assets/css/dataframe.min.css" rel="stylesheet">
 	</head>
+	<style>
+		table {
+		  border-collapse: collapse;
+		}
+
+		table, td, th {
+		  border: 1px solid black;
+		}
+	</style>
 
 	<body>		
 		<main>

SearchEngine/InvertedIndexSimulator/templates/apps/preprocessing2.html

@@ -6,6 +6,15 @@
 		<title>Song Lyric Search Engine</title>
 		<link href="../../static/assets/css/dataframe.min.css" rel="stylesheet">
 	</head>
+	<style>
+		table {
+		  border-collapse: collapse;
+		}
+
+		table, td, th {
+		  border: 1px solid black;
+		}
+	</style>
 
 	<body>		
 		<main>

SearchEngine/InvertedIndexSimulator/templates/apps/preprocessing3.html

@@ -6,6 +6,15 @@
 		<title>Song Lyric Search Engine</title>
 		<link href="../../static/assets/css/dataframe.min.css" rel="stylesheet">
 	</head>
+	<style>
+		table {
+		  border-collapse: collapse;
+		}
+
+		table, td, th {
+		  border: 1px solid black;
+		}
+	</style>
 
 	<body>		
 		<main>

SearchEngine/InvertedIndexSimulator/templates/apps/preprocessing4.html

@@ -6,6 +6,15 @@
 		<title>Song Lyric Search Engine</title>
 		<link href="../../static/assets/css/dataframe.min.css" rel="stylesheet">
 	</head>
+	<style>
+		table {
+		  border-collapse: collapse;
+		}
+
+		table, td, th {
+		  border: 1px solid black;
+		}
+	</style>
 
 	<body>		
 		<main>
@@ -48,7 +57,7 @@
 		}
 			
 		function pageRedirect_next() {
-			window.location.href = "/indexing";
+			window.location.href = "/search";
 		}
 	</script>
 

SearchEngine/InvertedIndexSimulator/templates/apps/resultphrase.html

+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+  <meta name="description" content="">
+  <meta name="author" content="">
+  
+  <title>Inverted Index</title>
+
+  <!-- Bootstrap core CSS -->
+  <link href="../../static/assets/vendor/bootstrap/css/bootstrap.min.css" rel="stylesheet">
+
+  <!-- Custom fonts for this template -->
+  <link href="../../static/assets/vendor/fontawesome-free/css/all.min.css" rel="stylesheet">
+  <link href="../../static/assets/vendor/simple-line-icons/css/simple-line-icons.css" rel="stylesheet" type="text/css">
+  <link href="https://fonts.googleapis.com/css?family=Lato:300,400,700,300italic,400italic,700italic" rel="stylesheet" type="text/css">
+
+  <!-- Custom styles for this template -->
+  <link href="../../static/assets/css/landing-page.min.css" rel="stylesheet">
+
+</head>
+
+<body>
+	<nav class="navbar navbar-light bg-light static-top">
+		<div class="container">
+			<a class="navbar-brand" href="/">Search Simulator</a>
+		</div>
+	</nav>
+  
+	<section class="testimonials text-center bg-light">
+		<div class="container">
+		<h2 class="mb-5">Lagu yang sesuai dengan query "{{ query }}"</h2>
+			<div class="row">
+				{% for key, values in res.items %}
+				<div class="col-lg-4">
+					<div class="testimonial-item mx-auto mb-5 mb-lg-0">
+						<h5><a href="/lyric/{{ key }}">Lagu No: {{ key }}</a></h5>
+						<h5>"{{ values }}"</h5>
+					</div>
+				</div>
+				{% endfor %}
+			</div>
+		</div>
+  </section>
+
+  
+  <!-- Bootstrap core JavaScript -->
+  <script src="../../static/assets/vendor/jquery/jquery.min.js"></script>
+  <script src="../../static/assets/vendor/bootstrap/js/bootstrap.bundle.min.js"></script>
+
+</body>
+
+</html>

SearchEngine/InvertedIndexSimulator/templates/apps/search.html

+<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<meta name="viewport" content="width=device-width, initial-scale=1">
+		<title>Song Lyric Search Engine</title>
+		<link href="../../static/assets/css/trying.min.css" rel="stylesheet">
+	</head>
+
+	<body>
+		<main>
+			<div id="content">				
+				<article class="card">	
+					<div>
+						<div>
+							<button onclick="pageRedirect_prev()" class="button" style="vertical-align:middle"><span>Previous</span></button>					
+						</div>
+					</div>
+					<div class="row">
+						<center><h1 style="font-size:45px">Searching!<br></h1>
+						<p style="font-size:20px"><strong>Silahkan pilih metode searching yang anda inginkan</strong></p>
+						<table>
+							<tr>
+								<th><button onclick="proximity()" class="button" style="vertical-align:middle"><span> Proximity Search </span></button></th>
+								<td><button onclick="phrase()"class="button" style="vertical-align:middle"><span>Phrase Search </span></button></td>
+							</tr>
+						</table>
+					</div>
+					</center>
+				</article>
+			</div>
+		</main>
+		
+		
+
+	</body>
+	
+	<script>
+		function pageRedirect_prev() {
+			window.location.href = "/preproseccing4";
+		}
+		
+		function proximity() {
+			window.location.href = "/indexing";
+		}
+		
+		function phrase() {
+			window.location.href = "/phrases";
+		}
+	</script>
+
+</html>

SearchEngine/InvertedIndexSimulator/urls.py

@@ -14,7 +14,10 @@ urlpatterns = [
     path('preprocessing3/', views.preprocessing3),
     path('preprocessing4/', views.preprocessing4),
     path('indexing/', views.indexing),
+    path('search/', views.search),
     path('index/', views.index),
+    path('phrase/', views.phrase),
     path('result/', views.result),
+    path('resultphrases/', views.resultphrases),
     path('lyric/<int:id>', views.lyric, name='lyric'),
 ]
\ No newline at end of file

SearchEngine/InvertedIndexSimulator/views.py

@@ -19,7 +19,6 @@ except ImportError: # not 2.6+ or is 3.x
     except ImportError:
         pass
 
-
 def home(request):
     return render(request, 'apps/home.html')
 
@@ -146,12 +145,18 @@ def indexing(request):
 def index(request):
     return render(request, 'apps/index.html')
 
+def phrases(request):
+    return render(request, 'apps/phrases.html')
+    
+def search(request):
+    return render(request, 'apps/search.html')
+
 def result(request):
     dcmnt_xml = minidom.parse("InvertedIndexSimulator/data/dataset_STBI.xml")
     
     if request.method == 'POST':
         query = request.POST['querysearch']
-        res = main.searching(dcmnt_xml, query)
+        res = main.proximity(dcmnt_xml, query)
 
         content = {
             'res':res,
@@ -160,6 +165,20 @@ def result(request):
         
     return render(request, 'apps/result.html', content)
 
+def resultphrase(request):
+    dcmnt_xml = minidom.parse("InvertedIndexSimulator/data/dataset_STBI.xml")
+    
+    if request.method == 'POST':
+        query = request.POST['querysearch']
+        res = main.phrase(dcmnt_xml, query)
+
+        content = {
+            'res':res,
+            'query':query
+        }
+        
+    return render(request, 'apps/resultphrase.html', content)
+
 def lyric(request,id):
 
     lyrics, judul = main.detail(id)

SearchEngine/SearchEngine/urls.py

@@ -26,8 +26,11 @@ urlpatterns = [
     path('preprocessing3/', views.preprocessing3),
     path('preprocessing4/', views.preprocessing4),
     path('indexing/', views.indexing),
+    path('search/', views.search),
     path('index/', views.index),
+    path('phrases/', views.phrases),
     path('result/', views.result),
+    path('resultphrase/', views.resultphrase),
     path('lyric/<int:id>', views.lyric, name='lyric'),
     
 ]