Skip to content

S
SearchEngine
Commit 028fb649 by Rosa Delima Mendrofa
Options

Query Searching

parent d2786b64
Showing with 0 additions and 654 deletions
SearchEngine-master/SearchEngine/HandlerAPI-master/.gitattributes deleted 100644 → 0
# Auto detect text files and perform LF normalization
* text=auto
SearchEngine-master/SearchEngine/HandlerAPI-master/.gitignore deleted 100644 → 0
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
SearchEngine-master/SearchEngine/HandlerAPI-master/HandlerAPI.py deleted 100644 → 0
#!/usr/bin/env python3
import requests
import json
import pandas as pd
from operator import add
def parsed_api(url):
response = requests.get(url)
data = response.text
parsed = json.loads(data)
return parsed
def count_state(api_data):
# arr = [] #array
data = {} #dictionary{key:value}
count = 0
user_state = {}
for i in range (len(api_data)):
temp = api_data[i]['user']['login'] #temp untuk menyimpan nama developer
count+=1
if temp not in data: #jika temp tidak ada dalam dictionary, dia belum jadi key
data[temp] = set() #set temp sebagai key dalam dictionary
data[temp].add(count) #tambahkan value untuk key yang saat ini
for user, value in data.items(): #dictionary di looping sebanyak panjang dictionary
user_state[user] = set() #set user sebagai key dalam dictionary user_state
user_state[user] = len(value) #tambahkan value dalam key user_state sepanjang nilai value
return user_state
def count_state_commit(api_data):
data = {} #dictionary{key:value}
count = 0
user_state = {}
for i in range (len(api_data)):
temp = api_data[i]['author']['login'] #temp untuk menyimpan nama developer
count+=1
if temp not in data: #jika temp tidak ada dalam data dictionary, dia belum jadi key
data[temp] = set() #set temp sebagai key dalam dictionary
data[temp].add(count) #tambahkan value untuk key yang saat ini
for user , value in data.items(): #dictionary di looping sebanyak panjang dictionary
user_state[user] = set() #set user sebagai key dalam dictionary user_state
user_state[user] = len(value) #tambahkan value dalam key user_state sepanjang nilai value
return user_state
def count_loc(api_commit):
locArr = []
for i in range(len(api_commit)):
a = api_commit[i]['weeks']
arrA = []
for j in range(len(api_commit[i]['weeks'])):
tempDict = api_commit[i]['weeks'][j]
arr = []
for key, value in tempDict.items():
arr.append(value)
arrA.append(arr)
locArr.append(arrA)
ArrAdd = []
ArrDel = []
for i in range(len(locArr)):
tempArr= []
tempArr2=[]
for j in range(len(locArr[i])):
tempArr.append(locArr[i][j][1])
tempArr2.append(locArr[i][j][2])
#print(j)
ArrAdd.append(tempArr)
ArrDel.append(tempArr2)
result_list = []
for i in range(len(ArrAdd)):
result_list.append(list(map(add, ArrAdd[i], ArrDel[i])))
loc_tot = [ ]
for i in range(len(result_list)):
loc_tot.append(sum(result_list[i]))
LOC_dict = {}
for i in range(len(loc_tot)):
developer = api_commit[i]['author']['login']
LOC_dict[developer] = loc_tot[i]
return LOC_dict
SearchEngine-master/SearchEngine/HandlerAPI-master/LICENSE deleted 100644 → 0
MIT License
Copyright (c) 2019 Eva
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
SearchEngine-master/SearchEngine/HandlerAPI-master/__init__.py deleted 100644 → 0
#!/usr/bin/env python3
from HandlerAPI.HandlerAPI import parsed_api
from HandlerAPI.HandlerAPI import count_state
from HandlerAPI.HandlerAPI import count_state_commit
from HandlerAPI.HandlerAPI import count_loc
__all__ = [
'parsed_api',
'count_state',
'count_state_commit',
'count_loc'
]
SearchEngine/InvertedIndexSimulator/templates/apps/home.html deleted 100644 → 0
<!DOCTYPE html>
<html lang="en">
<head>
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Song Lyric Search Engine</title>
<link href="../../static/assets/css/landing-page.min.css" rel="stylesheet">
</head>
<body>
<header>
<div class="jumbotron">
<h1>Song Lyric Search Engine<br>- Simulator -</h1>
<p>Search engine yang pake inverted index untuk indexing nya</p>
</div>
</header>
<main>
<div id="content">
<article class="card">
<center><h1>Pilih Dataset</h1><br>
<table>
<tr>
<th><button onclick="pageRedirect()" class="button" style="vertical-align:middle"><span>International Billboard Song </span></button></th>
<td><button class="button" style="vertical-align:middle"><span>Indonesian Song </span></button></td>
</tr>
</table>
</center>
</article>
</div>
</main>
<footer>
<p>&copy; STBI-2020-03</p>
</footer>
</body>
<script>
function pageRedirect() {
window.location.href = "/dataframe";
}
</script>
</html>
SearchEngine/InvertedIndexSimulator/templates/apps/index.html deleted 100644 → 0
<!DOCTYPE html>
<html lang="en">
<head>
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Song Lyric Search Engine</title>
<link href="../../static/assets/css/dataframe.min.css" rel="stylesheet">
</head>
<body>
<main>
<div id="content">
<article class="card">
<div>
<div>
<button onclick="pageRedirect_prev()" class="button" style="vertical-align:middle"><span>Previous</span></button>
</div>
</div>
<div class="row">
<center><h1 style="font-size:45px">Searching!<br></h1>
<p style="font-size:20px"><strong>Silahkan masukkan lirik dari lagu yang ingin Anda temukan</strong></p>
<form method="POST" action="/result/">
{% csrf_token %}
<div class="form-row">
<input type="text" name="querysearch" placeholder="Masukkan Query Anda..."> <br>
<button type="submit">Cari!</button>
</div>
</form>
</div>
</center>
</article>
</div>
</main>
</body>
<script>
function pageRedirect_prev() {
window.location.href = "/indexing";
}
</script>
</html>
SearchEngine/InvertedIndexSimulator/views.py deleted 100644 → 0
from django.shortcuts import render
from django.http import HttpResponse
from InvertedIndexSimulator.inverted import main
import pandas as pd
import xml.etree.ElementTree as et
def home(request):
return render(request, 'apps/home.html')
def dataframe(request):
parse_data = et.parse("InvertedIndexSimulator/data/dataset_STBI.xml")
data = parse_data.getroot()
df_cols = ["DOCNO", "SONG", "ARTIST", "LYRICS"]
rows = []
for node in data:
s_docno = node.find("DOCNO").text if node is not None else None
s_song = node.find("SONG").text if node is not None else None
s_artist = node.find("ARTIST").text if node is not None else None
s_lyrics = node.find("LYRICS").text if node is not None else None
rows.append({"DOCNO": s_docno, "SONG": s_song, "ARTIST": s_artist, "LYRICS": s_lyrics})
DataFrame = pd.DataFrame(rows, columns = df_cols)
dictionary = DataFrame.set_index('DOCNO').T.to_dict('list')
nilai = list(dictionary.values())
nomornya = list(dictionary.keys())
lagunya = [sublist[0] for sublist in nilai]
artisnya = [sublist[1] for sublist in nilai]
liriknya = [sublist[2] for sublist in nilai]
context = {"DOCNO": nomornya, "SONG": lagunya, "ARTIST": artisnya, "LYRICS": liriknya}
return render(request, 'apps/dataframe.html', context)
def preprocessing(request):
from xml.etree.ElementTree import ElementTree
tree = ElementTree()
tree.parse("InvertedIndexSimulator/data/dataset_STBI.xml")
all_doc_no = []
all_song = []
all_text = []
for node in tree.iter("DOCNO"):
all_doc_no.append(node.text)
for node in tree.iter("SONG"):
all_song.append(node.text)
for node in tree.iter("LYRICS"):
all_text.append(node.text)
N_DOC = len(all_text)
all_sentence_doc = []
for i in range(N_DOC):
all_sentence_doc.append(all_song[i] + all_text[i])
tokens_doc = []
for i in range(N_DOC):
tokens_doc.append(main.remove_punc_tokenize(all_sentence_doc[i]))
context = {"tokens_doc": tokens_doc}
return render(request, 'apps/preprocessing.html', context)
def preprocessing2(request):
from xml.etree.ElementTree import ElementTree
tree = ElementTree()
tree.parse("InvertedIndexSimulator/data/dataset_STBI.xml")
all_doc_no = []
all_song = []
all_text = []
for node in tree.iter("DOCNO"):
all_doc_no.append(node.text)
for node in tree.iter("SONG"):
all_song.append(node.text)
for node in tree.iter("LYRICS"):
all_text.append(node.text)
N_DOC = len(all_text)
all_sentence_doc = []
for i in range(N_DOC):
all_sentence_doc.append(all_song[i] + all_text[i])
tokens_doc = []
for i in range(N_DOC):
tokens_doc.append(main.remove_punc_tokenize(all_sentence_doc[i]))
for i in range(N_DOC):
tokens_doc[i] = main.to_lower(tokens_doc[i])
context = {"tokens_doc": tokens_doc}
return render(request, 'apps/preprocessing2.html', context)
def preprocessing3(request):
from xml.etree.ElementTree import ElementTree
tree = ElementTree()
tree.parse("InvertedIndexSimulator/data/dataset_STBI.xml")
all_doc_no = []
all_song = []
all_text = []
for node in tree.iter("DOCNO"):
all_doc_no.append(node.text)
for node in tree.iter("SONG"):
all_song.append(node.text)
for node in tree.iter("LYRICS"):
all_text.append(node.text)
N_DOC = len(all_text)
all_sentence_doc = []
for i in range(N_DOC):
all_sentence_doc.append(all_song[i] + all_text[i])
tokens_doc = []
for i in range(N_DOC):
tokens_doc.append(main.remove_punc_tokenize(all_sentence_doc[i]))
for i in range(N_DOC):
tokens_doc[i] = main.to_lower(tokens_doc[i])
for i in range(N_DOC):
tokens_doc[i] = main.stop_word_token(tokens_doc[i])
for i in range(N_DOC):
tokens_doc[i] = ([w for w in tokens_doc[i] if not any(j.isdigit() for j in w)])
context = {"tokens_doc": tokens_doc}
return render(request, 'apps/preprocessing3.html', context)
def preprocessing4(request):
from xml.etree.ElementTree import ElementTree
tree = ElementTree()
tree.parse("InvertedIndexSimulator/data/dataset_STBI.xml")
all_doc_no = []
all_song = []
all_text = []
for node in tree.iter("DOCNO"):
all_doc_no.append(node.text)
for node in tree.iter("SONG"):
all_song.append(node.text)
for node in tree.iter("LYRICS"):
all_text.append(node.text)
N_DOC = len(all_text)
all_sentence_doc = []
for i in range(N_DOC):
all_sentence_doc.append(all_song[i] + all_text[i])
tokens_doc = []
for i in range(N_DOC):
tokens_doc.append(main.remove_punc_tokenize(all_sentence_doc[i]))
for i in range(N_DOC):
tokens_doc[i] = main.to_lower(tokens_doc[i])
for i in range(N_DOC):
tokens_doc[i] = main.stop_word_token(tokens_doc[i])
for i in range(N_DOC):
tokens_doc[i] = ([w for w in tokens_doc[i] if not any(j.isdigit() for j in w)])
for i in range(N_DOC):
tokens_doc[i] = main.stemming(tokens_doc[i])
context = {"tokens_doc": tokens_doc}
return render(request, 'apps/preprocessing4.html', context)
def indexing(request):
from sklearn.feature_extraction.text import CountVectorizer
from xml.etree.ElementTree import ElementTree
tree = ElementTree()
tree.parse("InvertedIndexSimulator/data/dataset_STBI.xml")
all_doc_no = []
all_song = []
all_text = []
for node in tree.iter("DOCNO"):
all_doc_no.append(node.text)
for node in tree.iter("SONG"):
all_song.append(node.text)
for node in tree.iter("LYRICS"):
all_text.append(node.text)
N_DOC = len(all_text)
all_sentence_doc = []
for i in range(N_DOC):
all_sentence_doc.append(all_song[i] + all_text[i])
tokens_doc = []
for i in range(N_DOC):
tokens_doc.append(main.remove_punc_tokenize(all_sentence_doc[i]))
for i in range(N_DOC):
tokens_doc[i] = main.to_lower(tokens_doc[i])
for i in range(N_DOC):
tokens_doc[i] = main.stop_word_token(tokens_doc[i])
for i in range(N_DOC):
tokens_doc[i] = ([w for w in tokens_doc[i] if not any(j.isdigit() for j in w)])
for i in range(N_DOC):
tokens_doc[i] = main.stemming(tokens_doc[i])
all_tokens =[]
for i in range(N_DOC):
for j in tokens_doc[i]:
all_tokens.append(j)
new_sentences = ' '.join([w for w in all_tokens])
for j in CountVectorizer().build_tokenizer()(new_sentences):
all_tokens.append(j)
all_tokens = set(all_tokens)
from itertools import count
try:
from future_builtins import zip
except ImportError: # not 2.6+ or is 3.x
try:
from itertools import izip as zip # < 2.5 or 3.x
except ImportError:
pass
proximity_index = {}
for token in all_tokens:
dict_doc_position = {}
for n in range(N_DOC):
if(token in tokens_doc[n]):
dict_doc_position[all_doc_no[n].firstChild.data] = [i+1 for i, j in zip(count(), tokens_doc[n]) if j == token]
proximity_index[token] = dict_doc_position
import collections
proximity_index = collections.OrderedDict(sorted(proximity_index.items()))
for key, value in proximity_index.items():
indexnya = (key, value)
context = {"indexnya": indexnya}
return render(request, 'apps/indexing.html', context)
def index(request):
return render(request, 'apps/index.html')
def lyric(request,id):
text, judul = main.detail(id)
content={
'no': id,
'judul':judul,
'text':text
}
return render(request, 'apps/lyric.html', content)
def result(request):
#%%
# proximity_index = collections.OrderedDict(sorted(proximity_index.items()))
# for key, value in proximity_index.items():
# # print (key, value)
if request.method == 'POST':
query = request.POST['querysearch']
hasil= main.main(query)
content={
'hasil':hasil,
'query':query
}
return render(request, 'apps/result.html', content)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment