parent
bef0a64b41
commit
940ddca4dc
@ -0,0 +1,271 @@
|
||||
import time
|
||||
import db.mysql_connection as mysql
|
||||
import db.constants as const
|
||||
|
||||
|
||||
def add_book_to_db(book_code, book_dict):
|
||||
# print('Adding Book Code :: ', book_code, ' Dict :: ', book_dict)
|
||||
|
||||
conn = mysql.get_new_mysql_connection('db_config.ini')
|
||||
if conn is None:
|
||||
return False
|
||||
|
||||
db_cursor = conn.cursor(buffered=True)
|
||||
|
||||
# add book data to the Table First
|
||||
book_row = {
|
||||
'code': book_code,
|
||||
'added_at': int(time.time())
|
||||
}
|
||||
|
||||
# returns the last row id, if row added to the table successfully
|
||||
last_rowid = add_book_row_to_table(db_cursor, const.BOOK_INSERT_QUERY, book_row)
|
||||
book_id = last_rowid
|
||||
print('Book Row Id :: ', last_rowid)
|
||||
|
||||
book_info_dict = book_dict['bookInfo']
|
||||
if last_rowid > 0:
|
||||
book_info_row = {
|
||||
'title': book_info_dict['title'],
|
||||
'description': book_info_dict['description'] if 'description' in book_info_dict else None,
|
||||
'lang': book_info_dict['lang'],
|
||||
'source': book_info_dict['source'],
|
||||
'is_translation': 'true' == book_info_dict['isTranslation'].lower(),
|
||||
'total_chapters': book_info_dict['totalChapters'],
|
||||
'isbn': book_info_dict['isbn'] if 'isbn' in book_info_dict else None,
|
||||
'book': book_id
|
||||
}
|
||||
|
||||
# returns the last row id, if row added to the table successfully
|
||||
last_rowid = add_book_info_row_to_table(db_cursor, const.BOOK_INFO_INSERT_QUERY, book_info_row)
|
||||
print('Book Info Row Id :: ', last_rowid)
|
||||
|
||||
if last_rowid > 0:
|
||||
book_info_id = last_rowid
|
||||
authors_list = book_info_dict['authors']
|
||||
for author in authors_list:
|
||||
author_row = {
|
||||
'id': -1,
|
||||
'name': author['name'].strip().lower(),
|
||||
'total_books': 1
|
||||
}
|
||||
author_row = search_author(db_cursor, const.AUTHOR_SEARCH_QUERY, author_row)
|
||||
print('Author Search Result :: ', author_row)
|
||||
if author_row['id'] > 0:
|
||||
author_row['total_books'] = author_row['total_books'] + 1
|
||||
last_rowid = update_author_book_count(db_cursor, const.AUTHOR_UPDATE_QUERY, author_row)
|
||||
print('Author Update Row count :: ', last_rowid)
|
||||
if last_rowid <= 0:
|
||||
break
|
||||
else:
|
||||
author_row['name'] = author['name']
|
||||
author_row['total_books'] = 1
|
||||
last_rowid = add_author_to_table(db_cursor, const.AUTHOR_INSERT_QUERY, author_row)
|
||||
print('Add Author Row Id :: ', last_rowid)
|
||||
if last_rowid > 0:
|
||||
author_row['id'] = last_rowid
|
||||
|
||||
if author_row['id'] > 0:
|
||||
author_is_translator = False
|
||||
if 'translator' in author:
|
||||
author_is_translator = 'true' == author['translator'].lower()
|
||||
map_author_book = {
|
||||
'author': author_row['id'],
|
||||
'book': book_info_id,
|
||||
'translator': author_is_translator
|
||||
}
|
||||
|
||||
last_rowid = add_author_book_mapping(db_cursor, const.BOOK_AUTHOR_INSERT_QUERY, map_author_book)
|
||||
print('Author Book Mapping Row ID :: ', last_rowid)
|
||||
if last_rowid < 0:
|
||||
break
|
||||
|
||||
if last_rowid > 0:
|
||||
book_content_row = {
|
||||
'book': book_id
|
||||
}
|
||||
|
||||
# returns the last row id, if row added to the table successfully
|
||||
last_rowid = add_book_content_row_to_table(db_cursor, const.CONTENT_INSERT_QUERY, book_content_row)
|
||||
print('Book Content Row Id :: ', last_rowid)
|
||||
|
||||
if last_rowid > 0:
|
||||
content_id = last_rowid
|
||||
book_chapters_list = book_dict['content']['chapters']
|
||||
for chapter in book_chapters_list:
|
||||
book_chapter_row = {
|
||||
'c_num': chapter['num'],
|
||||
'name': chapter['name'] if 'name' in chapter else None,
|
||||
'book_content': content_id
|
||||
}
|
||||
chapter_id = add_book_chapter_to_table(db_cursor, const.CHAPTER_INSERT_QUERY, book_chapter_row)
|
||||
print('Book Chapter Row Id :: ', chapter_id)
|
||||
if chapter_id > 0:
|
||||
sentences_dict = chapter['sentences']
|
||||
for s_num in sentences_dict.keys():
|
||||
sentence_row = {
|
||||
's_num': s_num,
|
||||
'text': sentences_dict[s_num],
|
||||
'chapter': chapter_id
|
||||
}
|
||||
sen_id = add_book_sentence_to_table(db_cursor, const.SENTENCE_INSERT_QUERY, sentence_row)
|
||||
print('Book Sentence Id :: ', sen_id)
|
||||
if sen_id <= 0:
|
||||
break
|
||||
else:
|
||||
last_rowid = sen_id
|
||||
else:
|
||||
break
|
||||
|
||||
db_cursor.close()
|
||||
|
||||
is_success = False
|
||||
if last_rowid > 0:
|
||||
conn.commit()
|
||||
is_success = True
|
||||
else:
|
||||
conn.rollback()
|
||||
is_success = False
|
||||
|
||||
conn.close()
|
||||
|
||||
return is_success
|
||||
|
||||
|
||||
def add_book_row_to_table(db_cursor, book_insert_query, book_row):
|
||||
try:
|
||||
# Insert this Book row to Table
|
||||
db_cursor.execute(book_insert_query, book_row)
|
||||
book_id = db_cursor.lastrowid
|
||||
if book_id is not None:
|
||||
return book_id
|
||||
else:
|
||||
return -1
|
||||
|
||||
except Exception as e:
|
||||
print(str(e))
|
||||
return -1
|
||||
|
||||
|
||||
def add_book_info_row_to_table(db_cursor, book_info_insert_query, book_info_row):
|
||||
try:
|
||||
# Insert this BookInfo row
|
||||
db_cursor.execute(book_info_insert_query, book_info_row)
|
||||
book_info_id = db_cursor.lastrowid
|
||||
if book_info_id is not None:
|
||||
return book_info_id
|
||||
else:
|
||||
return -1
|
||||
|
||||
except Exception as e:
|
||||
print(str(e))
|
||||
return -1
|
||||
|
||||
|
||||
def add_book_content_row_to_table(db_cursor, book_content_insert_query, book_content_row):
|
||||
try:
|
||||
# Insert Book Content row
|
||||
db_cursor.execute(book_content_insert_query, book_content_row)
|
||||
book_content_id = db_cursor.lastrowid
|
||||
if book_content_id is not None:
|
||||
return book_content_id
|
||||
else:
|
||||
return -1
|
||||
|
||||
except Exception as e:
|
||||
print(str(e))
|
||||
return -1
|
||||
|
||||
|
||||
def add_book_chapter_to_table(db_cursor, book_chapter_insert_query, book_chapter_row):
|
||||
try:
|
||||
# Insert Book chapter row
|
||||
db_cursor.execute(book_chapter_insert_query, book_chapter_row)
|
||||
book_chapter_id = db_cursor.lastrowid
|
||||
if book_chapter_id is not None:
|
||||
return book_chapter_id
|
||||
else:
|
||||
return -1
|
||||
|
||||
except Exception as e:
|
||||
print(str(e))
|
||||
return -1
|
||||
|
||||
|
||||
def add_book_sentence_to_table(db_cursor, book_sentence_insert_query, book_sentence):
|
||||
try:
|
||||
# Insert sentence
|
||||
db_cursor.execute(book_sentence_insert_query, book_sentence)
|
||||
book_sen_id = db_cursor.lastrowid
|
||||
if book_sen_id is not None:
|
||||
return book_sen_id
|
||||
else:
|
||||
return -1
|
||||
|
||||
except Exception as e:
|
||||
print(str(e))
|
||||
return -1
|
||||
|
||||
|
||||
def add_author_to_table(db_cursor, author_insert_query, author_data):
|
||||
try:
|
||||
# Insert Author
|
||||
db_cursor.execute(author_insert_query, author_data)
|
||||
author_id = db_cursor.lastrowid
|
||||
if author_id is not None:
|
||||
return author_id
|
||||
else:
|
||||
return -1
|
||||
|
||||
except Exception as e:
|
||||
print(str(e))
|
||||
return -1
|
||||
|
||||
|
||||
def add_author_book_mapping(db_cursor, book_author_insert_query, book_author_data):
|
||||
try:
|
||||
# Insert Book Author Mapping
|
||||
db_cursor.execute(book_author_insert_query, book_author_data)
|
||||
map_id = db_cursor.rowcount
|
||||
if map_id > 0:
|
||||
return map_id
|
||||
else:
|
||||
return -1
|
||||
|
||||
except Exception as e:
|
||||
print(str(e))
|
||||
return -1
|
||||
|
||||
|
||||
def search_author(db_cursor, author_search_query, author_data):
|
||||
try:
|
||||
# Search Author
|
||||
db_cursor.execute(author_search_query, author_data)
|
||||
row = db_cursor.fetchone()
|
||||
if row is not None:
|
||||
author_data['id'] = int(row[0])
|
||||
author_data['total_books'] = int(row[2])
|
||||
return author_data
|
||||
else:
|
||||
author_data['id'] = -1
|
||||
return author_data
|
||||
|
||||
except Exception as e:
|
||||
print(str(e))
|
||||
author_data['id'] = -1
|
||||
return author_data
|
||||
|
||||
|
||||
def update_author_book_count(db_cursor, author_update_query, author_data):
|
||||
try:
|
||||
# Update Author Book Count
|
||||
db_cursor.execute(author_update_query, author_data)
|
||||
row_cnt = db_cursor.rowcount
|
||||
if row_cnt > 0:
|
||||
return row_cnt
|
||||
else:
|
||||
return -1
|
||||
|
||||
except Exception as e:
|
||||
print(str(e))
|
||||
return -1
|
||||
@ -0,0 +1,24 @@
|
||||
BOOK_INSERT_QUERY = "INSERT INTO dim_book (code, added_at) " \
|
||||
"VALUES (%(code)s, %(added_at)s)"
|
||||
|
||||
AUTHOR_INSERT_QUERY = "INSERT INTO dim_author (name, total_books) " \
|
||||
"VALUES (%(name)s, %(total_books)s)"
|
||||
|
||||
BOOK_INFO_INSERT_QUERY = "INSERT INTO dim_book_info (title, description, lang, source, is_translation, " \
|
||||
"total_chapters, isbn, book) " \
|
||||
"VALUES (%(title)s, %(description)s, %(lang)s, %(source)s, %(is_translation)s, " \
|
||||
"%(total_chapters)s, %(isbn)s, %(book)s) "
|
||||
|
||||
BOOK_AUTHOR_INSERT_QUERY = "INSERT INTO map_book_author (author, book, translator) " \
|
||||
"VALUES (%(author)s, %(book)s, %(translator)s)"
|
||||
|
||||
CONTENT_INSERT_QUERY = "INSERT INTO dim_book_content (book) VALUES(%(book)s)"
|
||||
|
||||
CHAPTER_INSERT_QUERY = "INSERT INTO dim_book_chapter (c_num, name, book_content) " \
|
||||
"VALUES (%(c_num)s, %(name)s, %(book_content)s)"
|
||||
|
||||
SENTENCE_INSERT_QUERY = "INSERT INTO dim_book_sentence (s_num, text, chapter) VALUES (%(s_num)s, %(text)s, %(chapter)s)"
|
||||
|
||||
AUTHOR_SEARCH_QUERY = "SELECT * FROM dim_author WHERE dim_author.name = %(name)s"
|
||||
|
||||
AUTHOR_UPDATE_QUERY = "UPDATE dim_author SET dim_author.total_books = %(total_books)s WHERE id = %(id)s"
|
||||
@ -0,0 +1,31 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
import xml_parser.read_xml as read_xml
|
||||
import db.add_book as adb
|
||||
|
||||
json_file_path = Path('json/books.json')
|
||||
|
||||
with open(json_file_path, 'r') as json_file:
|
||||
json_data = json.load(json_file)
|
||||
json_file.close()
|
||||
|
||||
books_json = json_data['books']
|
||||
for book_code in books_json.keys():
|
||||
books_list = books_json[book_code]
|
||||
for book in books_list:
|
||||
# TODO :: Add not for the below check later (after doing XSD)
|
||||
if book['is_validated']:
|
||||
print('Book : ', book['xml_file'], ' is not validated against XSD')
|
||||
continue
|
||||
if not book['is_saved_to_db']:
|
||||
print('Saving Book : ', book['xml_file'], ' in the DB')
|
||||
book_dict = read_xml.parse_xml_file(book['xml_file_path'])
|
||||
result = adb.add_book_to_db(book_code, book_dict)
|
||||
book['is_saved_to_db'] = result
|
||||
|
||||
json_data['books'] = books_json
|
||||
|
||||
with open(json_file_path, 'w') as updated_json:
|
||||
updated_json.write(json.dumps(json_data, indent=4))
|
||||
updated_json.close()
|
||||
|
||||
Loading…
Reference in new issue