From d7208088d466e690814a4cdcfe60866dbad15307 Mon Sep 17 00:00:00 2001 From: Pavan Mandava Date: Tue, 21 Jan 2020 19:54:54 +0100 Subject: [PATCH] Added more Print/Warning Messages & Improved Code Flow --- db/add_book.py | 2 +- db/constants.py | 2 ++ db/mysql_connection.py | 11 +++++++---- db/read_config.py | 5 +++++ db/test.py | 9 --------- db/test_db.py | 10 ++++++++++ db_config.ini | 2 +- run.py | 19 +++++++++++-------- utils/constants.py | 14 +++++++++++++- utils/env_utils.py | 15 +++++++++++++++ xml_parser/{test.py => test_parser.py} | 6 ++---- xml_parser/validate.py | 2 +- 12 files changed, 68 insertions(+), 29 deletions(-) delete mode 100644 db/test.py create mode 100644 db/test_db.py create mode 100644 utils/env_utils.py rename xml_parser/{test.py => test_parser.py} (78%) diff --git a/db/add_book.py b/db/add_book.py index b0a7ad4..c3acb86 100644 --- a/db/add_book.py +++ b/db/add_book.py @@ -6,7 +6,7 @@ import db.constants as const def add_book_to_db(book_code, book_dict): # print('Adding Book Code :: ', book_code, ' Dict :: ', book_dict) - conn = mysql.get_new_mysql_connection('db_config.ini') + conn = mysql.get_new_mysql_connection(const.DB_CONFIG_FILE) if conn is None: return False diff --git a/db/constants.py b/db/constants.py index d88a98d..fdd5fdf 100644 --- a/db/constants.py +++ b/db/constants.py @@ -1,3 +1,5 @@ +DB_CONFIG_FILE = 'db_config.ini' + BOOK_INSERT_QUERY = "INSERT INTO dim_book (code, added_at) " \ "VALUES (%(code)s, %(added_at)s)" diff --git a/db/mysql_connection.py b/db/mysql_connection.py index cef0db8..8c0d81f 100644 --- a/db/mysql_connection.py +++ b/db/mysql_connection.py @@ -1,11 +1,13 @@ import mysql.connector from mysql.connector import errorcode - import db.read_config as config +import utils.constants as const +import os -def get_new_mysql_connection(config_file_path): +def get_new_mysql_connection(config_file_name): + config_file_path = os.path.dirname(os.path.dirname(__file__))+'/'+config_file_name db_config = config.read_db_config(config_file_path, 'mysql') connection = None @@ -14,15 +16,16 @@ def get_new_mysql_connection(config_file_path): connection = mysql.connector.connect(**db_config) except mysql.connector.Error as err: if err.errno == errorcode.ER_ACCESS_DENIED_ERROR: - print('Invalid Database User and Password') + print(const.WARNING, 'Invalid Database User and Password', const.END) elif err.errno == errorcode.ER_BAD_DB_ERROR: - print('Database doesn\'t exist ') + print(const.WARNING, 'Database doesn\'t exist ', const.END) else: print(err) if connection is not None: if connection.is_connected(): connection.autocommit = False + print(const.GREEN, 'MySQL Connection Successful => Connection ID :: ', connection.connection_id, const.END) else: connection = None diff --git a/db/read_config.py b/db/read_config.py index 9bbf5f3..decf3e6 100644 --- a/db/read_config.py +++ b/db/read_config.py @@ -25,4 +25,9 @@ def read_db_config(filename, section): except KeyError: print('Please set the Environment Variable ', db['password']) + try: + db['host'] = os.environ[db['host']] + except KeyError: + print('Please set the Environment Variable ', db['host']) + return db diff --git a/db/test.py b/db/test.py deleted file mode 100644 index 2c24380..0000000 --- a/db/test.py +++ /dev/null @@ -1,9 +0,0 @@ -import db.mysql_connection as connection - -conn = connection.get_new_mysql_connection('../db_config.ini') - -print(conn.charset) -print('isConnected :: ', conn.is_connected()) - -conn.close() - diff --git a/db/test_db.py b/db/test_db.py new file mode 100644 index 0000000..1c048e9 --- /dev/null +++ b/db/test_db.py @@ -0,0 +1,10 @@ +import db.mysql_connection as connection +import db.constants as const + +conn = connection.get_new_mysql_connection(const.DB_CONFIG_FILE) + +print('MySQL Server version :: ', conn.get_server_info()) +print('isConnected :: ', conn.is_connected()) + +conn.close() + diff --git a/db_config.ini b/db_config.ini index 85719a5..402e8c0 100644 --- a/db_config.ini +++ b/db_config.ini @@ -1,5 +1,5 @@ [mysql] -host = 127.0.0.1 +host = MYSQL_HOST port = 3306 database = bitext-aligner user = root diff --git a/run.py b/run.py index cc2761a..6f1055a 100644 --- a/run.py +++ b/run.py @@ -1,9 +1,13 @@ -import json import xml_parser.read_xml as read_xml import db.add_book as adb import xml_parser.validate as validate import utils.json_utils as json_utils import utils.constants as const +import utils.env_utils as env + + +def validate_all_xml_files(): + validate.validate_all_xml_files() def save_validated_files_to_db(): @@ -13,20 +17,19 @@ def save_validated_files_to_db(): books_list = books_json[book_code] for book in books_list: if not book['is_validated']: - print('Book : ', book['xml_file'], ' is not validated against XSD') + print(const.WARNING, 'Book : ', book['xml_file'], ' is not validated against XSD', const.END) continue if not book['is_saved_to_db']: - print('Saving Book : ', book['xml_file'], ' in the DB') + print(const.BLUE, 'Adding Book : ', book['xml_file'], ' to the DB', const.END) book_dict = read_xml.parse_xml_file(book['xml_file_path']) result = adb.add_book_to_db(book_code, book_dict) book['is_saved_to_db'] = result + print(const.BLUE, 'Result :: ', result, const.END, '\n') json_data['books'] = books_json json_utils.write_json_file(const.JSON_PATH, json_data) -def validate_all_xml_files(): - validate.validate_all_xml_files() - - -validate_all_xml_files() \ No newline at end of file +if env.check_env_variables(): + validate_all_xml_files() + save_validated_files_to_db() \ No newline at end of file diff --git a/utils/constants.py b/utils/constants.py index 1ce96a6..f8c4860 100644 --- a/utils/constants.py +++ b/utils/constants.py @@ -1,3 +1,15 @@ JSON_PATH = 'json/books.json' -XSD_PATH = 'xml_files/book.xsd' \ No newline at end of file +XSD_PATH = 'xml_files/book.xsd' + +TRANSLATE_ENV_VAR = 'GOOGLE_APPLICATION_CREDENTIALS' + +MYSQL_PASS_ENV_VAR = 'MYSQL_PASSWORD' + +MYSQL_HOST_ENV_VAR = 'MYSQL_HOST' + +WARNING = '\033[91m' +END = '\033[0m' + +BLUE = '\033[94m' +GREEN = '\033[92m' \ No newline at end of file diff --git a/utils/env_utils.py b/utils/env_utils.py new file mode 100644 index 0000000..ea65302 --- /dev/null +++ b/utils/env_utils.py @@ -0,0 +1,15 @@ +import os +import utils.constants as const + + +def check_env_variables(): + if const.TRANSLATE_ENV_VAR not in os.environ: + print(const.WARNING, 'Please set the ', const.TRANSLATE_ENV_VAR, ' Environment Variable to continue....', const.END) + return False + if const.MYSQL_PASS_ENV_VAR not in os.environ: + print(const.WARNING, 'Please set the ', const.MYSQL_PASS_ENV_VAR, ' Environment Variable to continue....', const.END) + return False + if const.MYSQL_HOST_ENV_VAR not in os.environ: + print(const.WARNING, 'Please set the ', const.MYSQL_HOST_ENV_VAR, ' Environment Variable to continue....', const.END) + return False + return True diff --git a/xml_parser/test.py b/xml_parser/test_parser.py similarity index 78% rename from xml_parser/test.py rename to xml_parser/test_parser.py index af6fda8..0ac315c 100644 --- a/xml_parser/test.py +++ b/xml_parser/test_parser.py @@ -5,13 +5,11 @@ import xmlschema from pathlib import Path import xml_parser.validate as validate -file_path = create_xml.create_xml_file(get_book_content(), get_book_metadata()) +# file_path = create_xml.create_xml_file(get_book_content(), get_book_metadata()) -print(file_path) +# print(file_path) validate.validate_all_xml_files() # book_dict = read_xml.parse_xml_file('/Users/pavanmandava/PythonWorkspace/bitext-aligner/xml_files/abcdef_en.xml') - - diff --git a/xml_parser/validate.py b/xml_parser/validate.py index 2ea710e..d763feb 100644 --- a/xml_parser/validate.py +++ b/xml_parser/validate.py @@ -26,7 +26,7 @@ def validate_all_xml_files(): books_list = books_json[book_code] for book in books_list: if book['is_validated']: - print('Book : ', book['xml_file'], ' is valid') + print(const.BLUE, 'Book : ', book['xml_file'], ' is valid', const.END) continue else: if 'xml_file_path' in book: