diff --git a/README.md b/README.md index c4a6215..7bbc48f 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,121 @@ -# hackathon-season2 -Hackathon Challenge Season2 +# Devmountain Tech Fest +> Hackathon Season2 +> +> Qualifiers Round +> Transform xml employee data to sqlite with view and on-demand user query db - python3 + +## Team members +> We Are `mVATOR` +> - Mr.Theerawat [@sajarudth](https://github.com/sajarudth) +> - Mr.Paphavich [@thaiversion](https://github.com/thaiversion) +> - Mr.Thongrit [@badoss](https://github.com/badoss) + +## Data Visualization Demo Dashboard + +> - hosted website (cloudflare) +> - url[https://hackathon-season2-data-analysis.pages.dev] +> - github +> - url[https://github.com/badoss/hackathon-season2-data-analysis.git] + + +## Required Environment +- python 3.10.4 + +## How to Implement + +1. clone this repository +``` +git clone https://github.com/thaiversion/hackathon-season2_devmountaintechfest.git +``` + +2. change directory to project path +for example +``` +cd hackathon-season2_devmountaintechfest +``` + +3. start program +description of each program is shown on below section + + +#### program #1 starter_transform_empdata_tosqlite.py +functionality >> convert xml file (data-devclub-1.xml) and filter data then add to sqlite db table "employee" with 3 following sqlite views +and the final save .csv which each file contain employee record group by same nationality > file format devclub_\.csv +``` +list of sqlite view that focus on user query perspective +1. vEMPLOYEEBYDEPT +2. vEMPLOYEEBYNATIONALITY +3. vEMPLOYEEBYREGION +``` + + +How to run file +``` +python starter_transform_empdata_tosqlite.py +``` +\*sqlite db store in filename "devclub.db" +\*example screenshot db on file "./screenshot/ss_example_sqlitedb_viewnotincluded.png" +\*example screenshot script on file "./screenshot/ss_starter_transform_empdata_tosqlite.py_importxml_tosqlite.png" + + +#### program #2 userquery_searchby_region.py +functionality >> user ondemand script to query database, ask user for input region name on python cmd +and get retrive employee record(s) with 'REGION' field condition then display json format to user on python cmd +and save json format string to file "lastestresult_userquery_searchby_region.json" + + +How to run file +``` +python userquery_searchby_region.py +``` +then user must input some charecters to python cmd +\*valid charecter is A-Z and a-z , must not input number or any special charecter +\*result json file maybe empty cuz lastest user input is invalid then clear any result remain on .json +\*example screenshot script case found record on file "./screenshot/ss_userquery_case_found.png" +\*example screenshot script case no record return on file "./screenshot/ss_userquery_case_notfound.png" +\*example screenshot script case user input invalid charecter on file "./screenshot/ss_userquery_case_notfound.png" + + +#### program #3 userquery_searchby_dept.py +functionality >> user ondemand script to query database, ask user for input department name on python cmd +and get retrive employee record(s) with 'DEPT' field condition then display json format to user on python cmd +and save json format string to file "lastestresult_userquery_searchby_dept.json" + + +How to run file +``` +python userquery_searchby_dept.py +``` +then user must input some charecters to python cmd +\*valid charecter is A-Z and a-z , must not input number or any special charecter +\*result json file maybe empty cuz lastest user input is invalid then clear any result remain on .json +\*example screenshot script case found record on file "./screenshot/ss_userquery_case_found.png" +\*example screenshot script case no record return on file "./screenshot/ss_userquery_case_notfound.png" +\*example screenshot script case user input invalid charecter on file "./screenshot/ss_userquery_case_notfound.png" + + +#### program #4 userquery_searchby_nationality.py +functionality >> user ondemand script to query database, ask user for input nationality name on python cmd +and get retrive employee record(s) with 'NATIONALITY' field condition then display json format to user on python cmd +and save json format string to file "lastestresult_userquery_searchby_nationality.json" + + +How to run file +``` +python userquery_searchby_nationality.py +``` +then user must input some charecters to python cmd +\*valid charecter is A-Z and a-z , must not input number or any special charecter +\*result json file maybe empty cuz lastest user input is invalid then clear any result remain on .json +\*example screenshot script case found record on file "./screenshot/ss_userquery_case_found.png" +\*example screenshot script case no record return on file "./screenshot/ss_userquery_case_notfound.png" +\*example screenshot script case user input invalid charecter on file "./screenshot/ss_userquery_case_notfound.png" + + +## Once Again +Data Visualization Demo Dashboard is displayed on website [https://hackathon-season2-data-analysis.pages.dev] + +> +> mVATOR is coming to train 🚉💪 +--- +Syntex Guide on GitHub [GitHub Help](https://help.github.com/articles/basic-writing-and-formatting-syntax/). diff --git a/devclub.db b/devclub.db new file mode 100644 index 0000000..0766858 Binary files /dev/null and b/devclub.db differ diff --git a/devclub_France.csv b/devclub_France.csv new file mode 100644 index 0000000..6c20c41 --- /dev/null +++ b/devclub_France.csv @@ -0,0 +1,2 @@ +EMPID,PASSPORT,FIRSTNAME,LASTNAME,GENDER,BIRTHDAY,NATIONALITY,HIRED,DEPT,POSITION,STATUS,REGION +93,UXL43IOW6OV,Honorato,Maxwell,1,1982-03-09,France,2017-02-04,Aircraft Maintenance,Airhostess,1,Europe \ No newline at end of file diff --git a/devclub_Germany.csv b/devclub_Germany.csv new file mode 100644 index 0000000..ca9e48b --- /dev/null +++ b/devclub_Germany.csv @@ -0,0 +1,2 @@ +EMPID,PASSPORT,FIRSTNAME,LASTNAME,GENDER,BIRTHDAY,NATIONALITY,HIRED,DEPT,POSITION,STATUS,REGION +29,CMK62UAD3VK,Rowan,Leonard,1,1974-07-15,Germany,2004-03-27,Aircraft Maintenance,Pilot,1,Ocenia \ No newline at end of file diff --git a/devclub_Indonesia.csv b/devclub_Indonesia.csv new file mode 100644 index 0000000..b375620 --- /dev/null +++ b/devclub_Indonesia.csv @@ -0,0 +1,2 @@ +EMPID,PASSPORT,FIRSTNAME,LASTNAME,GENDER,BIRTHDAY,NATIONALITY,HIRED,DEPT,POSITION,STATUS,REGION +33,EWD45RJW5YK,Carter,Velasquez,0,1967-11-23,Indonesia,2005-02-27,Flight Planning,Pilot,1,APAC \ No newline at end of file diff --git a/devclub_Italy.csv b/devclub_Italy.csv new file mode 100644 index 0000000..44fec80 --- /dev/null +++ b/devclub_Italy.csv @@ -0,0 +1,3 @@ +EMPID,PASSPORT,FIRSTNAME,LASTNAME,GENDER,BIRTHDAY,NATIONALITY,HIRED,DEPT,POSITION,STATUS,REGION +34,BFS82MEY3CX,Selma,Bush,0,1972-03-26,Italy,2008-10-10,Flight Attendance,Airhostess,1,USA +80,EUC74ENE9ZK,Ryan,Rush,0,1998-06-13,Italy,2019-07-31,Aircraft Maintenance,Pilot,1,Middle East \ No newline at end of file diff --git a/devclub_Mexico.csv b/devclub_Mexico.csv new file mode 100644 index 0000000..5e97c1e --- /dev/null +++ b/devclub_Mexico.csv @@ -0,0 +1,2 @@ +EMPID,PASSPORT,FIRSTNAME,LASTNAME,GENDER,BIRTHDAY,NATIONALITY,HIRED,DEPT,POSITION,STATUS,REGION +50,MRC33GHJ2KW,Calvin,Roach,1,1999-04-16,Mexico,2011-03-18,Flight Attendance,Steward,1,Europe \ No newline at end of file diff --git a/devclub_Netherlands.csv b/devclub_Netherlands.csv new file mode 100644 index 0000000..a883b7d --- /dev/null +++ b/devclub_Netherlands.csv @@ -0,0 +1,2 @@ +EMPID,PASSPORT,FIRSTNAME,LASTNAME,GENDER,BIRTHDAY,NATIONALITY,HIRED,DEPT,POSITION,STATUS,REGION +23,NFH65BYM0VB,Armand,Horn,0,1987-05-24,Netherlands,2007-06-19,Aircraft Maintenance,Airhostess,1,Ocenia \ No newline at end of file diff --git a/devclub_Pakistan.csv b/devclub_Pakistan.csv new file mode 100644 index 0000000..505ea12 --- /dev/null +++ b/devclub_Pakistan.csv @@ -0,0 +1,2 @@ +EMPID,PASSPORT,FIRSTNAME,LASTNAME,GENDER,BIRTHDAY,NATIONALITY,HIRED,DEPT,POSITION,STATUS,REGION +3,JUI65YBK7AF,Jada,Bender,0,1963-05-28,Pakistan,2001-02-11,Pilot,Pilot,1,Canada \ No newline at end of file diff --git a/devclub_Peru.csv b/devclub_Peru.csv new file mode 100644 index 0000000..dea39c2 --- /dev/null +++ b/devclub_Peru.csv @@ -0,0 +1,2 @@ +EMPID,PASSPORT,FIRSTNAME,LASTNAME,GENDER,BIRTHDAY,NATIONALITY,HIRED,DEPT,POSITION,STATUS,REGION +97,SUF73DKV4QE,Dante,Hart,0,1999-12-21,Peru,2016-02-22,Pilot,Pilot,1,Europe \ No newline at end of file diff --git a/devclub_Philippines.csv b/devclub_Philippines.csv new file mode 100644 index 0000000..0d686f9 --- /dev/null +++ b/devclub_Philippines.csv @@ -0,0 +1,2 @@ +EMPID,PASSPORT,FIRSTNAME,LASTNAME,GENDER,BIRTHDAY,NATIONALITY,HIRED,DEPT,POSITION,STATUS,REGION +66,WKV12UQC6QF,Zachery,Valentine,0,1971-06-04,Philippines,2011-08-25,Flight Attendance,Steward,1,Middle East \ No newline at end of file diff --git a/devclub_Singapore.csv b/devclub_Singapore.csv new file mode 100644 index 0000000..2cc7022 --- /dev/null +++ b/devclub_Singapore.csv @@ -0,0 +1,2 @@ +EMPID,PASSPORT,FIRSTNAME,LASTNAME,GENDER,BIRTHDAY,NATIONALITY,HIRED,DEPT,POSITION,STATUS,REGION +95,OUP31WOE2IE,Dara,Wilcox,1,1996-06-29,Singapore,2011-05-18,Flight Attendance,Airhostess,1,Canada \ No newline at end of file diff --git a/devclub_Ukraine.csv b/devclub_Ukraine.csv new file mode 100644 index 0000000..a906b7a --- /dev/null +++ b/devclub_Ukraine.csv @@ -0,0 +1,2 @@ +EMPID,PASSPORT,FIRSTNAME,LASTNAME,GENDER,BIRTHDAY,NATIONALITY,HIRED,DEPT,POSITION,STATUS,REGION +5,AZE20CSG4MU,Lillian,Reese,0,1982-12-03,Ukraine,2002-05-19,Flight Planning,Steward,1,Canada \ No newline at end of file diff --git a/lastestresult_userquery_searchby_dept.json b/lastestresult_userquery_searchby_dept.json new file mode 100644 index 0000000..e69de29 diff --git a/lastestresult_userquery_searchby_nationality.json b/lastestresult_userquery_searchby_nationality.json new file mode 100644 index 0000000..e69de29 diff --git a/lastestresult_userquery_searchby_region.json b/lastestresult_userquery_searchby_region.json new file mode 100644 index 0000000..fbaf448 --- /dev/null +++ b/lastestresult_userquery_searchby_region.json @@ -0,0 +1 @@ +[{"REGION": "APAC", "NATIONALITY": "Indonesia", "EMPID": 33, "PASSPORT": "EWD45RJW5YK", "FIRSTNAME": "Carter", "LASTNAME": "Velasquez", "GENDER": 0, "GENDER_TEXT": "Male", "BIRTHDAY": "1967-11-23", "AGE_of_EMPLOYEE": 54, "HIRED": "2005-02-27", "AGE_of_HIRED": 17, "DEPT": "Flight Planning", "POSITION": "Pilot", "STATUS": 1, "STATUS_TEXT": "Active"}] \ No newline at end of file diff --git a/screenshot/ss_demo_dashbord.png b/screenshot/ss_demo_dashbord.png new file mode 100644 index 0000000..6814caa Binary files /dev/null and b/screenshot/ss_demo_dashbord.png differ diff --git a/screenshot/ss_example_sqlitedb_viewnotincluded.png b/screenshot/ss_example_sqlitedb_viewnotincluded.png new file mode 100644 index 0000000..dcfbf3a Binary files /dev/null and b/screenshot/ss_example_sqlitedb_viewnotincluded.png differ diff --git a/screenshot/ss_starter_transform_empdata_tosqlite.py_importxml_tosqlite.png b/screenshot/ss_starter_transform_empdata_tosqlite.py_importxml_tosqlite.png new file mode 100644 index 0000000..15423ae Binary files /dev/null and b/screenshot/ss_starter_transform_empdata_tosqlite.py_importxml_tosqlite.png differ diff --git a/screenshot/ss_userquery_case_found.png b/screenshot/ss_userquery_case_found.png new file mode 100644 index 0000000..f6cd9cc Binary files /dev/null and b/screenshot/ss_userquery_case_found.png differ diff --git a/screenshot/ss_userquery_case_notfound.png b/screenshot/ss_userquery_case_notfound.png new file mode 100644 index 0000000..087fb58 Binary files /dev/null and b/screenshot/ss_userquery_case_notfound.png differ diff --git a/screenshot/ss_userquery_notvalidinput.png b/screenshot/ss_userquery_notvalidinput.png new file mode 100644 index 0000000..78b20f4 Binary files /dev/null and b/screenshot/ss_userquery_notvalidinput.png differ diff --git a/starter_transform_empdata_tosqlite.py b/starter_transform_empdata_tosqlite.py new file mode 100644 index 0000000..f3db3d7 --- /dev/null +++ b/starter_transform_empdata_tosqlite.py @@ -0,0 +1,420 @@ +#-*-coding: utf-8 -*- +# Dev Mountain Tech Fest - Qualifiers Round +# [python] xml/sqlite employee tranform data +# +# v1.0.0 09/10/65 +# - Initial Release +# +# Author: Team MFEC + + +# ================== Import library ================== +import pprint #pretty print +import sqlite3 #sqlite +import datetime #datetime +import xml.etree.ElementTree as ET #xml reader +from time import perf_counter #stopwatch +from dateutil.relativedelta import relativedelta #timedelta + + +# ============== defined variable ==================== +static_employee_xml_filepath = 'data-devclub-1.xml' +static_sqlitedb_filepath = 'devclub.db' + + +# ================== Function ======================== +# the custom recursive fuction that convert xml to json +""" +NOTE! explain the xml element +xml to json using built-in python library "xml" +child.tag << key +root.attrib << {} if not lowest level +root.text << value / none if not lowest level +""" +def XMLtoJSON(_nesteddata): + temp_data_dict = {} + temp_data_list = [] + for child in _nesteddata: + # check if level have child nest + if isinstance(child.text, type(None)): + # this level have child nest , store with list + temp_data_list.append(XMLtoJSON(child)) #use RECURSIVE!! + else: + # this level have no child nest , store with dict + temp_data_dict[child.tag] = child.text + if temp_data_list: + return {_nesteddata.tag:temp_data_list} + return {_nesteddata.tag:temp_data_dict} + +# read xml file and return table json +""" +NOTE! example xmldata_dict structure +{ + "records" : [ + { + "record" : { + "EMPID" : "1234", + "PASSPORT" : "A567", + ... + "REGION" : "THAI" + } + }, + .... + ] +} +""" +def ReadXML(_xmlfilepath=""): + # read raw file + with open(_xmlfilepath, 'r') as f: + rawfiledata = f.read() + rawfiledata = rawfiledata.replace("\n","").replace("\t","") + # custom parser xml + xmldata_xml = ET.fromstring(rawfiledata) + xmldata_dict = XMLtoJSON(xmldata_xml) + # pprint.pprint(xmldata_dict["records"][0]["record"]) + return xmldata_dict #employee_rawdata + +# clean up employee data by BRUTE FORCE method and insert data to sqlite +""" +NOTE! filter out with conditions +- POSITION is in "Airhostess" or "Pilot" or "Steward" +- Remove Worked less than 3 years +- Remove duplicate data << what is this ??? +- Ramove duplicate employeeId and passportNo and fullname +- Remove employess that have Resigned/Retired status +shown as 6 filter condition on below +""" +def ClearEmployeeData(_xmldata=[],_conn=None): + # variable counter + valid_employee_list = [] + temp_empid_list = [] + temp_passport_list = [] + temp_fullname_list = [] + temp_time_current = datetime.datetime.now() + temp_time_3yearsago = temp_time_current - relativedelta(years=3) + # loop for each record/row + for this_employee in _xmldata: + this_record = this_employee.get("record",{}) + # filter condition #1 : check Active status is not "Resigned" or "Retired" + if (thisemp_status := this_record.get("STATUS","")) not in ["1"]: + continue + # filter condition #2 : check if position is in "Airhostess" or "Pilot" or "Steward" + if (thisemp_position := this_record.get("POSITION","")) not in ["Airhostess","Pilot","Steward"]: + continue + # filter condition #3 : check if worked more than 3 years + thisemp_hired_dt = datetime.datetime.strptime(this_record.get("HIRED",""), '%d-%m-%Y') + if thisemp_hired_dt > temp_time_3yearsago: + continue + # filter condition #4 : check duplicate employeeId + if (thisemp_empid := this_record.get("EMPID","")) in temp_empid_list: + continue + temp_empid_list.append(thisemp_empid) + # filter condition #5 : check duplicate passportNo + if (thisemp_passport := this_record.get("PASSPORT","")) in temp_passport_list: + continue + temp_passport_list.append(thisemp_passport) + # filter condition #6 : check duplicate fullname + thisemp_fullname = this_record.get("FIRSTNAME","") + this_record.get("LASTNAME","") + if thisemp_fullname in temp_fullname_list: + continue + temp_fullname_list.append(thisemp_fullname) + # this employee is valid condition + thisemp_hired_float = datetime.datetime.timestamp(thisemp_hired_dt) #datetime to float + thisemp_hired_str = datetime.datetime.fromtimestamp(thisemp_hired_float).strftime('%Y-%m-%d') + thisemp_birthday_dt = datetime.datetime.strptime(this_record.get("BIRTHDAY",""), '%d-%m-%Y') + thisemp_birthday_float = datetime.datetime.timestamp(thisemp_birthday_dt) #datetime to float + thisemp_birthday_str = datetime.datetime.fromtimestamp(thisemp_birthday_float).strftime('%Y-%m-%d') + this_valid_employee = { + "EMPID" : int(thisemp_empid), + "PASSPORT" : thisemp_passport, + "FIRSTNAME" : this_record.get("FIRSTNAME",""), + "LASTNAME" : this_record.get("LASTNAME",""), + "GENDER" : int(this_record.get("GENDER","")), + "BIRTHDAY" : thisemp_birthday_str, + "NATIONALITY" : this_record.get("NATIONALITY",""), + "HIRED" : thisemp_hired_str, + "DEPT" : this_record.get("DEPT",""), + "POSITION" : thisemp_position, + "STATUS" : int(thisemp_status), + "REGION" : this_record.get("REGION","") + } + # pprint.pprint(this_valid_employee) #test print valid employee record + apistatus = db_create_employee(_conn,this_valid_employee) + valid_employee_list.append(this_valid_employee) + return valid_employee_list #employee_cleaned + +# sqlite db create connection +def db_create_connection(_dbfilepath=""): + """ create a database connection to the SQLite database + specified by _dbfilepath + :param _dbfilepath: database file + :return: Connection object or None + """ + conn = None + try: + conn = sqlite3.connect(_dbfilepath) + return conn + except sqlite3.Error as e: + print(e) + return conn + +# sqlite db create table +def db_create_table(_conn=None): + """ create a table from the create_table_sql statement + :param _conn: Connection object + :return: apistatus + """ + try: + mycursor = _conn.cursor() + create_table_sql = """ + CREATE TABLE IF NOT EXISTS employee( + EMPID INTEGER, + PASSPORT TEXT, + FIRSTNAME TEXT, + LASTNAME TEXT, + GENDER INTEGER, + BIRTHDAY NUMERIC, + NATIONALITY TEXT, + HIRED NUMERIC, + DEPT TEXT, + POSITION TEXT, + STATUS INTEGER, + REGION TEXT + ); + """ + mycursor.execute(create_table_sql) + create_view1_dept_sql = """ + CREATE VIEW IF NOT EXISTS vEMPLOYEEBYDEPT AS SELECT + DEPT, + EMPID, + PASSPORT, + FIRSTNAME, + LASTNAME, + GENDER, + CASE GENDER + WHEN 0 THEN 'Male' + WHEN 1 THEN 'Female' + ELSE '-' END GENDER_TEXT, + BIRTHDAY, + CAST((((JulianDay('now')) - JulianDay(BIRTHDAY))/365.25) AS INTEGER) AS AGE_of_EMPLOYEE, + REGION, + NATIONALITY, + HIRED, + CAST((((JulianDay('now')) - JulianDay(HIRED))/365.25) AS INTEGER) AS AGE_of_HIRED, + POSITION, + STATUS, + CASE STATUS + WHEN 1 THEN 'Active' + WHEN 2 THEN 'Resigned' + WHEN 3 THEN 'Retired' + ELSE 'anomaly information' END STATUS_TEXT + FROM employee + ORDER By DEPT ASC; + """ + mycursor.execute(create_view1_dept_sql) + create_view2_nationality_sql = """ + CREATE VIEW IF NOT EXISTS vEMPLOYEEBYNATIONALITY AS SELECT + NATIONALITY, + REGION, + EMPID, + PASSPORT, + FIRSTNAME, + LASTNAME, + GENDER, + CASE GENDER + WHEN 0 THEN 'Male' + WHEN 1 THEN 'Female' + ELSE '-' END GENDER_TEXT, + BIRTHDAY, + CAST((((JulianDay('now')) - JulianDay(BIRTHDAY))/365.25) AS INTEGER) AS AGE_of_EMPLOYEE, + HIRED, + CAST((((JulianDay('now')) - JulianDay(HIRED))/365.25) AS INTEGER) AS AGE_of_HIRED, + DEPT, + POSITION, + STATUS, + CASE STATUS + WHEN 1 THEN 'Active' + WHEN 2 THEN 'Resigned' + WHEN 3 THEN 'Retired' + ELSE 'anomaly information' END STATUS_TEXT + FROM employee + ORDER By NATIONALITY ASC; + """ + mycursor.execute(create_view2_nationality_sql) + create_view3_region_sql = """ + CREATE VIEW IF NOT EXISTS vEMPLOYEEBYREGION AS SELECT + REGION, + NATIONALITY, + EMPID, + PASSPORT, + FIRSTNAME, + LASTNAME, + GENDER, + CASE GENDER + WHEN 0 THEN 'Male' + WHEN 1 THEN 'Female' + ELSE '-' END GENDER_TEXT, + BIRTHDAY, + CAST((((JulianDay('now')) - JulianDay(BIRTHDAY))/365.25) AS INTEGER) AS AGE_of_EMPLOYEE, + HIRED, + CAST((((JulianDay('now')) - JulianDay(HIRED))/365.25) AS INTEGER) AS AGE_of_HIRED, + DEPT, + POSITION, + STATUS, + CASE STATUS + WHEN 1 THEN 'Active' + WHEN 2 THEN 'Resigned' + WHEN 3 THEN 'Retired' + ELSE 'anomaly information' END STATUS_TEXT + FROM employee + ORDER By REGION ASC; + """ + mycursor.execute(create_view3_region_sql) + + return True #apistatus + except sqlite3.Error as e: + print(e) + return False #apistatus + +# sqlite db delete old on employee table +def db_delete_old(_conn=None): + """ delete old data from employee table + :param _conn: Connection object + :return: apistatus + """ + try: + mycursor = _conn.cursor() + delete_old_sql = """ + DELETE FROM employee; + """ + mycursor.execute(delete_old_sql) + return True #apistatus + except sqlite3.Error as e: + print(e) + return False #apistatus + +# sqlite create employee record on employee table +def db_create_employee(_conn=None, _employee={}): + """ create a new employee record + :param _conn: + :param _employee: + :return: apistatus + """ + try: + myquery = """INSERT INTO employee(EMPID,PASSPORT,FIRSTNAME,LASTNAME,GENDER,BIRTHDAY,NATIONALITY,HIRED,DEPT,POSITION,STATUS,REGION) + VALUES(?,?,?,?,?,?,?,?,?,?,?,?)""" + myemployee = ( + _employee["EMPID"], + _employee["PASSPORT"], + _employee["FIRSTNAME"], + _employee["LASTNAME"], + _employee["GENDER"], + _employee["BIRTHDAY"], + _employee["NATIONALITY"], + _employee["HIRED"], + _employee["DEPT"], + _employee["POSITION"], + _employee["STATUS"], + _employee["REGION"] + ) + mycursor = _conn.cursor() + mycursor.execute(myquery, myemployee) + _conn.commit() + return True #apistatus + except sqlite3.Error as e: + print("sql error", e) + return False #apistatus + # return mycursor.lastrowid + return True #apistatus + +# save csv by nationality +def SaveCSVByNationality(_employee_list=[]): + temp_employee_nation_list = [] + temp_scaned_nation_list = [] + # find all nationality and list to temp nation list + for this_record in _employee_list: + if this_record["NATIONALITY"] not in temp_scaned_nation_list: + temp_scaned_nation_list.append(this_record["NATIONALITY"]) + # for each nationality, create csv file + for this_nation in temp_scaned_nation_list: + csv_filename = "devclub_" + this_nation + ".csv" + render_csv_infile = [] + render_csv_header = ['EMPID','PASSPORT','FIRSTNAME','LASTNAME','GENDER','BIRTHDAY','NATIONALITY','HIRED','DEPT','POSITION','STATUS','REGION'] + render_csv_infile.append(render_csv_header) + for this_employee in _employee_list: + if this_nation == this_employee["NATIONALITY"]: + # this record is match with nationaity + render_csv_infile.append([ + str(this_employee["EMPID"]), + this_employee["PASSPORT"], + this_employee["FIRSTNAME"], + this_employee["LASTNAME"], + str(this_employee["GENDER"]), + this_employee["BIRTHDAY"], + this_employee["NATIONALITY"], + this_employee["HIRED"], + this_employee["DEPT"], + this_employee["POSITION"], + str(this_employee["STATUS"]), + this_employee["REGION"] + ]) + #NOTE! now render_csv_infile is contain all employee record of this_nation + # write to csv file + with open(csv_filename, 'w+') as stream: + stream.truncate() + len_row = len(render_csv_infile) + for i in range(len_row): + strmid = "," #csv delimiter + strrow = strmid.join(render_csv_infile[i]) + stream.write(strrow) + if i < (len_row - 1) : #add new row while not end file + stream.write('\n') + stream.close() + return True #apistatus + + +# ================== Main Sector ===================== +def main(): + # STEP #0 PREPARING + time_then = perf_counter() + + # STEP #1 GET EMPLOYEE DATA FROM XML + employee_rawdata = ReadXML(static_employee_xml_filepath) + + # STEP #2 CREATE SQLLITE DB + conn = db_create_connection(static_sqlitedb_filepath) + if conn is not None: + apistatus = db_create_table(conn) + if apistatus: + # CLEAN OLD DATA + apistatus = db_delete_old(conn) + else: + print("Error! cannot create the database connection.") + + # STEP #3 CLEAN UP EMPLOYEE DATA AND INSERT TO SQLLITE + employee_cleaned = ClearEmployeeData(employee_rawdata.get("records",[]),conn) + del employee_rawdata + + # STEP #4 SAVE CSV FILE GROUP BY NATIONALITY FIELD + apistatus = SaveCSVByNationality(employee_cleaned) + + # FINALIZE + time_now = perf_counter() + print("Elapsed time : " + str(time_now-time_then) + " seconds") + +if __name__ == "__main__": + main() + + + +# notes + +# tested on python 3.10.4(CPU Apple Silicon) +# required library +# NO EXTERNAL LIBRARY IS REQUIRED + +# REF sqlite built-in lib : https://www.sqlitetutorial.net/sqlite-python/ +# REF xml etree built-in lib : https://docs.python.org/3/library/xml.etree.elementtree.html +# REF save file mode : https://mkyong.com/python/python-difference-between-r-w-and-a-in-open/ + +#very end \ No newline at end of file diff --git a/userquery_searchby_dept.py b/userquery_searchby_dept.py new file mode 100644 index 0000000..3f5b14e --- /dev/null +++ b/userquery_searchby_dept.py @@ -0,0 +1,129 @@ +#-*-coding: utf-8 -*- +# Dev Mountain Tech Fest - Qualifiers Round +# [python] user query search by department +# +# v1.0.0 09/10/65 +# - Initial Release +# +# Author: Team MFEC + + +# ================== Import library ================== +import json #json form +import pprint #pretty print +import sqlite3 #sqlite +# import datetime #datetime +from time import perf_counter + + +# ============== defined variable ==================== +static_sqlitedb_filepath = 'devclub.db' +static_savejson_filepath = 'lastestresult_userquery_searchby_dept.json' + + +# ================== Function ======================== +# get user input +def GetUserInput(): + print("Script search employee by department") + searchkeyword = input("Enter department name : ") + return str(searchkeyword) + +def SearchSQLiteDB(_searchfield="",_searchkeyword=""): + #basic handle + if not _searchkeyword: + print("No Input search keyword") + jsonfile_clear() + return False, None #apistatus, result_json + if not _searchkeyword.isalpha(): + print("Input search keyword is not valid") + jsonfile_clear() + return False, None #apistatus, result_json + #create a connection to the database + conn = None + try: + conn = sqlite3.connect(static_sqlitedb_filepath) + except sqlite3.Error as e: + print("cannot connect to database") + jsonfile_clear() + return False, None #apistatus, result_json + #query databasecad + myquery = "SELECT * " + \ + "FROM vEMPLOYEEBYDEPT WHERE " + _searchfield + " = '" +_searchkeyword + "'" + mycursor = conn.cursor() + mycursor.execute(myquery) + rows = mycursor.fetchall() + if not rows: + print("SQL result : Not found record with department name : " + _searchkeyword) + jsonfile_clear() + return False, None #apistatus, result_json + temp_result_list = [] + for row in rows: + # row is a tuple + this_record_dict = { + 'DEPT' : row[0], + 'EMPID' : row[1], + 'PASSPORT' : row[2], + 'FIRSTNAME' : row[3], + 'LASTNAME' : row[4], + 'GENDER' : row[5], + 'GENDER_TEXT' : row[6], + 'BIRTHDAY' : row[7], + 'AGE_of_EMPLOYEE' : row[8], + 'REGION' : row[9], + 'NATIONALITY' : row[10], + 'HIRED' : row[11], + 'AGE_of_HIRED' : row[12], + 'POSITION' : row[13], + 'STATUS' : row[14], + 'STATUS_TEXT' : row[15] + } + temp_result_list.append(this_record_dict) + # pprint.pprint(temp_result_list) #json form + return True, temp_result_list #apistatus, result_json + +# if no result clear json file +def jsonfile_clear(): + with open(static_savejson_filepath, 'w+') as stream: + stream.truncate() + stream.close() + +# if found result record write to json file +def jsonfile_write(render_infile=""): + with open(static_savejson_filepath, 'w+') as stream: + stream.truncate() + stream.write(render_infile) + stream.close() + + +# ================== Main Sector ===================== +def main(): + # STEP #1 GET USER INPUT + searchkeyword = GetUserInput() + + # STEP #2 SEARCH + time_then = perf_counter() + apistatus, result_json = SearchSQLiteDB("DEPT",searchkeyword) + + # STEP #3 PRINT RESULT AS JSON + time_now = perf_counter() + print("Elapsed running time : " + str(time_now-time_then) + " seconds") + if apistatus: + print("SQL Result : found record with department name : " + searchkeyword + " , total record : " + str(len(result_json))) + jsonfile_write(json.dumps(result_json)) + print("save result to json file : " + static_savejson_filepath) + print("result as json format is shown below") + print("==========================================================================") + pprint.pprint(result_json) + +if __name__ == "__main__": + main() + + + +# notes + +# tested on python 3.10.4(CPU Apple Silicon) +# required library +# NO EXTERNAL LIBRARY IS REQUIRED + +#very end \ No newline at end of file diff --git a/userquery_searchby_nationality.py b/userquery_searchby_nationality.py new file mode 100644 index 0000000..7fbdcb2 --- /dev/null +++ b/userquery_searchby_nationality.py @@ -0,0 +1,129 @@ +#-*-coding: utf-8 -*- +# Dev Mountain Tech Fest - Qualifiers Round +# [python] user query search by nationality +# +# v1.0.0 09/10/65 +# - Initial Release +# +# Author: Team MFEC + + +# ================== Import library ================== +import json #json form +import pprint #pretty print +import sqlite3 #sqlite +# import datetime #datetime +from time import perf_counter + + +# ============== defined variable ==================== +static_sqlitedb_filepath = 'devclub.db' +static_savejson_filepath = 'lastestresult_userquery_searchby_nationality.json' + + +# ================== Function ======================== +# get user input +def GetUserInput(): + print("Script search employee by nationality") + searchkeyword = input("Enter nationality name : ") + return str(searchkeyword) + +def SearchSQLiteDB(_searchfield="",_searchkeyword=""): + #basic handle + if not _searchkeyword: + print("No Input search keyword") + jsonfile_clear() + return False, None #apistatus, result_json + if not _searchkeyword.isalpha(): + print("Input search keyword is not valid") + jsonfile_clear() + return False, None #apistatus, result_json + #create a connection to the database + conn = None + try: + conn = sqlite3.connect(static_sqlitedb_filepath) + except sqlite3.Error as e: + print("cannot connect to database") + jsonfile_clear() + return False, None #apistatus, result_json + #query databasecad + myquery = "SELECT * " + \ + "FROM vEMPLOYEEBYNATIONALITY WHERE " + _searchfield + " = '" +_searchkeyword + "'" + mycursor = conn.cursor() + mycursor.execute(myquery) + rows = mycursor.fetchall() + if not rows: + print("SQL result : Not found record with nationality name : " + _searchkeyword) + jsonfile_clear() + return False, None #apistatus, result_json + temp_result_list = [] + for row in rows: + # row is a tuple + this_record_dict = { + 'NATIONALITY' : row[0], + 'REGION' : row[1], + 'EMPID' : row[2], + 'PASSPORT' : row[3], + 'FIRSTNAME' : row[4], + 'LASTNAME' : row[5], + 'GENDER' : row[6], + 'GENDER_TEXT' : row[7], + 'BIRTHDAY' : row[8], + 'AGE_of_EMPLOYEE' : row[9], + 'HIRED' : row[10], + 'AGE_of_HIRED' : row[11], + 'DEPT' : row[12], + 'POSITION' : row[13], + 'STATUS' : row[14], + 'STATUS_TEXT' : row[15] + } + temp_result_list.append(this_record_dict) + # pprint.pprint(temp_result_list) #json form + return True, temp_result_list #apistatus, result_json + +# if no result clear json file +def jsonfile_clear(): + with open(static_savejson_filepath, 'w+') as stream: + stream.truncate() + stream.close() + +# if found result record write to json file +def jsonfile_write(render_infile=""): + with open(static_savejson_filepath, 'w+') as stream: + stream.truncate() + stream.write(render_infile) + stream.close() + + +# ================== Main Sector ===================== +def main(): + # STEP #1 GET USER INPUT + searchkeyword = GetUserInput() + + # STEP #2 SEARCH + time_then = perf_counter() + apistatus, result_json = SearchSQLiteDB("NATIONALITY",searchkeyword) + + # STEP #3 PRINT RESULT AS JSON + time_now = perf_counter() + print("Elapsed running time : " + str(time_now-time_then) + " seconds") + if apistatus: + print("SQL Result : found record with nationality name : " + searchkeyword + " , total record : " + str(len(result_json))) + jsonfile_write(json.dumps(result_json)) + print("save result to json file : " + static_savejson_filepath) + print("result as json format is shown below") + print("==========================================================================") + pprint.pprint(result_json) + +if __name__ == "__main__": + main() + + + +# notes + +# tested on python 3.10.4(CPU Apple Silicon) +# required library +# NO EXTERNAL LIBRARY IS REQUIRED + +#very end \ No newline at end of file diff --git a/userquery_searchby_region.py b/userquery_searchby_region.py new file mode 100644 index 0000000..add3385 --- /dev/null +++ b/userquery_searchby_region.py @@ -0,0 +1,129 @@ +#-*-coding: utf-8 -*- +# Dev Mountain Tech Fest - Qualifiers Round +# [python] user query search by region +# +# v1.0.0 09/10/65 +# - Initial Release +# +# Author: Team MFEC + + +# ================== Import library ================== +import json #json form +import pprint #pretty print +import sqlite3 #sqlite +# import datetime #datetime +from time import perf_counter + + +# ============== defined variable ==================== +static_sqlitedb_filepath = 'devclub.db' +static_savejson_filepath = 'lastestresult_userquery_searchby_region.json' + + +# ================== Function ======================== +# get user input +def GetUserInput(): + print("Script search employee by region") + searchkeyword = input("Enter region name : ") + return str(searchkeyword) + +def SearchSQLiteDB(_searchfield="",_searchkeyword=""): + #basic handle + if not _searchkeyword: + print("No Input search keyword") + jsonfile_clear() + return False, None #apistatus, result_json + if not _searchkeyword.isalpha(): + print("Input search keyword is not valid") + jsonfile_clear() + return False, None #apistatus, result_json + #create a connection to the database + conn = None + try: + conn = sqlite3.connect(static_sqlitedb_filepath) + except sqlite3.Error as e: + print("cannot connect to database") + jsonfile_clear() + return False, None #apistatus, result_json + #query databasecad + myquery = "SELECT * " + \ + "FROM vEMPLOYEEBYREGION WHERE " + _searchfield + " = '" +_searchkeyword + "'" + mycursor = conn.cursor() + mycursor.execute(myquery) + rows = mycursor.fetchall() + if not rows: + print("SQL result : Not found record with region name : " + _searchkeyword) + jsonfile_clear() + return False, None #apistatus, result_json + temp_result_list = [] + for row in rows: + # row is a tuple + this_record_dict = { + 'REGION' : row[0], + 'NATIONALITY' : row[1], + 'EMPID' : row[2], + 'PASSPORT' : row[3], + 'FIRSTNAME' : row[4], + 'LASTNAME' : row[5], + 'GENDER' : row[6], + 'GENDER_TEXT' : row[7], + 'BIRTHDAY' : row[8], + 'AGE_of_EMPLOYEE' : row[9], + 'HIRED' : row[10], + 'AGE_of_HIRED' : row[11], + 'DEPT' : row[12], + 'POSITION' : row[13], + 'STATUS' : row[14], + 'STATUS_TEXT' : row[15] + } + temp_result_list.append(this_record_dict) + # pprint.pprint(temp_result_list) #json form + return True, temp_result_list #apistatus, result_json + +# if no result clear json file +def jsonfile_clear(): + with open(static_savejson_filepath, 'w+') as stream: + stream.truncate() + stream.close() + +# if found result record write to json file +def jsonfile_write(render_infile=""): + with open(static_savejson_filepath, 'w+') as stream: + stream.truncate() + stream.write(render_infile) + stream.close() + + +# ================== Main Sector ===================== +def main(): + # STEP #1 GET USER INPUT + searchkeyword = GetUserInput() + + # STEP #2 SEARCH + time_then = perf_counter() + apistatus, result_json = SearchSQLiteDB("REGION",searchkeyword) + + # STEP #3 PRINT RESULT AS JSON + time_now = perf_counter() + print("Elapsed running time : " + str(time_now-time_then) + " seconds") + if apistatus: + print("SQL Result : found record with region name : " + searchkeyword + " , total record : " + str(len(result_json))) + jsonfile_write(json.dumps(result_json)) + print("save result to json file : " + static_savejson_filepath) + print("result as json format is shown below") + print("==========================================================================") + pprint.pprint(result_json) + +if __name__ == "__main__": + main() + + + +# notes + +# tested on python 3.10.4(CPU Apple Silicon) +# required library +# NO EXTERNAL LIBRARY IS REQUIRED + +#very end \ No newline at end of file