Python on Council of Elrond

Smart Way to do Regex

Fri, 06 Oct 2023 23:23:10 +0800

Below is a sample code that I found to get complex regex pattern instead of write it manually good example to make smaller parts first the combine to a very complex pattern.


import re

# convert the {name} test to rules in predefined format in macros
def regex_expand(macros, pattern, guarded = True):
 output = []
 pos = 0
 size = len(pattern)
 while pos < size:
 ch = pattern[pos]
 if ch == '\\':
 output.append(pattern[pos:pos + 2])
 pos += 2
 continue
 elif ch != '{':
 output.append(ch)
 pos += 1
 continue
 p2 = pattern.find('}', pos)
 if p2 < 0:
 output.append(ch)
 pos += 1
 continue
 p3 = p2 + 1
 name = pattern[pos + 1:p2].strip('\r\n\t ')
 if name == '':
 output.append(pattern[pos:p3])
 pos = p3
 continue
 elif name[0].isdigit():
 output.append(pattern[pos:p3])
 pos = p3
 continue
 elif ('<' in name) or ('>' in name):
 raise ValueError('invalid pattern name "%s"'%name)
 if name not in macros:
 raise ValueError('{%s} is undefined'%name)
 if guarded:
 output.append('(?:' + macros[name] + ')')
 else:
 output.append(macros[name])
 pos = p3
 return ''.join(output)

# given a set of rules, create rules dictionary
def regex_build(code, macros = None, capture = True):
 defined = {}
 if macros is not None:
 for k, v in macros.items():
 defined[k] = v
 line_num = 0
 for line in code.split('\n'):
 line_num += 1
 line = line.strip('\r\n\t ')
 if (not line) or line.startswith('#'):
 continue
 pos = line.find('=')
 if pos < 0:
 raise ValueError('%d: not a valid rule'%line_num)
 head = line[:pos].strip('\r\n\t ')
 body = line[pos + 1:].strip('\r\n\t ')
 if (not head):
 raise ValueError('%d: empty rule name'%line_num)
 elif head[0].isdigit():
 raise ValueError('%d: invalid rule name "%s"'%(line_num, head))
 elif ('<' in head) or ('>' in head):
 raise ValueError('%d: invalid rule name "%s"'%(line_num, head))
 try:
 pattern = regex_expand(defined, body, guarded = not capture)
 except ValueError as e:
 raise ValueError('%d: %s'%(line_num, str(e)))
 try:
 re.compile(pattern)
 except re.error:
 raise ValueError('%d: invalid pattern "%s"'%(line_num, pattern))
 if not capture:
 defined[head] = pattern
 else:
 defined[head] = '(?P<%s>%s)'%(head, pattern)
 return defined

# define a set of rules
rules = r'''
 protocol = http|https
 login_name = [^:@\r\n\t ]+
 login_pass = [^@\r\n\t ]+
 login = {login_name}(:{login_pass})?
 host = [^:/@\r\n\t ]+
 port = \d+
 optional_port = (?:[:]{port})?
 path = /[^\r\n\t ]*
 url = {protocol}://({login}[@])?{host}{optional_port}{path}?
'''

# convert above rules to dictionary
m = regex_build(rules, capture = True)

# print dictionary info
for k, v in m.items():
 print(k, '=', v)

print()

# use final rule "url" to regex a text
pattern = m['url']
s = re.match(pattern, 'https://name:pass@www.test.com:8080/abcdefg')

# print the complete list of matches
print('matched: "%s"'%s.group(0))
print()

# print the matched items based on rule defined names
for name in ('url', 'login_name', 'login_pass', 'host', 'port', 'path'):
 print('subgroup:', name, '=', s.group(name))

Output:

Python_One_line

Mon, 22 Nov 2021 22:08:48 +0800

Tricks to write Python in one line

Get dict output value

dict_test = {"A": 100, "B": 99, "C": 33, "D": 120}

output = [(key, value) for key, value in dict_test.items() if value >= 100]

print(f"the output is ：{output}")

the output is ：[('A', 100), ('D', 120)]

type(output)
list

Filter words that is more than 3 characters in a sentence

text = '''Outside, the moon is shedding its cold light on the cold snow,\nand the white-bearded fir-trees round Camp Villa1 are casting a blue shadow across the white ground, \nwhile the Rev2'''
output = [[x for x in line.split() if len(x) < 3] for line in text.split("\n")]
print(f"the output is：{output}")

the output is：[['is', 'on'], ['a'], []]

#logic is: 1st,split the text to a list, separater is "\n"(new line),
#2nd, for each line, split by space, which becomes a list again. 
#3rd, for each word in 2nd list, if the length of the word is less than 3(less than 3 characters) 
#put 3rd result in to a list. then for each line you get a list, combine it to a matrix

Remove spaces before/after the text

I modified log file to make a simple demo.

Python_Regex

Tue, 12 Oct 2021 20:59:46 +0800

Some Advanced usage on regex

get float number:

import re
target = 'total growth is : 8%-8.5%'
rate = re.findall('(\d+(?:\.\d+)?)', target)
print(rate)
['8', '8.5']
#\d is any int, + is one to many
#(?:...)
#A non-capturing group allows you to apply quantifiers to part of your regex but does not capture/assign an ID.
#so it means: find int or, if int follows a "." and any int afterwards, combine the int with ".int"
#For example, repeating 1-3 digits and a period 3 times can be done like #this: /(?:\d{1,3}\.){3}\d{1,3}/
#above regex can match string like : 219.254.232.123 or 232.212.222.3434 (can only go until 232.212.222.343), 
#can not match 232.2122.222.3434 or 232.212.22a.3434

#`(\d+(?:\.\d+)?)` is a group, so result will be put in a list. 

#if you just want to match any int or .xxx or xxxx.xxx,just use

rate = re.findall('(\d+(\.\d+)?)', target)
print(rate)
[('8', ''), ('8.5', '.5')]
#which will return a list to you

#if you just want to find a float, use below: 

target = 'total growth is : 8%-8.5% or 9.546%'
rate = re.findall('(\d+\.\d+)', target)
print(rate)
['8.5', '9.546']

A bit deeper:

A(?=B) ：Match String A ends with String B

Python_mysql

Sun, 07 Feb 2021 13:21:07 +0800

Python Query mysql

To use Python to query mysql DB. There are two popular modules are used:

pymysql
sqlalchemy ( need to install mysqlconnector in my scenario) pip install mysql-connector-python

pymysql:

ekoudb = pymysql.connect(host='10.10.80.127', 
 port=3306, 
 user='ekou', 
 password='abcd12334', 
 database='ekoutest') 
 
 #get cursor 
cursor = ekoudb.cursor()

def insert_a_item(SqlLang): 
 """ 
 Insert items """ 
 try: 
 cursor.execute(SqlLang) 
 ekoudb.commit() 
 except Exception as e: 
 print('Insert data fail') 
 print(e) 
 ekoudb.rollback() 
 # release resource 
 cursor.close() 
 ekoudb.close() 
#insert_a_item(SQL_INSERT_A_ITEM)
SQL_INSERT_MANY_ITEMS = "INSERT INTO bird_table (origin, hostname,descr,ver,protocol,neighboraddr,neighboras,status,sinceday,sincetime) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"

MULTIENTRY = [["update","host1","descr1","ipv6","BGP","2123123","231233","Idle","2021-02-02","16:50:07"],\ 
["update","host2","descr2","ipv6","BGP","ipv6_addr1","65300","Idle","2021-02-02","16:50:07"],\ 
["update","host3","descr3","ipv4","BGP","10.10.1.1","65002","Established","2021-02-02","15:50:13"],\ 
["update","host4","descr4","ipv4","BGP","10.10.1.3","65003","Established","2021-02-02","15:50:13"]]

def insert_items(MANYTIMESFORMAT,SqlLang): 
 try: 
 cursor.executemany(MANYTIMESFORMAT, SqlLang) 
 ekoudb.commit() 
 except Exception as e: 
 print("Insert_error") 
 ekoudb.rollback() 
 # release resource 
 cursor.close() 
 ekoudb.close()

def query(SqlLang): 
 """query""" 
 # query all data 
 cursor.execute(SqlLang) 
 print('+--------+-----------+---------------------------+------+----------+-\ 
-------------------------+------------+-------------+------------+-----------+') 
 print('| origin | hostname | descr | ver | protocol | \ 
 neighboraddr | neighboras | status | sinceday | sincetime |') 
 print('+--------+-----------+---------------------------+------+----------+-\ 
-------------------------+------------+-------------+------------+-----------+') 
 # matrix data 
 rows = cursor.fetchall() 
 for row in rows: 
 origin = row[0] 
 hostname = row[1] 
 descr = row[2] 
 ver = row[3] 
 protocol = row[4] 
 neighboraddr = str(row[5]) 
 neighboras = str(row[6]) 
 status = row[7] 
 sinceday = str(row[7]) 
 sincetime = str(row[9]) 
 
# print('| ' + origin.rjust(2) + '|'.rjust(7) + hostname.rjust(2) + '|'.rjust(10) + descr.rjust(2) + '|'.rjust(26) + ver.rjust(2) + '|'.rjust(5) + protocol.rjust(2) + '|'.rjust(9) + neighboraddr.rjust(2) + + '|'.rjust(25) + neighboras.rjust(2) + '|'.rjust(11) + status.rjust(2) + '|'.rjust(12) + sinceday.rjust(2) + '|'.rjust(11) + sincetime.rjust(2)) 
# print('| ' + origin.rjust(2) + '\t|' + '\t' + hostname + '\t|' + '\t'+ descr + '\t|' + '\t' + ver + '\t|' + '\t' + protocol + '\t|'+'\t'+neighboras + '\t|'+'\t'+neighboraddr + '\t|'+'\t'+status + '\t|'+ '\t' + sinceday + '\t|'+ '\t' +sincetime + '\t|') 
 print('|' + format(origin,"<8") + format("|","<2") + format(hostname,"<10") + format("|","<2") + format(descr,"<26") + format("|","<2") + format(ver,"<5") + format("|","<2") + format(protocol,"<9") + format("|","<2") + format(neighboraddr,"<25") + format("|","<2") + format(neighboras,"<11") + format("|","<2") + format(status,"<12") + format("|","<2") + format(sinceday,"<11") + format("|","<2") + format(sincetime,"<10") + "|" ) 
 print('+--------+-----------+---------------------------+------+----------+-\ 
-------------------------+------------+-------------+------------+-----------+') 
 # release resource 
 cursor.close() 
 ekoudb.close()
 


SQL_UPDATE = "UPDATE bird_table SET hostname='%s',status='%s' WHERE descr='%s'" 
 
def update(SqlLang): 
 """ 
 update 
 :return: """ sql_update = SqlLang % ('hostname3', 'connect', 'descr_1') 
 try: 
 cursor.execute(sql_update) 
 ekoudb.commit() 
 except Exception as e: 
 ekoudb.rollback() 
 print('error when update data') 
 print(e) 
 # release resource 
 cursor.close() 
 ekoudb.close() 
 
update(SQL_UPDATE)
SQL_DELETE = "DELETE FROM bird_table WHERE descr='%s'" 
 
def delete(SqlLang,delete_string): 
 """ 
 delete record 
 :return: """ try: 
 # delete sql 
 sql_del = SqlLang % (delete_string) 
 cursor.execute(sql_del) 
 ekoudb.commit() 
 except Exception as e: 
 # rollback when error 
 ekoudb.rollback() 
 print(e) 
 # release resource 
 cursor.close() 
 ekoudb.close() 
 
delete(SQL_DELETE,'23123213213')

sqlalchemy:

engine = create_engine("mysql+mysqlconnector://ekou:PASSWORD@10.10.80.127:3306/ekoutest?charset=utf8mb4")

metadata = MetaData(engine) 
 
Session = sessionmaker() # create a customized session 
Session.configure(bind=engine) # link to this session 
session = Session()
users_table = Table('bird_table', metadata, autoload=True)

result=users_table.select(and_(users_table.c.descr == "descr", users_table.c.neighboraddr == "NEIGHBOR_ADDR")).execute() 
for item in result.fetchall(): 
 print (item)

''' 
create table 
''' 
users_table = Table('users', 
 metadata, 
 Column('id', Integer, primary_key=True), 
 Column('name', String(40)), 
 Column('email', String(120))) 
if not users_table.exists(): 
 users_table.create() 
 
''' 
load table 
''' 
users_table = Table('users', metadata, autoload=True) 
 
''' 
insert 
''' 
users_table.insert().execute(name="sss", email="eee@eee.com") 
users_table.insert().execute(name="ttt", email="tt@eee.com") 
''' 
update 
''' 
users_table.update(users_table.c.name == "ttt").execute(name="ddd")

conclusion:

for easier get on hand, pymsql would be good enough to use. but for advanced, need to use sqlalchemy.