Automation on Council of Elrond

Smart Way to do Regex

Fri, 06 Oct 2023 23:23:10 +0800

Below is a sample code that I found to get complex regex pattern instead of write it manually good example to make smaller parts first the combine to a very complex pattern.


import re

# convert the {name} test to rules in predefined format in macros
def regex_expand(macros, pattern, guarded = True):
 output = []
 pos = 0
 size = len(pattern)
 while pos < size:
 ch = pattern[pos]
 if ch == '\\':
 output.append(pattern[pos:pos + 2])
 pos += 2
 continue
 elif ch != '{':
 output.append(ch)
 pos += 1
 continue
 p2 = pattern.find('}', pos)
 if p2 < 0:
 output.append(ch)
 pos += 1
 continue
 p3 = p2 + 1
 name = pattern[pos + 1:p2].strip('\r\n\t ')
 if name == '':
 output.append(pattern[pos:p3])
 pos = p3
 continue
 elif name[0].isdigit():
 output.append(pattern[pos:p3])
 pos = p3
 continue
 elif ('<' in name) or ('>' in name):
 raise ValueError('invalid pattern name "%s"'%name)
 if name not in macros:
 raise ValueError('{%s} is undefined'%name)
 if guarded:
 output.append('(?:' + macros[name] + ')')
 else:
 output.append(macros[name])
 pos = p3
 return ''.join(output)

# given a set of rules, create rules dictionary
def regex_build(code, macros = None, capture = True):
 defined = {}
 if macros is not None:
 for k, v in macros.items():
 defined[k] = v
 line_num = 0
 for line in code.split('\n'):
 line_num += 1
 line = line.strip('\r\n\t ')
 if (not line) or line.startswith('#'):
 continue
 pos = line.find('=')
 if pos < 0:
 raise ValueError('%d: not a valid rule'%line_num)
 head = line[:pos].strip('\r\n\t ')
 body = line[pos + 1:].strip('\r\n\t ')
 if (not head):
 raise ValueError('%d: empty rule name'%line_num)
 elif head[0].isdigit():
 raise ValueError('%d: invalid rule name "%s"'%(line_num, head))
 elif ('<' in head) or ('>' in head):
 raise ValueError('%d: invalid rule name "%s"'%(line_num, head))
 try:
 pattern = regex_expand(defined, body, guarded = not capture)
 except ValueError as e:
 raise ValueError('%d: %s'%(line_num, str(e)))
 try:
 re.compile(pattern)
 except re.error:
 raise ValueError('%d: invalid pattern "%s"'%(line_num, pattern))
 if not capture:
 defined[head] = pattern
 else:
 defined[head] = '(?P<%s>%s)'%(head, pattern)
 return defined

# define a set of rules
rules = r'''
 protocol = http|https
 login_name = [^:@\r\n\t ]+
 login_pass = [^@\r\n\t ]+
 login = {login_name}(:{login_pass})?
 host = [^:/@\r\n\t ]+
 port = \d+
 optional_port = (?:[:]{port})?
 path = /[^\r\n\t ]*
 url = {protocol}://({login}[@])?{host}{optional_port}{path}?
'''

# convert above rules to dictionary
m = regex_build(rules, capture = True)

# print dictionary info
for k, v in m.items():
 print(k, '=', v)

print()

# use final rule "url" to regex a text
pattern = m['url']
s = re.match(pattern, 'https://name:pass@www.test.com:8080/abcdefg')

# print the complete list of matches
print('matched: "%s"'%s.group(0))
print()

# print the matched items based on rule defined names
for name in ('url', 'login_name', 'login_pass', 'host', 'port', 'path'):
 print('subgroup:', name, '=', s.group(name))

Output:

Git_cheat_sheet

Sat, 25 Dec 2021 01:31:18 +0800

Git cheat sheet

Configure

Configure system/global level of current repository username, email

git config --global user.name "Jerry Mouse" 
git config --global user.email "jerry@yiibai.com"`

List git configs

git config --list 
git config -l`

initialize

git init

Modify commit

git add a.html 
git commit -m "提交文件a.html"

git revert- is about making a new commit that reverts the changes made by other commits.git reset is about updating your branch, moving the tip in order to add or remove commits from the branch. This operation changes the commit history. it can also be used to restore the index, overlapping with git restore.git-restore is about restoring files in the working tree from either the index or another commit. This command does not update your branch. The command can also be used to restore files in the index from another commit.

Revert will reverse the changes from a commitment and is used mostly to undo faulty commits. This means that you can revert specific commits without affecting others. Revert works by making a new commit with the changes removed. You can revert any past commit. This can be the last commit, $ git revert HEAD.

Python_One_line

Mon, 22 Nov 2021 22:08:48 +0800

Tricks to write Python in one line

Get dict output value

dict_test = {"A": 100, "B": 99, "C": 33, "D": 120}

output = [(key, value) for key, value in dict_test.items() if value >= 100]

print(f"the output is ：{output}")

the output is ：[('A', 100), ('D', 120)]

type(output)
list

Filter words that is more than 3 characters in a sentence

text = '''Outside, the moon is shedding its cold light on the cold snow,\nand the white-bearded fir-trees round Camp Villa1 are casting a blue shadow across the white ground, \nwhile the Rev2'''
output = [[x for x in line.split() if len(x) < 3] for line in text.split("\n")]
print(f"the output is：{output}")

the output is：[['is', 'on'], ['a'], []]

#logic is: 1st,split the text to a list, separater is "\n"(new line),
#2nd, for each line, split by space, which becomes a list again. 
#3rd, for each word in 2nd list, if the length of the word is less than 3(less than 3 characters) 
#put 3rd result in to a list. then for each line you get a list, combine it to a matrix

Remove spaces before/after the text

I modified log file to make a simple demo.

Python_Regex

Tue, 12 Oct 2021 20:59:46 +0800

Some Advanced usage on regex

get float number:

import re
target = 'total growth is : 8%-8.5%'
rate = re.findall('(\d+(?:\.\d+)?)', target)
print(rate)
['8', '8.5']
#\d is any int, + is one to many
#(?:...)
#A non-capturing group allows you to apply quantifiers to part of your regex but does not capture/assign an ID.
#so it means: find int or, if int follows a "." and any int afterwards, combine the int with ".int"
#For example, repeating 1-3 digits and a period 3 times can be done like #this: /(?:\d{1,3}\.){3}\d{1,3}/
#above regex can match string like : 219.254.232.123 or 232.212.222.3434 (can only go until 232.212.222.343), 
#can not match 232.2122.222.3434 or 232.212.22a.3434

#`(\d+(?:\.\d+)?)` is a group, so result will be put in a list. 

#if you just want to match any int or .xxx or xxxx.xxx,just use

rate = re.findall('(\d+(\.\d+)?)', target)
print(rate)
[('8', ''), ('8.5', '.5')]
#which will return a list to you

#if you just want to find a float, use below: 

target = 'total growth is : 8%-8.5% or 9.546%'
rate = re.findall('(\d+\.\d+)', target)
print(rate)
['8.5', '9.546']

A bit deeper:

A(?=B) ：Match String A ends with String B

Python_mysql

Sun, 07 Feb 2021 13:21:07 +0800

Python Query mysql

To use Python to query mysql DB. There are two popular modules are used:

pymysql
sqlalchemy ( need to install mysqlconnector in my scenario) pip install mysql-connector-python

pymysql:

ekoudb = pymysql.connect(host='10.10.80.127', 
 port=3306, 
 user='ekou', 
 password='abcd12334', 
 database='ekoutest') 
 
 #get cursor 
cursor = ekoudb.cursor()

def insert_a_item(SqlLang): 
 """ 
 Insert items """ 
 try: 
 cursor.execute(SqlLang) 
 ekoudb.commit() 
 except Exception as e: 
 print('Insert data fail') 
 print(e) 
 ekoudb.rollback() 
 # release resource 
 cursor.close() 
 ekoudb.close() 
#insert_a_item(SQL_INSERT_A_ITEM)
SQL_INSERT_MANY_ITEMS = "INSERT INTO bird_table (origin, hostname,descr,ver,protocol,neighboraddr,neighboras,status,sinceday,sincetime) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"

MULTIENTRY = [["update","host1","descr1","ipv6","BGP","2123123","231233","Idle","2021-02-02","16:50:07"],\ 
["update","host2","descr2","ipv6","BGP","ipv6_addr1","65300","Idle","2021-02-02","16:50:07"],\ 
["update","host3","descr3","ipv4","BGP","10.10.1.1","65002","Established","2021-02-02","15:50:13"],\ 
["update","host4","descr4","ipv4","BGP","10.10.1.3","65003","Established","2021-02-02","15:50:13"]]

def insert_items(MANYTIMESFORMAT,SqlLang): 
 try: 
 cursor.executemany(MANYTIMESFORMAT, SqlLang) 
 ekoudb.commit() 
 except Exception as e: 
 print("Insert_error") 
 ekoudb.rollback() 
 # release resource 
 cursor.close() 
 ekoudb.close()

def query(SqlLang): 
 """query""" 
 # query all data 
 cursor.execute(SqlLang) 
 print('+--------+-----------+---------------------------+------+----------+-\ 
-------------------------+------------+-------------+------------+-----------+') 
 print('| origin | hostname | descr | ver | protocol | \ 
 neighboraddr | neighboras | status | sinceday | sincetime |') 
 print('+--------+-----------+---------------------------+------+----------+-\ 
-------------------------+------------+-------------+------------+-----------+') 
 # matrix data 
 rows = cursor.fetchall() 
 for row in rows: 
 origin = row[0] 
 hostname = row[1] 
 descr = row[2] 
 ver = row[3] 
 protocol = row[4] 
 neighboraddr = str(row[5]) 
 neighboras = str(row[6]) 
 status = row[7] 
 sinceday = str(row[7]) 
 sincetime = str(row[9]) 
 
# print('| ' + origin.rjust(2) + '|'.rjust(7) + hostname.rjust(2) + '|'.rjust(10) + descr.rjust(2) + '|'.rjust(26) + ver.rjust(2) + '|'.rjust(5) + protocol.rjust(2) + '|'.rjust(9) + neighboraddr.rjust(2) + + '|'.rjust(25) + neighboras.rjust(2) + '|'.rjust(11) + status.rjust(2) + '|'.rjust(12) + sinceday.rjust(2) + '|'.rjust(11) + sincetime.rjust(2)) 
# print('| ' + origin.rjust(2) + '\t|' + '\t' + hostname + '\t|' + '\t'+ descr + '\t|' + '\t' + ver + '\t|' + '\t' + protocol + '\t|'+'\t'+neighboras + '\t|'+'\t'+neighboraddr + '\t|'+'\t'+status + '\t|'+ '\t' + sinceday + '\t|'+ '\t' +sincetime + '\t|') 
 print('|' + format(origin,"<8") + format("|","<2") + format(hostname,"<10") + format("|","<2") + format(descr,"<26") + format("|","<2") + format(ver,"<5") + format("|","<2") + format(protocol,"<9") + format("|","<2") + format(neighboraddr,"<25") + format("|","<2") + format(neighboras,"<11") + format("|","<2") + format(status,"<12") + format("|","<2") + format(sinceday,"<11") + format("|","<2") + format(sincetime,"<10") + "|" ) 
 print('+--------+-----------+---------------------------+------+----------+-\ 
-------------------------+------------+-------------+------------+-----------+') 
 # release resource 
 cursor.close() 
 ekoudb.close()
 


SQL_UPDATE = "UPDATE bird_table SET hostname='%s',status='%s' WHERE descr='%s'" 
 
def update(SqlLang): 
 """ 
 update 
 :return: """ sql_update = SqlLang % ('hostname3', 'connect', 'descr_1') 
 try: 
 cursor.execute(sql_update) 
 ekoudb.commit() 
 except Exception as e: 
 ekoudb.rollback() 
 print('error when update data') 
 print(e) 
 # release resource 
 cursor.close() 
 ekoudb.close() 
 
update(SQL_UPDATE)
SQL_DELETE = "DELETE FROM bird_table WHERE descr='%s'" 
 
def delete(SqlLang,delete_string): 
 """ 
 delete record 
 :return: """ try: 
 # delete sql 
 sql_del = SqlLang % (delete_string) 
 cursor.execute(sql_del) 
 ekoudb.commit() 
 except Exception as e: 
 # rollback when error 
 ekoudb.rollback() 
 print(e) 
 # release resource 
 cursor.close() 
 ekoudb.close() 
 
delete(SQL_DELETE,'23123213213')

sqlalchemy:

engine = create_engine("mysql+mysqlconnector://ekou:PASSWORD@10.10.80.127:3306/ekoutest?charset=utf8mb4")

metadata = MetaData(engine) 
 
Session = sessionmaker() # create a customized session 
Session.configure(bind=engine) # link to this session 
session = Session()
users_table = Table('bird_table', metadata, autoload=True)

result=users_table.select(and_(users_table.c.descr == "descr", users_table.c.neighboraddr == "NEIGHBOR_ADDR")).execute() 
for item in result.fetchall(): 
 print (item)

''' 
create table 
''' 
users_table = Table('users', 
 metadata, 
 Column('id', Integer, primary_key=True), 
 Column('name', String(40)), 
 Column('email', String(120))) 
if not users_table.exists(): 
 users_table.create() 
 
''' 
load table 
''' 
users_table = Table('users', metadata, autoload=True) 
 
''' 
insert 
''' 
users_table.insert().execute(name="sss", email="eee@eee.com") 
users_table.insert().execute(name="ttt", email="tt@eee.com") 
''' 
update 
''' 
users_table.update(users_table.c.name == "ttt").execute(name="ddd")

conclusion:

for easier get on hand, pymsql would be good enough to use. but for advanced, need to use sqlalchemy.