Packages

Package Characteristics

1. A package is a folder containing a __init__.py file; importing a package means importing the __init__.py file
2. A package is always imported as a module file, and the module search path is based on the execution file path

Absolute vs Relative Imports

Absolute Imports

Based on the root directory of the package

Relative Imports

Based on the directory where the currently imported module resides

• . represents files in the current directory
• .. represents files in the parent directory
• ... represents files in the grandparent directory

Both imports can only be used with in packages

Package Purpose

When extending module functionality, multiple module files are placed within one package (folder) without changing the import mechanism, similar to function decorators

Time and Datetime Modules

import time

# time module: time-related operations
# 1. time.time() --- returns current timestamp, calculated from January 1, 1970, 00:00:00

# 2. formatted time
print(time.strftime('%Y-%m-%d %H:%M:%S'))
print(time.strftime('%Y-%m-%d %X'))
print(time.strftime('%x %X'))

# 3. structured time
print(time.localtime()) # displays nine elements: year/month/day/hour/minute/second/day of week (starting Sunday)/day of year/DST flag


import datetime

# datetime module: date/time arithmetic

current_time = datetime.datetime.now()
print(current_time)  # seconds precise to six decimal places

# time arithmetic
print(current_time + datetime.timedelta(days=3))  # add three days
print(current_time + datetime.timedelta(weeks=3))  # add three weeks
print(current_time + datetime.timedelta(hours=3))  # add three hours

# time replacement
print(current_time.replace(1949, 10, 1, 10, 1, 10, 0)) 

Random Module

import random
import time
# random module: generating random numbers

# random.seed(10)

# common functions
print(random.random())  # random float between (0-1)

print(random.randint(1, 3))  # random integer between [1-3]

print(random.randrange(5))  # random integer in [0,5)

numbers = [1, 2, 3]
random.shuffle(numbers)  # shuffle list randomly
print(numbers)

print(random.choice(numbers))    # select random item from iterable

# random seed, controls single random generation
# random.seed(10)
print(random.random())

random.seed(time.time())    # Mersenne Twister algorithm
print(random.random())

print(random.sample(numbers, 2))  # sampling

Hashlib and Hmac Modules

# hashlib module
import hashlib
hash_obj = hashlib.md5()
hash_obj.update(b'hello')
hash_obj.update(b'world')
print(hash_obj.hexdigest())  # aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d

hash_obj = hashlib.md5()
hash_obj.update(b'helloworld')
print(hash_obj.hexdigest())    # aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d
# Cumulative property: multiple updates with different content yield same result as single update with total content

# Cracking hashlib encryption
    # 1. Create common password dictionary
    # 2. Generate hash values for common passwords through iteration
    # 3. Compare intercepted packet hash values to get correct password
password_list = ['test', 'password', '123456', 'hello']
hashed_password = 'aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d'
for pwd in password_list:
    hash_obj = hashlib.md5()
    hash_obj.update(pwd.encode('utf8'))
    result = hash_obj.hexdigest()
    if result == hashed_password:
        print('Password is:', pwd)

# hmac module: secret key (salt)
import hmac
hash_result = hmac.new(b'secret_key')
hash_result.update(b'user_data')
print(hash_result.hexdigest())    # varies based on key

hash_result = hmac.new(b'different_key')
hash_result.update(b'user_data')
print(hash_result.hexdigest())    # different hmac result with different key
# Same password, different keys produce different hmac results

Typing Module

Used with functions to control parameter data types, providing types beyond basic ones

from typing import List, Dict
def process_data(items: List[str], mapping: Dict[str, int]) -> bool:
    return True

Requests and Re Modules

Requests Module

# requests module: web scraping --> collecting data, simulating browser requests to URLs, retrieving data
# URL: Uniform Resource Locator, never duplicated
import requests
response = requests.get('https://example.com/')
data = response.text

# regex re: extracting strings with specific patterns from larger strings, results stored as lists
import re
# Extract titles using regex
res_titles = re.findall(r'<title>(.*?)</title>', data, re.S)
print(res_titles)

Regular Expressions

# Regular expressions
import re
    # 1. "^..." matches strings starting with ..., "^" itself doesn't consume characters
sample_text = '''xyzxyz1  
23'''
res = re.findall('^xyz', sample_text)
print(res)
res = re.findall('xyz', sample_text)
print(res)
res = re.findall('\^xyz', sample_text) # "\" escapes the following character
print(res)

    # 2. "...$" matches strings ending with ..., "$" itself doesn't consume characters
res = re.findall('yz$', sample_text)
print(res)
res = re.findall('yz', sample_text)
print(res)

    # 3. "." matches any character except "\n"
res = re.findall('x.y.', sample_text)
print(res)

    # 4. "\w" matches word characters: letters, digits, underscore [A-Z,a-z,0-9,'_']
res = re.findall('\w', sample_text)
print(res)

    # 5. "\W" matches non-word characters
res = re.findall('1\W\W\W2', sample_text)
print(res)

    # 6. "\d" matches digit characters
res = re.findall('\d\d', sample_text)
print(res)

    # 7. "\D" matches non-digit characters
res = re.findall('\D', sample_text)
print(res)

    # 8. "\s" matches whitespace
res = re.findall('\s', sample_text)
print(res)

    # 9. "\S" matches non-whitespace
res = re.findall('\S', sample_text)
print(res)

    # 10. "+" matches previous character [1,+∞) times
res = re.findall('xyzz+', 'xyzzzzz xyz xyx')
print(res)

    # 11. "?" matches previous character [0,1] times
res = re.findall('xyz?', 'xyzzzzz xyz xyx')
print(res)

    # 12. "*" matches previous character [0,+∞) times
res = re.findall('xyz*', 'xyzzzzz xyz xyx')
print(res)

    # 13. "[]" matches any character inside brackets
res = re.findall('[xyz]y', 'xyzyyyzyzyzy')
print(res)

    # 14. "[^]" matches any character not inside brackets
res = re.findall('[^xyz]yz', 'xyzyyyy yz')
print(res)

    # 15. "|" matches either left or right string
res = re.findall('yyz|xyz', 'xyz yyz yz')
print(res)

    # 16. "{m,n}" matches previous character [m,n] times
res = re.findall('xyz{1,2}', 'xyzz xyzxyzzz')
print(res)

Greedy vs Non-Greedy Patterns

import re
# Greedy pattern (maximum span): ".*", where ".": any character, "*": [0,+∞) occurrences
text = 'qmnopqrstqqqqqqqt'
print(re.findall('q.*t', text))

# Non-greedy pattern (minimum span): ".*?", where ".": any character, "*": [0,+∞) occurrences, "?": enables non-greedy mode
print(re.findall('q.*?t', text))

Special Constructs

# Special constructs (basic understanding)
# "q(?=\d)" matches "q" followed by digit but excludes the digit, doesn't consume "\d" content
sample = 'q123 qqqq q234 qbc'
print(re.findall('q(?=\d)', sample))
print(re.findall('q(?=\w)', sample))

# Email matching practice
sample = '#@#@#@test@example.com$$$$////test@domain.co.uk$$#$#$[]]1234567890@example.org@$2423@423lksdlf@j#'
results = re.findall('\w+@\w+\.\w+', sample)  # "\w" : letters/digits/underscore
print(results)

Common Re Module Functions

import re
# Common re module functions (beyond "findall")
#1. re.compile
sample = 'pqrs pqrrr pqr'
compiled_regex = re.compile('pqr*')
print(compiled_regex)
print(re.findall(compiled_regex, sample))
    # Practical use
event_pattern = re.compile('\w+@\w+\.\w+')
telephone_pattern = re.compile('\d{10}')

# Next two are very important for interviews
    #2. re.match---stops after first match, must start with specified string
sample = 'pqrs pqrrr pqr'
result = re.match('pqr*', sample)
print(result)
print(result.group())

    #3. re.search---stops after first match, doesn't need to start with specified string
result = re.search('pqr*', sample)
print(result)
print(result.group())

#4. re.split---splits based on specified regex
sample = 'pq12323pqrsss123123pqrasdfjlasjdk123l23lk4j2kl34kl25k3j2kl3j5lkj'
print(re.split('\d+', sample))

#5. re.sub == replace replaces regex pattern with specified characters
sample = 'pq12323pqrsss123123pqrasdfjlasjdk123l23lk4j2kl34kl25k3j2kl3j5lkj'
print(re.sub('\d+', '', sample))

#6. re.subn outputs replacement count in addition to re.sub
sample = 'pq12323pqrsss123123pqrasdfjlasjdk123l23lk4j2kl34kl25k3j2kl3j5lkj'
print(re.subn('\d+', '', sample))

Additional Features

import re
#7. Additional features
# Modifiers
    # re.S---very useful
sample = '''def
defdef*def
'''
print(re.findall('def.def', sample))  # "." doesn't match newline
print(re.findall('def.def', sample, re.S))  # "re.S" allows "." to match newline

#9. Grouping (very important): "()"---extracts content within parentheses, multiple groups store each complete result as tuple
sample = 'def defg dmfgg'
print(re.findall('d(.)f(g)', sample))

    # Named groups (basic knowledge)---usually used with "re.search", assigns name to "()" extracted content, output as dictionary
sample = 'def defg dmfgg'
print(re.search('d(?P<first>.)f(?P<second>g)', sample).groupdict())

    # Named groups with "re.sub", preserves named content from matched string, removes rest
sample = 'def123def123'
print(re.sub('f\d+d', ' ', sample))
print(re.sub('f(?P<number>\d+)d', ' \g<number> ', sample))