NumPy Module

Introduction

NumPy provides efficient array operations for numerical data analysis. It supports matrix computations similar to TensorFlow's tensor operations, which are common in mathematics, computer science, and physics research.

# Element-wise multiplication between two lists
list_a = [1, 2, 3]
list_b = [4, 5, 6]
result = [list_a[i] * list_b[i] for i in range(len(list_a))]
print(result)

One-Dimensional and Three-Dimensional Arrays

import numpy as np
# One-dimensional array - can use standard lists
# Three-dimensional arrays - typically handled by TensorFlow

Two-Dimensional Array Properties

import numpy as np
data = np.array([[1, 2, 3], [4, 5, 6]])

# Transpose - rows become columns
print(data.T)

# Data type of elements
print(data.dtype)

# Total number of elements
print(data.size)

# Number of dimensions
print(data.ndim)

# Shape - returns tuple of (rows, columns)
print(data.shape)
print(data.shape[0])  # row count
print(data.shape[1])  # column count

# Type conversion - NumPy arrays are mutable
data = data.astype(np.float64)
print(data)

Accessing and Modifying Two-Dimensional Arrays

import numpy as np
data = np.array([[1, 2, 3],
                 [4, 5, 6]])

# Single element access
print(data[0, 0])

# Slicing - includes start, excludes end
print(data[0:2, -2:])

# Boolean indexing - returns flattened array
print(data[data > 2])

# Assignment
data[0, 0] = 0
print(data)

data[0:1, :] = 0
print(data)

data[:, :] = 0
print(data)

Combining Arrays

import numpy as np
arr1 = np.array([[1, 2, 3], [4, 5, 6]])
arr2 = np.array([[55, 66, 77], [11, 22, 33]])

# Horizontal stacking
print(np.hstack((arr1, arr2)))

# Vertical stacking
print(np.vstack((arr1, arr2)))

# Concatenation - axis=0 by row, axis=1 by column
print(np.concatenate((arr1, arr2), axis=1))

Creating Arrays with Functions

import numpy as np
# Array of ones
print(np.ones((2, 3)))

# Array of zeros
print(np.zeros((2, 3)))

# Identity matrix
print(np.eye(3, 3))

# Arithmetic sequence with inclusive endpoints
print(np.linspace(0, 20, 5))

# Range of values
print(np.arange(2, 10))

# Reshape array dimensions
matrix = np.zeros((1, 12))
print(matrix.reshape(4, 3))

Array Operations

import numpy as np
# Scalar multiplication
print(np.ones((3, 4)) * 4)
print(np.sin(arr1))

# Matrix multiplication
matrix_a = np.ones((6, 5)) * 3
matrix_b = np.ones((5, 6)) * 4
print(np.dot(matrix_a, matrix_b))

# Matrix inversion
print(np.linalg.inv([[1, 2, 3],
                     [4, 5, 6],
                     [9, 8, 9]]))

Mathematical and Statistical Methods

import numpy as np
arr1 = np.ones((6, 5)) * 3
print(np.sum(arr1))
print(np.sum(arr1[0:1, :]))

Genertaing Random Numbers

# Uniform distribution over [0, 1)
print(np.random.rand(3, 4))

# Standard normal distribution
print(np.random.randn(3, 4))

# Random floats in [0, 1)
print(np.random.random((3, 4)))

# Using RandomState for reproducibility
rng = np.random.RandomState(1)
print(rng.random((3, 4)))

# Shuffle array in place
data = np.array([[55, 66, 77], [11, 22, 33]])
np.random.shuffle(data)
print(data)

# Random selection from 1D array
print(np.random.choice([1, 2, 3, 4, 5], 2))

# Random integers in range
print(np.random.randint(1, 100, (4, 5)))

Matplotlib Module

Bar Charts

from matplotlib import pyplot as plt
from matplotlib.font_manager import FontProperties

font = FontProperties(fname='C:\\Windows\\Fonts\\simsun.ttc')
plt.style.use('ggplot')

classes = ['Class 3', 'Class 4', 'Class 5', 'Class 6']
students = [50, 55, 45, 60]
indices = range(len(classes))

plt.bar(indices, students, color='green')
plt.xticks(indices, classes, FontProperties=font)
plt.xlabel('Class', FontProperties=font)
plt.ylabel('Student Count', FontProperties=font)
plt.title('Class Enrollment Bar Chart', FontProperties=font)
plt.show()

Histograms

import numpy as np
from matplotlib import pyplot as plt
from matplotlib.font_manager import FontProperties

plt.style.use('ggplot')
font = FontProperties(fname='C:\\Windows\\Fonts\\simsun.ttc')

x1 = np.random.randn(10000)
x2 = np.random.randn(10000)

fig = plt.figure()
ax1 = fig.add_subplot(1, 2, 1)
ax2 = fig.add_subplot(1, 2, 2)

ax1.hist(x1, bins=50, color='purple')
ax2.hist(x2, bins=50, color='orange')

ax1.set_title('x1 Normal Distribution', FontProperties=font)
ax2.set_title('x2 Normal Distribution', FontProperties=font)
fig.suptitle('Two Normal Distributions', FontProperties=font)
plt.show()

Line Charts

import numpy as np
from matplotlib import pyplot as plt
from matplotlib.font_manager import FontProperties

plt.style.use('ggplot')
font = FontProperties(fname='C:\\Windows\\Fonts\\simsun.ttc')

# Cumulative sum calculation
arr = np.array([[1, 2, 3]])
print(arr.cumsum())

np.random.seed(1)
x1 = np.random.randn(40).cumsum()
x2 = np.random.randn(40).cumsum()
x3 = np.random.randn(40).cumsum()
x4 = np.random.randn(40).cumsum()

plt.plot(x1, color='blue', linestyle='-', marker='o', label='Blue Circle')
plt.plot(x2, color='pink', linestyle='--', marker='x', label='Pink Dashed')
plt.plot(x3, color='green', linestyle='-.', marker='*', label='Green Star')
plt.plot(x4, color='purple', linestyle=':', marker='s', label='Purple Square')

plt.legend(loc='best', prop=font)
plt.show()

Scatter Plots Combined with Line Charts

import numpy as np
from matplotlib import pyplot as plt
from matplotlib.font_manager import FontProperties

plt.style.use('ggplot')
font = FontProperties(fname='C:\\Windows\\Fonts\\simsun.ttc')

fig = plt.figure()
ax1 = fig.add_subplot(1, 2, 1)
ax2 = fig.add_subplot(1, 2, 2)

x = np.arange(20)
y1 = x * 2
y2 = x ** 2

ax1.scatter(x, y1, c='b', label='Blue Dots')
ax1.plot(x, y2, c='r', label='Red Line')
ax2.plot(x, y1, c='black', label='Black Line')
ax2.scatter(x, y2, c='green', label='Green Dots')

ax1.legend(prop=font)
ax2.legend(prop=font)
ax1.set_title('Scatter Line Chart 1', FontProperties=font)
ax2.set_title('Scatter Line Chart 2', FontProperties=font)
fig.suptitle('Two Charts', FontProperties=font)
plt.show()

Pandas Module

Data Types

Pandas handles Excel and CSV file processing, providing a higher-level interface over NumPy with additional data manipulatoin capabilities.

import pandas as pd
import numpy as np

# Series - one-dimensional (less common now)
series = pd.Series(np.array([1, 2, 3, 4]))
print(series)

# DataFrame - two-dimensional
dates = pd.date_range('20190101', periods=6, freq='M')
np.random.seed(1)
values = np.random.rand(6, 4) * 10
columns = ['c1', 'c2', 'c3', 'c4']
df = pd.DataFrame(values, index=dates, columns=columns)
print(df)

DataFrame Properties

import pandas as pd
import numpy as np

dates = pd.date_range('20190101', periods=6, freq='M')
np.random.seed(1)
values = np.random.rand(6, 4) * 10
columns = ['c1', 'c2', 'c3', 'c4']
df = pd.DataFrame(values, index=dates, columns=columns)

# Column data types
print(df.dtypes)

# Row labels
print(df.index)

# Column labels
print(df.columns)

# Statistical summary
print(df.describe())

# Transpose
print(df.T)

# Sort by column values
print(df.sort_values('c2'))

# Sort by index
print(df.sort_index(ascending=False))

Accessing and Modifying DataFrame

import pandas as pd
import numpy as np

dates = pd.date_range('20190101', periods=6, freq='M')
np.random.seed(1)
values = np.random.rand(6, 4) * 10
columns = ['c1', 'c2', 'c3', 'c4']
df = pd.DataFrame(values, index=dates, columns=columns)

# Select columns by label
print(df[['c1', 'c2']])

# Select rows by custom label range
print(df.loc['2019-03-31':'2019-05-31'])

# Select rows by integer position
print(df[1:4])

# Single value access
print(df.values[0, 0])

# Integer-based indexing
print(df.iloc[0, 0])
print(df.iloc[:, 0])

# Boolean filtering
print(df['c1'] > 4)
print(df[df['c1'] > 4])

# Modify value
df.iloc[0, 0] = 1
print(df)

Working with Tables

import pandas as pd
from io import StringIO

sample_csv = '''
5.1,,1.4,0.2
4.9,3.0,1.4,0.2
4.7,3.2,,0.2
7.0,3.2,4.7,1.4
6.4,3.2,4.5,1.5
6.9,3.1,4.9,
'''

buffer = StringIO(sample_csv)
df = pd.read_csv(buffer, header=None)
print(df)

# Handle missing values
print(df.dropna(axis=1))  # Drop columns with missing data
print(df.dropna(axis=0))  # Drop rows with missing data
print(df.dropna(thresh=4))  # Keep rows with at least 4 non-null values

# Merge data
df2 = pd.DataFrame(np.zeros((2, 3)))
df3 = pd.DataFrame(np.ones((2, 3)))
print(df2)
print(df3)
print(pd.concat((df2, df3)))  # Concatenate along rows
print(pd.concat((df2, df3), axis=1))  # Concatenate along columns
print(df2.append(df3, ignore_index=True))  # Append rows

# Export data
df.index = ['nick', 'json', 'tank', 'egon', 'alex', 'cai']
df.to_csv('output.csv')

Reading JSON Files

import pandas as pd

json_data = '[{"lottery":"min","issue":"20130801-3391","code":"8,4,5,2,9","code1":"297734529","code2":null,"time":1013395466000},'
json_data += '{"lottery":"min","issue":"20130801-3390","code":"7,8,2,1,2","code1":"298058212","code2":null,"time":1013395406000},'
json_data += '{"lottery":"min","issue":"20130801-3389","code":"5,9,1,2,9","code1":"298329129","code2":null,"time":1013395346000},'
json_data += '{"lottery":"min","issue":"20130801-3388","code":"3,8,7,3,3","code1":"298588733","code2":null,"time":1013395286000},'
json_data += '{"lottery":"min","issue":"20130801-3387","code":"0,8,5,2,7","code1":"298818527","code2":null,"time":1013395226000}]'

print(pd.read_json(json_data, orient='records'))

Collections Module

Named Tuples

from collections import namedtuple

Point = namedtuple('point', ['x', 'y'])
p = Point(1, 2)
print(p.x)
print(p.y)

DefaultDict

from collections import defaultdict

# Returns default value for missing keys
word_count = defaultdict(lambda: 'not_found')
word_count['a'] = 1
print(word_count['a'])
print(word_count['c'])

Deque

from collections import deque

# Double-ended queue for efficient append/pop at both ends
dq = deque([1, 2, 3])
dq.append(4)
print(dq)
dq.appendleft(0)
print(dq)
dq.popleft()
dq.popleft()
print(dq)

Counter

from collections import Counter

text = 'programming'

# Count character occurrences
char_counter = Counter()
for ch in text:
    char_counter[ch] += 1
print(char_counter)