Environment Setup

Establish the necessary development environment before proceeding. Ensure Python 3.x is installed. The following packages are required for data retrieval, manipulation, visualization, and modeling:

• requests: Fetches web-based data sources.
• pandas: Handles tabular data operations and cleaning.
• matplotlib: Generates standard plots.
• seaborn: Provides statistical data visualizations.
• scikit-learn: Implements machine learning algorithms.

Installation can be performed via pip:

pip install requests pandas matplotlib seaborn scikit-learn joblib

Data Acquisition

The project relies on historical records to build a regression model. The target variable is the Air Quality Index (AQI). Data is collected from an external weather service covering a full year of observations. A Python script iterates through months, constructs URLs dynamically, retrieves HTML tables, and appends them to a consolidated dataset file.

import requests
import pandas as pd
from pathlib import Path
import warnings

warnings.filterwarnings("ignore")

def fetch_air_quality_data(output_file: str = "raw_aqi.csv"):
    base_url = "http://www.tianqihoubao.com/aqi/changsha-2024{}"
    df_list = []

    for month in range(1, 13):
        padded_month = str(month).zfill(2)
        url = base_url.format(padded_month)
        
        try:
            response = requests.get(url)
            html_content = response.text
            raw_tables = pd.read_html(html_content, encoding="utf-8")
            
            if raw_tables:
                monthly_df = raw_tables[0]
                # Skip header row if present in parsing
                if len(monthly_df) > 1:
                    df_list.append(monthly_df.iloc[1:])
        except Exception as e:
            print(f"Failed to fetch {month}: {e}")
            continue

    combined_df = pd.concat(df_list, ignore_index=True)
    combined_df.to_csv(output_file, index=False)
    return output_file

if __name__ == "__main__":
    file_path = fetch_air_quality_data()
    print(f"Data saved to {Path(file_path).resolve()}")

Feature Engineering

Raw data requires cleaning and feature transformation. Duplicates must be removed, and temporal indices should be created. More important, lag features are essential for time-series forecasting, representing the values of pollutants from previous days to help predict the current day's AQI.

import pandas as pd
from datetime import datetime

INPUT_FILE = "raw_aqi.csv"
OUTPUT_FILE = "cleaned_dataset.csv"
POLLUTANTS = ["PM2.5", "PM10", "So2", "No2", "O3", "Co"]
LAG_DAYS = [1, 2]

def preprocess_data(source: str, target: str):
    df = pd.read_csv(source)
    
    # Convert date column and add time features
    df["Date"] = pd.to_datetime(df["Date"])
    df["Month"] = df["Date"].dt.month
    df["Day"] = df["Date"].dt.day
    df["Weekday"] = df["Date"].dt.weekday
    
    # Remove duplicate entries
    df.drop_duplicates(inplace=True)
    
    # Generate lag features for all pollutants and the target AQI
    for col in ["AQI指数"] + POLLUTANTS:
        if col not in df.columns:
            continue
        for lag in LAG_DAYS:
            new_col_name = f"{col}_{lag}d_lag"
            df[new_col_name] = df[col].shift(lag)
            
    # Drop rows where lags result in NaN
    df.dropna(inplace=True)
    
    df.to_csv(target, index=False)
    return df

processed_df = preprocess_data(INPUT_FILE, OUTPUT_FILE)
print(f"Processed shape: {processed_df.shape}")

Exploratory Data Analysis

Before training, it is crucial to understand relationships between variables. Correlation matrices identify strong dependencies, while visualizations reveal seasonal trends and distribution outliers. These insights guide feature selection.

import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

FILE_PATH = "cleaned_dataset.csv"
data = pd.read_csv(FILE_PATH)

# Set font handling for compatibility
plt.rcParams["font.sans-serif"] = ["Arial"] 
plt.rcParams["axes.unicode_minus"] = False

numeric_cols = data.select_dtypes(include=["number"]).columns
corr_matrix = data[numeric_cols].corr()

# Identify top correlations with target
target_corr = corr_matrix.loc[:, "AQI指数"].sort_values(ascending=False)
print("Correlations with AQI:\n", target_corr)

fig, axes = plt.subplots(2, 2, figsize=(14, 10))

# Trend over time
axes[0, 0].plot(data["Date"], data["AQI指数"])
axes[0, 0].set_title("AQI Temporal Trend")
axes[0, 0].set_xlabel("Date")
axes[0, 0].set_ylabel("Index Value")

# Distribution per month
sns.boxplot(x="Month", y="AQI指数", data=data, ax=axes[0, 1])
axes[0, 1].set_title("Monthly AQI Distribution")

# Heatmap of correlation
tick_locs = range(0, len(corr_matrix.columns), len(corr_matrix.columns)//5)
ax_heat = axes[1, 0]
sns.heatmap(corr_matrix, cmap="RdYlGn_r", annot=True, fmt=".1f", ax=ax_heat, square=True)
ax_heat.set_title("Feature Correlation Matrix")

# Save plot
plt.tight_layout()
plt.savefig("eda_results.png")
plt.close()

Model Training

A Random Forest Regressor is chosen for its robustness against overfitting and ability to handle non-linear relationships. The dataset is split into training and testing sets to evaluate generalization performance. Hyperparameters are set deterministically to ensure reproducibility.

from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_absolute_error
import joblib

DATA_SOURCE = "cleaned_dataset.csv"
MODEL_SAVE_PATH = "aqi_random_forest.joblib"

X_columns = [
    "AQI指数_1d_lag", "PM2.5_1d_lag", "PM10_1d_lag",
    "So2_1d_lag", "No2_1d_lag", "O3_1d_lag", "Co_1d_lag"
]
y_column = "AQI指数"

df = pd.read_csv(DATA_SOURCE)

X = df[X_columns]
y = df[y_column]

train_X, test_X, train_y, test_y = train_test_split(
    X, y, test_size=0.2, random_state=42
)

model = RandomForestRegressor(
    n_estimators=100,
    max_depth=10,
    random_state=42,
    verbose=0
)

model.fit(train_X, train_y)

y_pred = model.predict(test_X)
mape = (abs(y_pred - test_y) / test_y).mean() * 100
accuracy_within_30 = (abs(y_pred - test_y) <= 30).mean()

print(f"Mean Absolute Error: {mape:.2f}%")
print(f"Accuracy (error < 30): {accuracy_within_30:.2%}")

joblib.dump(model, MODEL_SAVE_PATH)
print(f"Model persisted at {MODEL_SAVE_PATH}")

Inference Interface

To utilize the trained model for future predictions, load the serialized artifact and accept user input for the relevant lagged features. This interface allows the system to forecast air quality based on recent conditions.

import joblib
import pandas as pd

def run_prediction():
    model_path = "aqi_random_forest.joblib"
    model = joblib.load(model_path)
    
    print("Enter lagged feature values for prediction:")
    print("Format: Float number")
    
    inputs = {}
    columns = [
        'AQI_1d_lag', 'PM2.5_1d_lag', 'PM10_1d_lag',
        'So2_1d_lag', 'No2_1d_lag', 'O3_1d_lag', 'Co_1d_lag'
    ]
    
    for col in columns:
        val = float(input(f"> Enter value for {col}: "))
        inputs[col] = val
    
    input_df = pd.DataFrame([inputs])
    predicted_value = model.predict(input_df)[0]
    
    print(f"\nForecasted AQI Index: {predicted_value:.2f}")
    print("End of prediction session.")

if __name__ == "__main__":
    run_prediction()