当前位置：首页 > news >正文

Python 数据科学实战：从数据到洞察

news 2026/5/7 20:18:50

Python 数据科学实战：从数据到洞察

数据科学的重要性

数据科学是当今最热门的领域之一，它结合了统计学、计算机科学和领域知识，通过分析数据来提取有价值的洞察。Python作为一种功能强大的编程语言，在数据科学领域有着广泛的应用。本文将介绍Python数据科学的核心概念、常用库和最佳实践。

基本概念

数据类型

数据科学中常见的数据类型包括：

结构化数据：如表格数据（CSV、Excel）
非结构化数据：如文本、图像、音频
半结构化数据：如JSON、XML

数据处理流程

数据科学的典型流程包括：

数据收集：获取原始数据
数据清洗：处理缺失值、异常值
数据探索：了解数据的基本特征
特征工程：提取有用的特征
模型构建：训练机器学习模型
模型评估：评估模型性能
模型部署：将模型应用到实际场景

常用库

NumPy

NumPy是Python的数值计算库，它提供了高效的数组操作和数学函数。

import numpy as np # 创建数组 arr = np.array([1, 2, 3, 4, 5]) print(arr) # 数组运算 arr2 = arr * 2 print(arr2) # 矩阵运算 matrix = np.array([[1, 2], [3, 4]]) matrix2 = np.array([[5, 6], [7, 8]]) result = np.dot(matrix, matrix2) print(result) # 统计函数 mean = np.mean(arr) std = np.std(arr) print(f"均值: {mean}, 标准差: {std}")

Pandas

Pandas是Python的数据分析库，它提供了数据结构和数据分析工具。

import pandas as pd # 创建DataFrame data = { 'name': ['Alice', 'Bob', 'Charlie'], 'age': [25, 30, 35], 'city': ['New York', 'London', 'Paris'] } df = pd.DataFrame(data) print(df) # 读取CSV文件 df = pd.read_csv('data.csv') # 基本操作 print(df.head()) # 查看前几行 print(df.describe()) # 统计描述 print(df.info()) # 查看数据信息 # 数据过滤 filtered_df = df[df['age'] > 30] print(filtered_df) # 数据分组 grouped = df.groupby('city').mean() print(grouped) # 数据合并 df1 = pd.DataFrame({'id': [1, 2, 3], 'name': ['Alice', 'Bob', 'Charlie']}) df2 = pd.DataFrame({'id': [1, 2, 3], 'age': [25, 30, 35]}) merged_df = pd.merge(df1, df2, on='id') print(merged_df)

Matplotlib

Matplotlib是Python的可视化库，它提供了各种绘图功能。

import matplotlib.pyplot as plt import numpy as np # 折线图 x = np.linspace(0, 10, 100) y = np.sin(x) plt.plot(x, y) plt.title('Sin Function') plt.xlabel('x') plt.ylabel('y') plt.show() # 散点图 x = np.random.randn(100) y = np.random.randn(100) plt.scatter(x, y) plt.title('Scatter Plot') plt.xlabel('x') plt.ylabel('y') plt.show() # 直方图 data = np.random.randn(1000) plt.hist(data, bins=30) plt.title('Histogram') plt.xlabel('Value') plt.ylabel('Frequency') plt.show() # 条形图 categories = ['A', 'B', 'C', 'D'] values = [10, 20, 15, 25] plt.bar(categories, values) plt.title('Bar Chart') plt.xlabel('Category') plt.ylabel('Value') plt.show()

Seaborn

Seaborn是基于Matplotlib的高级可视化库，它提供了更美观的绘图风格和更多的可视化类型。

import seaborn as sns import pandas as pd import numpy as np # 加载示例数据 df = sns.load_dataset('iris') # 散点图矩阵 sns.pairplot(df, hue='species') plt.title('Pair Plot') plt.show() # 箱线图 sns.boxplot(x='species', y='sepal_length', data=df) plt.title('Box Plot') plt.show() # 热图 corr = df.corr() sns.heatmap(corr, annot=True, cmap='coolwarm') plt.title('Correlation Heatmap') plt.show() # 小提琴图 sns.violinplot(x='species', y='sepal_length', data=df) plt.title('Violin Plot') plt.show()

Scikit-learn

Scikit-learn是Python的机器学习库，它提供了各种机器学习算法和工具。

from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler from sklearn.linear_model import LogisticRegression from sklearn.metrics import accuracy_score, confusion_matrix # 加载数据 data = load_iris() X = data.data y = data.target # 数据分割 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) # 数据标准化 scaler = StandardScaler() X_train_scaled = scaler.fit_transform(X_train) X_test_scaled = scaler.transform(X_test) # 模型训练 model = LogisticRegression() model.fit(X_train_scaled, y_train) # 模型预测 y_pred = model.predict(X_test_scaled) # 模型评估 accuracy = accuracy_score(y_test, y_pred) conf_matrix = confusion_matrix(y_test, y_pred) print(f"准确率: {accuracy}") print(f"混淆矩阵:\n{conf_matrix}")

数据清洗

处理缺失值

import pandas as pd import numpy as np # 创建包含缺失值的数据 data = { 'name': ['Alice', 'Bob', 'Charlie', 'David'], 'age': [25, np.nan, 35, 40], 'city': ['New York', 'London', np.nan, 'Paris'] } df = pd.DataFrame(data) print(df) # 检查缺失值 print(df.isnull()) print(df.isnull().sum()) # 删除包含缺失值的行 df_cleaned = df.dropna() print(df_cleaned) # 填充缺失值 df_filled = df.fillna({ 'age': df['age'].mean(), 'city': 'Unknown' }) print(df_filled) # 前向填充 df_forward = df.fillna(method='ffill') print(df_forward) # 后向填充 df_backward = df.fillna(method='bfill') print(df_backward)

处理异常值

import pandas as pd import numpy as np import matplotlib.pyplot as plt # 创建包含异常值的数据 np.random.seed(42) data = np.random.normal(100, 10, 100) data[0] = 1000 # 添加异常值 # 绘制箱线图 plt.boxplot(data) plt.title('Box Plot with Outlier') plt.show() # 使用IQR方法检测异常值 Q1 = np.percentile(data, 25) Q3 = np.percentile(data, 75) IQR = Q3 - Q1 lower_bound = Q1 - 1.5 * IQR upper_bound = Q3 + 1.5 * IQR outliers = data[(data < lower_bound) | (data > upper_bound)] print(f"异常值: {outliers}") # 处理异常值 # 方法1：删除异常值 cleaned_data = data[(data >= lower_bound) & (data <= upper_bound)] # 方法2：替换异常值为边界值 data_clipped = np.clip(data, lower_bound, upper_bound) # 绘制处理后的箱线图 plt.boxplot(data_clipped) plt.title('Box Plot without Outlier') plt.show()

特征工程

特征选择

from sklearn.datasets import load_breast_cancer from sklearn.feature_selection import SelectKBest, f_classif from sklearn.model_selection import train_test_split # 加载数据 data = load_breast_cancer() X = data.data y = data.target # 特征选择 selector = SelectKBest(f_classif, k=10) X_new = selector.fit_transform(X, y) # 查看选择的特征 selected_features = data.feature_names[selector.get_support()] print(f"选择的特征: {selected_features}") # 数据分割 X_train, X_test, y_train, y_test = train_test_split(X_new, y, test_size=0.2, random_state=42)

特征转换

import pandas as pd from sklearn.preprocessing import LabelEncoder, OneHotEncoder, StandardScaler, MinMaxScaler # 创建示例数据 data = { 'name': ['Alice', 'Bob', 'Charlie'], 'age': [25, 30, 35], 'city': ['New York', 'London', 'Paris'], 'salary': [50000, 60000, 70000] } df = pd.DataFrame(data) # 标签编码 le = LabelEncoder() df['city_encoded'] = le.fit_transform(df['city']) print(df) # 独热编码 one_hot = pd.get_dummies(df['city']) df = pd.concat([df, one_hot], axis=1) print(df) # 标准化 scaler = StandardScaler() df['salary_standardized'] = scaler.fit_transform(df[['salary']]) print(df) # 归一化 min_max_scaler = MinMaxScaler() df['salary_normalized'] = min_max_scaler.fit_transform(df[['salary']]) print(df)

机器学习模型

监督学习

分类模型

from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler from sklearn.linear_model import LogisticRegression from sklearn.tree import DecisionTreeClassifier from sklearn.ensemble import RandomForestClassifier from sklearn.svm import SVC from sklearn.metrics import accuracy_score, classification_report # 加载数据 data = load_iris() X = data.data y = data.target # 数据分割 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) # 数据标准化 scaler = StandardScaler() X_train_scaled = scaler.fit_transform(X_train) X_test_scaled = scaler.transform(X_test) # 逻辑回归 lr = LogisticRegression() lr.fit(X_train_scaled, y_train) y_pred_lr = lr.predict(X_test_scaled) print(f"逻辑回归准确率: {accuracy_score(y_test, y_pred_lr)}") print(classification_report(y_test, y_pred_lr)) # 决策树 dt = DecisionTreeClassifier() dt.fit(X_train, y_train) y_pred_dt = dt.predict(X_test) print(f"决策树准确率: {accuracy_score(y_test, y_pred_dt)}") print(classification_report(y_test, y_pred_dt)) # 随机森林 rf = RandomForestClassifier() rf.fit(X_train, y_train) y_pred_rf = rf.predict(X_test) print(f"随机森林准确率: {accuracy_score(y_test, y_pred_rf)}") print(classification_report(y_test, y_pred_rf)) # 支持向量机 svm = SVC() svm.fit(X_train_scaled, y_train) y_pred_svm = svm.predict(X_test_scaled) print(f"支持向量机准确率: {accuracy_score(y_test, y_pred_svm)}") print(classification_report(y_test, y_pred_svm))

回归模型

from sklearn.datasets import load_boston from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler from sklearn.linear_model import LinearRegression, Ridge, Lasso from sklearn.tree import DecisionTreeRegressor from sklearn.ensemble import RandomForestRegressor from sklearn.metrics import mean_squared_error, r2_score # 加载数据 from sklearn.datasets import fetch_california_housing data = fetch_california_housing() X = data.data y = data.target # 数据分割 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) # 数据标准化 scaler = StandardScaler() X_train_scaled = scaler.fit_transform(X_train) X_test_scaled = scaler.transform(X_test) # 线性回归 lr = LinearRegression() lr.fit(X_train_scaled, y_train) y_pred_lr = lr.predict(X_test_scaled) print(f"线性回归 MSE: {mean_squared_error(y_test, y_pred_lr)}") print(f"线性回归 R²: {r2_score(y_test, y_pred_lr)}") # Ridge回归 ridge = Ridge() ridge.fit(X_train_scaled, y_train) y_pred_ridge = ridge.predict(X_test_scaled) print(f"Ridge回归 MSE: {mean_squared_error(y_test, y_pred_ridge)}") print(f"Ridge回归 R²: {r2_score(y_test, y_pred_ridge)}") # Lasso回归 lasso = Lasso() lasso.fit(X_train_scaled, y_train) y_pred_lasso = lasso.predict(X_test_scaled) print(f"Lasso回归 MSE: {mean_squared_error(y_test, y_pred_lasso)}") print(f"Lasso回归 R²: {r2_score(y_test, y_pred_lasso)}") # 决策树回归 dt = DecisionTreeRegressor() dt.fit(X_train, y_train) y_pred_dt = dt.predict(X_test) print(f"决策树回归 MSE: {mean_squared_error(y_test, y_pred_dt)}") print(f"决策树回归 R²: {r2_score(y_test, y_pred_dt)}") # 随机森林回归 rf = RandomForestRegressor() rf.fit(X_train, y_train) y_pred_rf = rf.predict(X_test) print(f"随机森林回归 MSE: {mean_squared_error(y_test, y_pred_rf)}") print(f"随机森林回归 R²: {r2_score(y_test, y_pred_rf)}")

无监督学习

聚类

from sklearn.datasets import load_iris from sklearn.cluster import KMeans, DBSCAN from sklearn.preprocessing import StandardScaler import matplotlib.pyplot as plt # 加载数据 data = load_iris() X = data.data # 数据标准化 scaler = StandardScaler() X_scaled = scaler.fit_transform(X) # K-means聚类 kmeans = KMeans(n_clusters=3, random_state=42) y_kmeans = kmeans.fit_predict(X_scaled) # 可视化聚类结果 plt.scatter(X_scaled[:, 0], X_scaled[:, 1], c=y_kmeans, cmap='viridis') plt.scatter(kmeans.cluster_centers_[:, 0], kmeans.cluster_centers_[:, 1], s=300, c='red', marker='X') plt.title('K-means Clustering') plt.show() # DBSCAN聚类 dbscan = DBSCAN(eps=0.5, min_samples=5) y_dbscan = dbscan.fit_predict(X_scaled) # 可视化聚类结果 plt.scatter(X_scaled[:, 0], X_scaled[:, 1], c=y_dbscan, cmap='viridis') plt.title('DBSCAN Clustering') plt.show()

降维

from sklearn.datasets import load_iris from sklearn.decomposition import PCA, t_SNE from sklearn.preprocessing import StandardScaler import matplotlib.pyplot as plt # 加载数据 data = load_iris() X = data.data y = data.target # 数据标准化 scaler = StandardScaler() X_scaled = scaler.fit_transform(X) # PCA降维 pca = PCA(n_components=2) X_pca = pca.fit_transform(X_scaled) # 可视化降维结果 plt.scatter(X_pca[:, 0], X_pca[:, 1], c=y, cmap='viridis') plt.title('PCA Dimensionality Reduction') plt.show() # t-SNE降维 tsne = t_SNE(n_components=2, random_state=42) X_tsne = tsne.fit_transform(X_scaled) # 可视化降维结果 plt.scatter(X_tsne[:, 0], X_tsne[:, 1], c=y, cmap='viridis') plt.title('t-SNE Dimensionality Reduction') plt.show()

实用应用

房价预测

import pandas as pd from sklearn.datasets import fetch_california_housing from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler from sklearn.ensemble import RandomForestRegressor from sklearn.metrics import mean_squared_error, r2_score # 加载数据 data = fetch_california_housing() X = data.data y = data.target # 创建DataFrame df = pd.DataFrame(X, columns=data.feature_names) df['target'] = y # 数据探索 print(df.head()) print(df.describe()) # 数据分割 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) # 数据标准化 scaler = StandardScaler() X_train_scaled = scaler.fit_transform(X_train) X_test_scaled = scaler.transform(X_test) # 模型训练 model = RandomForestRegressor(n_estimators=100, random_state=42) model.fit(X_train_scaled, y_train) # 模型预测 y_pred = model.predict(X_test_scaled) # 模型评估 mse = mean_squared_error(y_test, y_pred) r2 = r2_score(y_test, y_pred) print(f"MSE: {mse}") print(f"R²: {r2}") # 特征重要性 feature_importance = pd.DataFrame({ 'feature': data.feature_names, 'importance': model.feature_importances_ }).sort_values('importance', ascending=False) print(feature_importance)

客户 churn 预测

import pandas as pd from sklearn.model_selection import train_test_split from sklearn.preprocessing import StandardScaler, LabelEncoder from sklearn.ensemble import RandomForestClassifier from sklearn.metrics import accuracy_score, confusion_matrix, classification_report # 加载数据（假设数据存在于csv文件中） df = pd.read_csv('customer_churn.csv') # 数据预处理 # 处理缺失值 df = df.dropna() # 标签编码 le = LabelEncoder() df['gender'] = le.fit_transform(df['gender']) df['Partner'] = le.fit_transform(df['Partner']) df['Dependents'] = le.fit_transform(df['Dependents']) df['PhoneService'] = le.fit_transform(df['PhoneService']) df['InternetService'] = le.fit_transform(df['InternetService']) df['Contract'] = le.fit_transform(df['Contract']) df['PaperlessBilling'] = le.fit_transform(df['PaperlessBilling']) df['PaymentMethod'] = le.fit_transform(df['PaymentMethod']) df['Churn'] = le.fit_transform(df['Churn']) # 特征和目标变量 X = df.drop('Churn', axis=1) y = df['Churn'] # 数据分割 X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) # 数据标准化 scaler = StandardScaler() X_train_scaled = scaler.fit_transform(X_train) X_test_scaled = scaler.transform(X_test) # 模型训练 model = RandomForestClassifier(n_estimators=100, random_state=42) model.fit(X_train_scaled, y_train) # 模型预测 y_pred = model.predict(X_test_scaled) # 模型评估 accuracy = accuracy_score(y_test, y_pred) conf_matrix = confusion_matrix(y_test, y_pred) class_report = classification_report(y_test, y_pred) print(f"准确率: {accuracy}") print(f"混淆矩阵:\n{conf_matrix}") print(f"分类报告:\n{class_report}") # 特征重要性 feature_importance = pd.DataFrame({ 'feature': X.columns, 'importance': model.feature_importances_ }).sort_values('importance', ascending=False) print(feature_importance)