mirror of
https://github.com/fenago/data-science.git
synced 2026-10-28 09:34:48 +00:00
208 KiB
208 KiB
In [1]:
# Import necessary modules
%matplotlib inline
import matplotlib as mpl
import seaborn as sns
import matplotlib.pyplot as plt
import statsmodels.formula.api as smf
import statsmodels.graphics.api as smg
import pandas as pd
import numpy as np
import patsy
from statsmodels.graphics.correlation import plot_corr
from sklearn.model_selection import train_test_split
plt.style.use('seaborn')In [2]:
# Load the dataset into a pandas dataframe
rawBostonData = pd.read_csv('https://raw.githubusercontent.com/PacktWorkshops/The-Data-Science-Workshop/master/Chapter02/Dataset/Boston.csv')In [3]:
# Inspect the dataframe
rawBostonData.head() Out [3]:
| CRIM | ZN | INDUS | CHAS | NOX | RM | AGE | DIS | RAD | TAX | PTRATIO | LSTAT | MEDV | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | 0.00632 | 18.0 | 2.31 | 0 | 0.538 | 6.575 | 65.2 | 4.0900 | 1 | 296 | 15.3 | 4.98 | 24.0 |
| 1 | 0.02731 | 0.0 | 7.07 | 0 | 0.469 | 6.421 | 78.9 | 4.9671 | 2 | 242 | 17.8 | 9.14 | 21.6 |
| 2 | 0.02729 | 0.0 | 7.07 | 0 | 0.469 | 7.185 | 61.1 | 4.9671 | 2 | 242 | 17.8 | 4.03 | 34.7 |
| 3 | 0.03237 | 0.0 | 2.18 | 0 | 0.458 | 6.998 | 45.8 | 6.0622 | 3 | 222 | 18.7 | 2.94 | 33.4 |
| 4 | 0.06905 | 0.0 | 2.18 | 0 | 0.458 | 7.147 | 54.2 | 6.0622 | 3 | 222 | 18.7 | 5.33 | 36.2 |
In [4]:
# Drop missing values from the dataframe
rawBostonData = rawBostonData.dropna()In [5]:
# Drop duplicate records from the dataframe
rawBostonData = rawBostonData.drop_duplicates()In [6]:
# List the dataframe column names
list(rawBostonData.columns)Out [6]:
['CRIM', ' ZN ', 'INDUS ', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX', 'PTRATIO', 'LSTAT', 'MEDV']
In [7]:
# Rename the dataframe column headings
renamedBostonData = rawBostonData.rename(columns = {'CRIM':'crimeRatePerCapita',
' ZN ':'landOver25K_sqft',
'INDUS ':'non-retailLandProptn',
'CHAS':'riverDummy',
'NOX':'nitrixOxide_pp10m',
'RM':'AvgNo.RoomsPerDwelling',
'AGE':'ProptnOwnerOccupied',
'DIS':'weightedDist',
'RAD':'radialHighwaysAccess',
'TAX':'propTaxRate_per10K',
'PTRATIO':'pupilTeacherRatio',
'LSTAT':'pctLowerStatus',
'MEDV':'medianValue_Ks'})In [8]:
# Inspect the types of data in the dataframe
renamedBostonData.info()<class 'pandas.core.frame.DataFrame'> Int64Index: 506 entries, 0 to 505 Data columns (total 13 columns): crimeRatePerCapita 506 non-null float64 landOver25K_sqft 506 non-null float64 non-retailLandProptn 506 non-null float64 riverDummy 506 non-null int64 nitrixOxide_pp10m 506 non-null float64 AvgNo.RoomsPerDwelling 506 non-null float64 ProptnOwnerOccupied 506 non-null float64 weightedDist 506 non-null float64 radialHighwaysAccess 506 non-null int64 propTaxRate_per10K 506 non-null int64 pupilTeacherRatio 506 non-null float64 pctLowerStatus 506 non-null float64 medianValue_Ks 506 non-null float64 dtypes: float64(10), int64(3) memory usage: 55.3 KB
In [9]:
# Calculate basic statistics using the values in the dataframe
renamedBostonData.describe(include=[np.number]).TOut [9]:
| count | mean | std | min | 25% | 50% | 75% | max | |
|---|---|---|---|---|---|---|---|---|
| crimeRatePerCapita | 506.0 | 3.613524 | 8.601545 | 0.00632 | 0.082045 | 0.25651 | 3.677082 | 88.9762 |
| landOver25K_sqft | 506.0 | 11.363636 | 23.322453 | 0.00000 | 0.000000 | 0.00000 | 12.500000 | 100.0000 |
| non-retailLandProptn | 506.0 | 11.136779 | 6.860353 | 0.46000 | 5.190000 | 9.69000 | 18.100000 | 27.7400 |
| riverDummy | 506.0 | 0.069170 | 0.253994 | 0.00000 | 0.000000 | 0.00000 | 0.000000 | 1.0000 |
| nitrixOxide_pp10m | 506.0 | 0.554695 | 0.115878 | 0.38500 | 0.449000 | 0.53800 | 0.624000 | 0.8710 |
| AvgNo.RoomsPerDwelling | 506.0 | 6.284634 | 0.702617 | 3.56100 | 5.885500 | 6.20850 | 6.623500 | 8.7800 |
| ProptnOwnerOccupied | 506.0 | 68.574901 | 28.148861 | 2.90000 | 45.025000 | 77.50000 | 94.075000 | 100.0000 |
| weightedDist | 506.0 | 3.795043 | 2.105710 | 1.12960 | 2.100175 | 3.20745 | 5.188425 | 12.1265 |
| radialHighwaysAccess | 506.0 | 9.549407 | 8.707259 | 1.00000 | 4.000000 | 5.00000 | 24.000000 | 24.0000 |
| propTaxRate_per10K | 506.0 | 408.237154 | 168.537116 | 187.00000 | 279.000000 | 330.00000 | 666.000000 | 711.0000 |
| pupilTeacherRatio | 506.0 | 18.455534 | 2.164946 | 12.60000 | 17.400000 | 19.05000 | 20.200000 | 22.0000 |
| pctLowerStatus | 506.0 | 12.653063 | 7.141062 | 1.73000 | 6.950000 | 11.36000 | 16.955000 | 37.9700 |
| medianValue_Ks | 506.0 | 22.532806 | 9.197104 | 5.00000 | 17.025000 | 21.20000 | 25.000000 | 50.0000 |
In [10]:
# Split the data set into training and test sets
X = renamedBostonData.drop('crimeRatePerCapita', axis = 1)
y = renamedBostonData[['crimeRatePerCapita']]
seed = 10
test_data_size = 0.3
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = test_data_size, random_state = seed)
train_data = pd.concat([X_train, y_train], axis = 1)
test_data = pd.concat([X_test, y_test], axis = 1)In [11]:
# Create and plot a correlation matrix
corrMatrix = train_data.corr(method = 'pearson')
xnames=list(train_data.columns)
ynames=list(train_data.columns)
plot_corr(corrMatrix, xnames=xnames, ynames=ynames,\
title=None, normcolor=False, cmap='RdYlBu_r')Out [11]:
In [12]:
# Use the seaborn function regplot to create a scatter plot and fit a regression line through it
fig, ax = plt.subplots(figsize=(10, 6))
sns.regplot(x='medianValue_Ks', y='crimeRatePerCapita', ci=None,
data=train_data, ax=ax, color='k', scatter_kws={"s": 20,"color":\
"royalblue", "alpha":1})
ax.set_ylabel('Crime rate per Capita', fontsize=15, fontname='DejaVu Sans')
ax.set_xlabel("Median value of owner-occupied homes in $1000's",\
fontsize=15, fontname='DejaVu Sans')
ax.set_xlim(left=None, right=None)
ax.set_ylim(bottom=None, top=30)
ax.tick_params(axis='both', which='major', labelsize=12)
fig.tight_layout()
In [13]:
# Use the seaborn function regplot to create a log-linear plot and fit a regression line through it
fig, ax = plt.subplots(figsize=(10, 6))
y = np.log(train_data['crimeRatePerCapita'])
sns.regplot(x='medianValue_Ks', y=y, ci=95, data=train_data, ax=ax,\
color='k', scatter_kws={"s": 20,"color": "royalblue", "alpha":1})
ax.set_ylabel('log of Crime rate per Capita', fontsize=15,\
fontname='DejaVu Sans')
ax.set_xlabel("Median value of owner-occupied homes in $1000's",\
fontsize=15, fontname='DejaVu Sans')
ax.set_xlim(left=None, right=None)
ax.set_ylim(bottom=None, top=None)
ax.tick_params(axis='both', which='major', labelsize=12)
fig.tight_layout()
In [14]:
# Use the statsmodels API to create a simple linear regression
linearModel = smf.ols(formula='crimeRatePerCapita ~ medianValue_Ks',\
data=train_data)
linearModelResult = linearModel.fit()
print(linearModelResult.summary()) OLS Regression Results
==============================================================================
Dep. Variable: crimeRatePerCapita R-squared: 0.144
Model: OLS Adj. R-squared: 0.141
Method: Least Squares F-statistic: 59.02
Date: Sat, 11 Jan 2020 Prob (F-statistic): 1.56e-13
Time: 17:34:53 Log-Likelihood: -1217.4
No. Observations: 354 AIC: 2439.
Df Residuals: 352 BIC: 2447.
Df Model: 1
Covariance Type: nonrobust
==================================================================================
coef std err t P>|t| [0.025 0.975]
----------------------------------------------------------------------------------
Intercept 11.2094 1.079 10.386 0.000 9.087 13.332
medianValue_Ks -0.3502 0.046 -7.683 0.000 -0.440 -0.261
==============================================================================
Omnibus: 447.354 Durbin-Watson: 1.928
Prob(Omnibus): 0.000 Jarque-Bera (JB): 39791.431
Skew: 5.897 Prob(JB): 0.00
Kurtosis: 53.583 Cond. No. 63.7
==============================================================================
Warnings:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
In [15]:
# Use the statsmodels API to create a log-linear regression model
logLinearModel = smf.ols(formula='np.log(crimeRatePerCapita) ~ medianValue_Ks',\
data=train_data)
logLinearModResult = logLinearModel.fit()
print(logLinearModResult.summary()) OLS Regression Results
======================================================================================
Dep. Variable: np.log(crimeRatePerCapita) R-squared: 0.238
Model: OLS Adj. R-squared: 0.236
Method: Least Squares F-statistic: 109.9
Date: Sat, 11 Jan 2020 Prob (F-statistic): 1.48e-22
Time: 17:34:53 Log-Likelihood: -727.67
No. Observations: 354 AIC: 1459.
Df Residuals: 352 BIC: 1467.
Df Model: 1
Covariance Type: nonrobust
==================================================================================
coef std err t P>|t| [0.025 0.975]
----------------------------------------------------------------------------------
Intercept 1.9107 0.271 7.062 0.000 1.379 2.443
medianValue_Ks -0.1198 0.011 -10.482 0.000 -0.142 -0.097
==============================================================================
Omnibus: 11.420 Durbin-Watson: 1.907
Prob(Omnibus): 0.003 Jarque-Bera (JB): 10.764
Skew: 0.376 Prob(JB): 0.00460
Kurtosis: 2.594 Cond. No. 63.7
==============================================================================
Warnings:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
In [16]:
# Use the statsmodels API to create a multiple linear regression model
multiLinearModel = smf.ols(formula=\
'crimeRatePerCapita ~ pctLowerStatus + radialHighwaysAccess +\
medianValue_Ks + nitrixOxide_pp10m', data=train_data)
multiLinearModResult = multiLinearModel.fit()
print(multiLinearModResult.summary()) OLS Regression Results
==============================================================================
Dep. Variable: crimeRatePerCapita R-squared: 0.398
Model: OLS Adj. R-squared: 0.391
Method: Least Squares F-statistic: 57.77
Date: Sat, 11 Jan 2020 Prob (F-statistic): 2.19e-37
Time: 17:34:53 Log-Likelihood: -1154.9
No. Observations: 354 AIC: 2320.
Df Residuals: 349 BIC: 2339.
Df Model: 4
Covariance Type: nonrobust
========================================================================================
coef std err t P>|t| [0.025 0.975]
----------------------------------------------------------------------------------------
Intercept 0.8912 2.670 0.334 0.739 -4.360 6.142
pctLowerStatus 0.1028 0.080 1.277 0.202 -0.055 0.261
radialHighwaysAccess 0.4948 0.048 10.216 0.000 0.400 0.590
medianValue_Ks -0.1103 0.058 -1.916 0.056 -0.224 0.003
nitrixOxide_pp10m -2.1039 4.131 -0.509 0.611 -10.229 6.021
==============================================================================
Omnibus: 534.476 Durbin-Watson: 1.999
Prob(Omnibus): 0.000 Jarque-Bera (JB): 100129.789
Skew: 7.866 Prob(JB): 0.00
Kurtosis: 83.876 Cond. No. 374.
==============================================================================
Warnings:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
In [17]:
# Use the statsmodels API to create a multiple log-linear regression model
multiLogLinMod = smf.ols(formula=\
'np.log(crimeRatePerCapita) ~ \
(pctLowerStatus + radialHighwaysAccess + medianValue_Ks + nitrixOxide_pp10m)**2',\
data=train_data)
multiLogLinModResult = multiLogLinMod.fit()
print(multiLogLinModResult.summary()) OLS Regression Results
======================================================================================
Dep. Variable: np.log(crimeRatePerCapita) R-squared: 0.884
Model: OLS Adj. R-squared: 0.881
Method: Least Squares F-statistic: 261.5
Date: Sat, 11 Jan 2020 Prob (F-statistic): 7.79e-154
Time: 17:34:53 Log-Likelihood: -394.39
No. Observations: 354 AIC: 810.8
Df Residuals: 343 BIC: 853.3
Df Model: 10
Covariance Type: nonrobust
==========================================================================================================
coef std err t P>|t| [0.025 0.975]
----------------------------------------------------------------------------------------------------------
Intercept -5.4707 1.490 -3.671 0.000 -8.402 -2.540
pctLowerStatus 0.1541 0.049 3.161 0.002 0.058 0.250
radialHighwaysAccess 0.4697 0.052 9.070 0.000 0.368 0.572
medianValue_Ks -0.1457 0.044 -3.325 0.001 -0.232 -0.059
nitrixOxide_pp10m 3.4509 3.000 1.150 0.251 -2.450 9.352
pctLowerStatus:radialHighwaysAccess -0.0006 0.001 -0.576 0.565 -0.003 0.002
pctLowerStatus:medianValue_Ks -0.0041 0.001 -4.159 0.000 -0.006 -0.002
pctLowerStatus:nitrixOxide_pp10m -0.0783 0.081 -0.964 0.336 -0.238 0.082
radialHighwaysAccess:medianValue_Ks -0.0027 0.001 -2.694 0.007 -0.005 -0.001
radialHighwaysAccess:nitrixOxide_pp10m -0.4234 0.066 -6.404 0.000 -0.553 -0.293
medianValue_Ks:nitrixOxide_pp10m 0.3552 0.092 3.869 0.000 0.175 0.536
==============================================================================
Omnibus: 4.124 Durbin-Watson: 1.966
Prob(Omnibus): 0.127 Jarque-Bera (JB): 4.107
Skew: 0.175 Prob(JB): 0.128
Kurtosis: 3.395 Cond. No. 3.29e+04
==============================================================================
Warnings:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
[2] The condition number is large, 3.29e+04. This might indicate that there are
strong multicollinearity or other numerical problems.