{"type":"doc","content":[{"type":"customImage","attrs":{"src":"https://images.unsplash.com/photo-1551288049-bebda4e38f71?w=1200&h=400&fit=crop","alt":"Preprocessing Banner","width":600,"height":200}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"A deep dive into detecting and treating outliers using IQR and Gaussian methods, and scaling features with Min-Max, Standardisation, and custom-range scalers — with hands-on Python code and real-world decision frameworks."}]}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"📑 Table of Contents"}]},{"type":"orderedList","attrs":{"start":1,"type":null},"content":[{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/e0586dbe-16f4-49af-8d70-ec3ad622f1fd#1-what-is-an-outlier","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"What Is an Outlier?"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/e0586dbe-16f4-49af-8d70-ec3ad622f1fd#2-box-plot--visual-outlier-detection","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Box Plot — Visual Outlier Detection"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/e0586dbe-16f4-49af-8d70-ec3ad622f1fd#3-iqr-method--detecting-outliers-numerically","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"IQR Method — Detecting Outliers Numerically"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/e0586dbe-16f4-49af-8d70-ec3ad622f1fd#4-handling-outliers--three-approaches","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Handling Outliers — Three Approaches"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/e0586dbe-16f4-49af-8d70-ec3ad622f1fd#5-capping-winsorizing-in-python","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Capping (Winsorizing) in Python"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/e0586dbe-16f4-49af-8d70-ec3ad622f1fd#6-trimming-in-python","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Trimming in Python"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/e0586dbe-16f4-49af-8d70-ec3ad622f1fd#7-gaussian-method-mean--3%CF%83","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Gaussian Method (Mean ± 3σ)"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/e0586dbe-16f4-49af-8d70-ec3ad622f1fd#8-comparing-outlier-methods","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Comparing Outlier Methods"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/e0586dbe-16f4-49af-8d70-ec3ad622f1fd#9-feature-scaling--why-it-matters","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Feature Scaling — Why It Matters"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/e0586dbe-16f4-49af-8d70-ec3ad622f1fd#10-normalisation-min-max-scaling","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Normalisation (Min-Max Scaling)"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/e0586dbe-16f4-49af-8d70-ec3ad622f1fd#11-standardisation-z-score-scaling","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Standardisation (Z-score Scaling)"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/e0586dbe-16f4-49af-8d70-ec3ad622f1fd#12-mean-max-custom-range-scaling","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Mean-Max (Custom Range Scaling)"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/e0586dbe-16f4-49af-8d70-ec3ad622f1fd#13-comparing-scaling-techniques","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Comparing Scaling Techniques"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/e0586dbe-16f4-49af-8d70-ec3ad622f1fd#14-feature-scaling-readiness-checklist","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Feature Scaling Readiness Checklist"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/e0586dbe-16f4-49af-8d70-ec3ad622f1fd#15-python-truthiness--logical-operators","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Python Truthiness & Logical Operators"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/e0586dbe-16f4-49af-8d70-ec3ad622f1fd#16-conditional-function-example--weather-advice","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Conditional Function Example — Weather Advice"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/e0586dbe-16f4-49af-8d70-ec3ad622f1fd#17-imbalanced-data--overview","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Imbalanced Data — Overview"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"link","attrs":{"href":"https://claude.ai/chat/e0586dbe-16f4-49af-8d70-ec3ad622f1fd#18-full-pre-processing-checklist","target":"_blank","rel":"noopener noreferrer nofollow","class":"text-blue-600 underline"}}],"text":"Full Pre-Processing Checklist"}]}]}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"1. What Is an Outlier?"}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Outlier"},{"type":"text","text":" — an observation that lies outside the expected range of a variable, either far below the minimum or far above the maximum of the bulk of the data."}]}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Outliers can arise from:"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Data entry errors"},{"type":"text","text":" — typing "},{"type":"text","marks":[{"type":"code"}],"text":"150"},{"type":"text","text":" instead of "},{"type":"text","marks":[{"type":"code"}],"text":"15"},{"type":"text","text":" for age"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Measurement failures"},{"type":"text","text":" — a broken sensor recording extreme values"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Genuine extremes"},{"type":"text","text":" — a billionaire in a salary survey (real, but atypical)"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Domain impossibilities"},{"type":"text","text":" — negative ages, salaries of ₹0"}]}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Examples"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Column"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Normal Range"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Outlier Example"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Salary"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"₹15–35 LPA"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"₹1 Crore LPA"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Age"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"1–90 years"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"150 years, -10 years"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"MPG (fuel efficiency)"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"10–40 mpg"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"80 mpg"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Temperature"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"-10°C to 50°C"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"500°C"}]}]}]}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"2. Box Plot — Visual Outlier Detection"}]},{"type":"paragraph","attrs":{"textAlign":null}},{"type":"customImage","attrs":{"src":"https://miro.medium.com/v2/resize:fit:1400/1*2c21SkzJMf3frPXPAR_gZA.png","alt":"Box Plot IQR Outliers","width":600,"height":300}},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"A "},{"type":"text","marks":[{"type":"bold"}],"text":"box plot"},{"type":"text","text":" (also called a box-and-whisker plot) summarises the entire distribution of a variable in a single graphic."}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Box Plot Components"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Symbol"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Meaning"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Minimum"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Smallest non-outlier value (Q1 − 1.5 × IQR)"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Q1"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"25th percentile — bottom of the box"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Median (Q2)"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"50th percentile — line inside the box"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Q3"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"75th percentile — top of the box"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Maximum"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Largest non-outlier value (Q3 + 1.5 × IQR)"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"IQR"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Q3 − Q1 (height of the box)"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Positive outlier"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Value > Q3 + 1.5 × IQR"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Negative outlier"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Value < Q1 − 1.5 × IQR"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Dots beyond whiskers"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Individual outlier observations"}]}]}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Plotting a box plot with seaborn"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"import seaborn as sns\nimport matplotlib.pyplot as plt\nimport pandas as pd\n\ndf = pd.read_csv('mpg.csv')\n\nsns.boxplot(y='mpg', data=df)\nplt.title('MPG Distribution — Box Plot')\nplt.show()\n"}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"3. IQR Method — Detecting Outliers Numerically"}]},{"type":"customImage","attrs":{"src":"https://67dc5f63b8b6a6f0ea39b99a.storage.fermion.app/public-cached/markdown-editor-file/file-unique-id/exqobaiueyxjflrmgz3ocyp5owgnkcoiqjjqkra0mhtyxuvsubfp_dmdot8wxj7tzmk44gs_v5hjnlsfwneqazcyf-tphrevdpqf","alt":null,"width":600,"height":327}},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"The "},{"type":"text","marks":[{"type":"bold"}],"text":"Interquartile Range (IQR) method"},{"type":"text","text":" is the industry-standard, distribution-free approach to outlier detection. It does not assume the data is normally distributed."}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Step-by-step implementation"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"import numpy as np\nimport pandas as pd\n\ndf = pd.read_csv('mpg.csv')\n\n# Step 1: Compute quartiles\nq1 = df['mpg'].quantile(0.25)\nq3 = df['mpg'].quantile(0.75)\niqr = q3 - q1\n\nprint(f\"Q1={q1:.2f}, Q3={q3:.2f}, IQR={iqr:.2f}\")\n\n# Step 2: Determine limits\nupper_limit = q3 + 1.5 * iqr\nlower_limit = q1 - 1.5 * iqr\n\nprint(f\"Upper limit: {upper_limit:.2f}\")\nprint(f\"Lower limit: {lower_limit:.2f}\")\n\n# Step 3: Identify outliers\npositive_outliers = df[df['mpg'] > upper_limit]\nnegative_outliers = df[df['mpg'] < lower_limit]\n\nprint(f\"Positive outliers: {len(positive_outliers)}\")\nprint(f\"Negative outliers: {len(negative_outliers)}\")\nprint(positive_outliers[['mpg']])\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Understanding the 1.5 × IQR factor"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"The factor "},{"type":"text","marks":[{"type":"bold"}],"text":"1.5"},{"type":"text","text":" is the industry default (proposed by John Tukey in 1977). It can be adjusted:"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"1.5 × IQR"},{"type":"text","text":" — standard; flags moderate outliers"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"3.0 × IQR"},{"type":"text","text":" — stricter; flags only extreme outliers"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Custom factor"},{"type":"text","text":" — discuss with your domain expert or client"}]}]}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"4. Handling Outliers — Three Approaches"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Once detected, outliers need a treatment strategy. The three main approaches are:"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Approach"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"How It Works"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Typical Use"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Drawback"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Trimming"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Remove rows that exceed limits"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Outliers are clearly erroneous"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Loss of data"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Capping (Winsorizing)"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Replace extreme values with the nearest limit"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Need to keep dataset size"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Alters original values"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Gaussian (Mean ± 3σ)"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Treat values outside μ ± 3σ as outliers"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Data is roughly normally distributed"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"May miss non-normal tails"}]}]}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Comparative summary"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Method"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Data Retained"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Modification Extent"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Preferred When"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Trimming"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"↓ rows removed"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"None (raw values kept)"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Outliers are clearly erroneous"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Capping"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ All rows kept"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Values altered only at extremes"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Need to keep dataset size for modelling"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Gaussian"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ All rows kept"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Values altered based on statistical bounds"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Data follows a near-normal distribution"}]}]}]}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"5. Capping (Winsorizing) in Python"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Capping"},{"type":"text","text":" (also called "},{"type":"text","marks":[{"type":"bold"}],"text":"Winsorizing"},{"type":"text","text":") replaces outlier values with the nearest boundary value — preserving row count while preventing extreme values from distorting the model."}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"import numpy as np\nimport pandas as pd\n\ndf = pd.read_csv('mpg.csv')\n\n# Compute limits\nq1 = df['mpg'].quantile(0.25)\nq3 = df['mpg'].quantile(0.75)\niqr = q3 - q1\nupper_limit = q3 + 1.5 * iqr\nlower_limit = q1 - 1.5 * iqr\n\n# Apply capping on a copy (preserve original)\ndf_cap = df.copy()\n\ndf_cap['mpg'] = np.where(\n df_cap['mpg'] > upper_limit,\n upper_limit, # cap above to upper limit\n np.where(\n df_cap['mpg'] < lower_limit,\n lower_limit, # cap below to lower limit\n df_cap['mpg'] # keep value if within range\n )\n)\n\n# Verify\nprint(f\"Original max: {df['mpg'].max():.2f}\")\nprint(f\"Capped max: {df_cap['mpg'].max():.2f}\")\n"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"MPG walkthrough result:"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Single positive outlier detected at ≈ 46.6 mpg"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Upper limit = 46.25 mpg"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"After capping: "},{"type":"text","marks":[{"type":"code"}],"text":"max"},{"type":"text","text":" becomes 46.25, outlier visually disappears from the box plot"}]}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Capping using "},{"type":"text","marks":[{"type":"code"}],"text":"clip"},{"type":"text","text":" (pandas shorthand)"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"df_cap['mpg'] = df_cap['mpg'].clip(lower=lower_limit, upper=upper_limit)\n"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"clip"},{"type":"text","text":" is more concise than "},{"type":"text","marks":[{"type":"code"}],"text":"np.where"},{"type":"text","text":" for simple two-sided capping."}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"6. Trimming in Python"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Trimming"},{"type":"text","text":" removes rows that contain outlier values entirely. Use this only when you are confident the outlier rows are erroneous and not just extreme but valid observations."}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"# Remove rows outside IQR limits\ndf_trim = df[(df['mpg'] >= lower_limit) & (df['mpg'] <= upper_limit)]\n\nprint(f\"Rows before trimming: {len(df)}\")\nprint(f\"Rows after trimming: {len(df_trim)}\")\n"}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"⚠️ Trimming permanently reduces your dataset. If you are preparing training data for a model, fewer rows can hurt generalisation. Prefer capping when sample size is limited."}]}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"7. Gaussian Method (Mean ± 3σ)"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"The "},{"type":"text","marks":[{"type":"bold"}],"text":"Gaussian method"},{"type":"text","text":" detects outliers using the properties of a normal distribution. In a normally distributed variable, approximately:"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"68% of values fall within μ ± 1σ"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"95% of values fall within μ ± 2σ"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"99.7% of values fall within μ ± 3σ"}]}]}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Any value outside μ ± 3σ is treated as an outlier."}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"import numpy as np\n\nmean = df['mpg'].mean()\nstd = df['mpg'].std()\n\nupper_g = mean + 3 * std\nlower_g = mean - 3 * std\n\nprint(f\"Gaussian upper: {upper_g:.2f}\")\nprint(f\"Gaussian lower: {lower_g:.2f}\")\n\n# Identify outliers\ngauss_outliers = df[(df['mpg'] > upper_g) | (df['mpg'] < lower_g)]\nprint(f\"Gaussian outlier count: {len(gauss_outliers)}\")\n\n# Cap using Gaussian limits\ndf_gauss = df.copy()\ndf_gauss['mpg'] = df_gauss['mpg'].clip(lower=lower_g, upper=upper_g)\n"}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"⚠️ "},{"type":"text","marks":[{"type":"bold"}],"text":"Use with caution on non-normal data."},{"type":"text","text":" If your distribution is skewed or multimodal, the Gaussian method can miss true outliers or flag valid extreme values. Always visualise first."}]}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"8. Comparing Outlier Methods"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"# Side-by-side comparison: IQR vs Gaussian bounds\nprint(f\"IQR lower / upper: {lower_limit:.2f} / {upper_limit:.2f}\")\nprint(f\"Gaussian lower / upper: {lower_g:.2f} / {upper_g:.2f}\")\n"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Property"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"IQR Method"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Gaussian Method"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Assumption"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"No distributional assumption"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Assumes near-normal distribution"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Sensitivity"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Moderate"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Can be aggressive on skewed data"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Resistant to extreme outliers"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ Yes (uses quartiles)"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"❌ Mean and std are pulled by outliers"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Industry default"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ Yes (1.5 × IQR)"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Common in finance/science"}]}]}]}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Rule of thumb:"}]},{"type":"orderedList","attrs":{"start":1,"type":null},"content":[{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Always visualise with a box plot first."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"If data is roughly symmetric → either method works."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"If data is skewed → prefer IQR."}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"If outliers are present → use median for any subsequent imputation, not mean."}]}]}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"9. Feature Scaling — Why It Matters"}]},{"type":"customImage","attrs":{"src":"https://67dc5f63b8b6a6f0ea39b99a.storage.fermion.app/public-cached/markdown-editor-file/file-unique-id/t_iiuuqfcjlvtkqkdvepx_fw4fmhgfwrplwjalcnqcgmc8b163psn4_piuqumefx3wmlqa7bvgg7j1vs6vd_qd7qn6yhettmkz9u","alt":null,"width":600,"height":327}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Feature scaling"},{"type":"text","text":" adjusts numeric features so they share a comparable range, preventing one variable from dominating visualisations or model training."}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"The problem without scaling"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Imagine a dataset with two columns:"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"age"},{"type":"text","text":" — values roughly 20–40"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"salary"},{"type":"text","text":" — values ₹50,000–₹300,000"}]}]}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"In a plot or distance calculation, salary completely swamps age. Age appears flat near zero. Distance-based algorithms (KNN, SVM, K-Means) will learn almost entirely from salary and ignore age."}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"import pandas as pd\nimport matplotlib.pyplot as plt\n\ndf = pd.DataFrame({\n 'age': [22, 30, 45, 25, 35],\n 'salary': [50000, 120000, 250000, 75000, 180000]\n})\n\n# Before scaling — salary axis dwarfs age\ndf.plot.scatter(x='age', y='salary')\nplt.title(\"Before Scaling — Salary dominates\")\nplt.show()\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Which algorithms need scaling?"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Algorithm"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Needs Scaling?"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Reason"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Linear / Logistic Regression"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ Yes"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Gradient descent step size depends on magnitude"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"KNN"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ Yes"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Distance metric directly affected by scale"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"SVM"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ Yes"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Margin maximisation uses distances"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"K-Means"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ Yes"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Euclidean distance for cluster assignment"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Neural Networks"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ Yes"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Gradient propagation is scale-sensitive"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Decision Trees"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"❌ No"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Split decisions are threshold-based"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Random Forest"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"❌ No"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Ensemble of trees — scale-invariant"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"XGBoost / LightGBM"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"❌ No"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Tree-based — scale-invariant"}]}]}]}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"💡 "},{"type":"text","marks":[{"type":"bold"}],"text":"Domain check:"},{"type":"text","text":" Always verify with the subject-matter expert (SME) before scaling features with real-world significance (e.g., stock prices, volume measurements). If a stakeholder insists the raw magnitude must stay intact, skip scaling for that feature."}]}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"10. Normalisation (Min-Max Scaling)"}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Normalisation"},{"type":"text","text":" rescales data to the interval [0, 1] (or any user-defined range)."}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Formula"}]},{"type":"codeBlock","attrs":{"language":"javascript"},"content":[{"type":"text","text":"x_norm = (x - x_min) / (x_max - x_min)\n"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Minimum value → maps to "},{"type":"text","marks":[{"type":"bold"}],"text":"0"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Maximum value → maps to "},{"type":"text","marks":[{"type":"bold"}],"text":"1"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"All other values scale proportionally"}]}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"When to use"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ No serious outliers in the data"},{"type":"hardBreak"},{"type":"text","text":"✅ You need all features bounded within a fixed interval"},{"type":"hardBreak"},{"type":"text","text":"✅ Algorithm requires input in [0, 1] (e.g., neural networks with sigmoid output)"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Python implementation"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"from sklearn.preprocessing import MinMaxScaler\nimport pandas as pd\n\ndf = pd.DataFrame({\n 'age': [22, 30, 45, 25, 35],\n 'salary': [50000, 120000, 250000, 75000, 180000]\n})\n\nscaler = MinMaxScaler(feature_range=(0, 1)) # default range\n\n# fit_transform on training data\ndf_norm = pd.DataFrame(\n scaler.fit_transform(df[['age', 'salary']]),\n columns=['age_norm', 'salary_norm']\n)\n\nprint(df_norm)\n# Both columns now range from 0.0 to 1.0\n"}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"⚠️ "},{"type":"text","marks":[{"type":"bold"}],"text":"Sensitive to outliers."},{"type":"text","text":" A single extreme value will compress all other values toward zero. If outliers exist, apply capping first, then normalise."}]}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"11. Standardisation (Z-score Scaling)"}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Standardisation"},{"type":"text","text":" centres data at zero mean and rescales to unit variance."}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Formula"}]},{"type":"codeBlock","attrs":{"language":"javascript"},"content":[{"type":"text","text":"z = (x - μ) / σ\n"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"where "},{"type":"text","marks":[{"type":"code"}],"text":"μ"},{"type":"text","text":" is the mean and "},{"type":"text","marks":[{"type":"code"}],"text":"σ"},{"type":"text","text":" is the standard deviation."}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Transformed values typically fall in approximately "},{"type":"text","marks":[{"type":"bold"}],"text":"[-3, +3]"}]}]},{"type":"listItem","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"The distribution shape is "},{"type":"text","marks":[{"type":"bold"}],"text":"preserved"},{"type":"text","text":" — only location and scale change"}]}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"When to use"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ Data has outliers (more robust than Min-Max)"},{"type":"hardBreak"},{"type":"text","text":"✅ Algorithm assumes features are approximately normally distributed"},{"type":"hardBreak"},{"type":"text","text":"✅ Regression and classification tasks where outliers should still be visible but not dominate"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Python implementation"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"from sklearn.preprocessing import StandardScaler\nimport pandas as pd\n\nscaler = StandardScaler()\n\ndf_std = pd.DataFrame(\n scaler.fit_transform(df[['age', 'salary']]),\n columns=['age_std', 'salary_std']\n)\n\nprint(df_std)\n# Values centered around 0; outlier salaries still visible but on same scale as age\nprint(f\"Age mean: {df_std['age_std'].mean():.4f}, std: {df_std['age_std'].std():.4f}\")\n# → Age mean: 0.0000, std: 1.0000\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Accessing the learned parameters"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"print(f\"Mean per feature: {scaler.mean_}\")\nprint(f\"Std per feature: {scaler.scale_}\")\n"}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"12. Mean-Max (Custom Range Scaling)"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Custom range scaling"},{"type":"text","text":" is a variant of Min-Max that maps values to any user-defined symmetric interval, such as [-1, 1]."}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Formula"}]},{"type":"codeBlock","attrs":{"language":"javascript"},"content":[{"type":"text","text":"x_scaled = ((x - x_min) / (x_max - x_min)) × (b - a) + a\n"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"where "},{"type":"text","marks":[{"type":"code"}],"text":"[a, b]"},{"type":"text","text":" is the target interval (e.g., "},{"type":"text","marks":[{"type":"code"}],"text":"[-1, 1]"},{"type":"text","text":")."}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Python implementation"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"from sklearn.preprocessing import MinMaxScaler\n\n# Scale to [-1, 1] instead of [0, 1]\nscaler_mm = MinMaxScaler(feature_range=(-1, 1))\n\ndf_mm = pd.DataFrame(\n scaler_mm.fit_transform(df[['age', 'salary']]),\n columns=['age_mm', 'salary_mm']\n)\n\nprint(df_mm)\n# Values now span -1 to +1\n\nprint(f\"Min: {df_mm.min().values}, Max: {df_mm.max().values}\")\n# → Min: [-1. -1.], Max: [1. 1.]\n"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"When to use:"},{"type":"hardBreak"},{"type":"text","text":"When a specific numerical interval is required — for example, neural network architectures that use "},{"type":"text","marks":[{"type":"code"}],"text":"tanh"},{"type":"text","text":" activation (which naturally outputs [-1, 1]) benefit from inputs in the same range."}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"13. Comparing Scaling Techniques"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Technique"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Output Range"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Sensitive to Outliers?"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Typical Scenario"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Normalisation (Min-Max)"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"[0, 1] or user-defined"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ Yes — extremes stretch the scale"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Uniform data, no major outliers"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Standardisation (Z-score)"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Usually [-3, 3]"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"⚠️ Partially — outliers affect μ/σ but not range"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Regression/classification with outliers"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Mean-Max (custom range)"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"User-defined (e.g., [-1, 1])"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ Yes — same sensitivity as Min-Max"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"When a specific numerical interval is required"}]}]}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Practical visualisation comparison"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"import matplotlib.pyplot as plt\nfrom sklearn.preprocessing import MinMaxScaler, StandardScaler\n\nfig, axes = plt.subplots(1, 3, figsize=(15, 4))\n\n# Raw\naxes[0].scatter(df['age'], df['salary'])\naxes[0].set_title('Raw Data')\n\n# Min-Max\nmm = MinMaxScaler()\ndf_mm_vals = mm.fit_transform(df[['age', 'salary']])\naxes[1].scatter(df_mm_vals[:, 0], df_mm_vals[:, 1])\naxes[1].set_title('Min-Max Normalised')\n\n# Standardised\nss = StandardScaler()\ndf_ss_vals = ss.fit_transform(df[['age', 'salary']])\naxes[2].scatter(df_ss_vals[:, 0], df_ss_vals[:, 1])\naxes[2].set_title('Standardised')\n\nplt.tight_layout()\nplt.show()\n"}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"14. Feature Scaling Readiness Checklist"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Before applying any scaler, ensure all preconditions are met:"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Step"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Action"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Code"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ Data type check"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Ensure all columns to scale are numeric"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"df.dtypes"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ Missing values"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Impute or drop first"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"df.isnull().sum()"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ Encoding"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Convert categorical variables to numeric"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"pd.get_dummies()"},{"type":"text","text":" or "},{"type":"text","marks":[{"type":"code"}],"text":"LabelEncoder"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ Outlier strategy"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Decide: trim, cap, or Gaussian — before scaling"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"IQR / Gaussian method"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ SME sign-off"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Confirm scaling aligns with business rules"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Domain discussion"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"✅ Scaling method"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Pick Normalisation, Standardisation, or Mean-Max"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Based on outlier presence and desired range"}]}]}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"⚠️ Integrating scaling with earlier preprocessing steps"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"# Wrong order — causes ValueError: could not convert string to float\nfrom sklearn.preprocessing import StandardScaler\nscaler = StandardScaler()\n# scaler.fit_transform(df) ← fails if df has object/string columns\n\n# Correct order\ndf_clean = df.drop(columns=['name']) # 1. Drop non-numeric\ndf_clean = pd.get_dummies(df_clean, drop_first=True) # 2. Encode categorical\ndf_clean['mpg'] = df_clean['mpg'].clip(lower=lower_limit, upper=upper_limit) # 3. Cap outliers\n# 4. Split first, then scale\nX_train_scaled = scaler.fit_transform(X_train)\nX_test_scaled = scaler.transform(X_test)\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"When NOT to scale"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Situation"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Why skip scaling"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Tree-based models (RF, XGBoost)"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Scale-invariant — splitting on thresholds, not distances"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Domain-critical magnitudes"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"A volume in metric tons has real-world meaning — scaling destroys it"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Stock price spikes"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"A one-off price jump (e.g., ₹20,000) may carry business signal"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"SME explicitly requests raw values"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Stakeholder requirements override technical defaults"}]}]}]}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"15. Python Truthiness & Logical Operators"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Every Python object, when evaluated in a Boolean context (e.g., inside an "},{"type":"text","marks":[{"type":"code"}],"text":"if"},{"type":"text","text":" statement), is either "},{"type":"text","marks":[{"type":"bold"}],"text":"truthy"},{"type":"text","text":" or "},{"type":"text","marks":[{"type":"bold"}],"text":"falsy"},{"type":"text","text":"."}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Falsy values"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Value"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Reason it is falsy"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"0"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Numeric zero"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"0.0"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Float zero"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"''"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Empty string"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"[]"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Empty list"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"{}"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Empty dict or set"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"None"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Null object"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"False"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Boolean false"}]}]}]}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Any other value — non-zero numbers, non-empty strings, non-empty collections — is "},{"type":"text","marks":[{"type":"bold"}],"text":"truthy"},{"type":"text","text":"."}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Demonstrating truthiness"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"values = [0, '', {}, [], None, False, 1, 'text']\n\nfor val in values:\n if val:\n print(f\"{repr(val):15} → truthy\")\n else:\n print(f\"{repr(val):15} → falsy\")\n\n# Output:\n# 0 → falsy\n# '' → falsy\n# {} → falsy\n# [] → falsy\n# None → falsy\n# False → falsy\n# 1 → truthy\n# 'text' → truthy\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Practical uses in data processing"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"# Skip empty strings in a list\nnames = ['Alice', '', 'Bob', None, 'Charlie']\nclean_names = [n for n in names if n]\nprint(clean_names) # ['Alice', 'Bob', 'Charlie']\n\n# Default value pattern\nuser_input = input(\"Enter a name: \")\nname = user_input if user_input else \"Anonymous\"\n\n# Guard against empty DataFrame\nif not df.empty:\n print(df.describe())\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Logical operators "},{"type":"text","marks":[{"type":"code"}],"text":"and"},{"type":"text","text":", "},{"type":"text","marks":[{"type":"code"}],"text":"or"},{"type":"text","text":", "},{"type":"text","marks":[{"type":"code"}],"text":"not"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"x = 15\n\n# and — both conditions must be True\nif x > 10 and x < 20:\n print(\"x is between 10 and 20\") # → prints\n\n# or — at least one condition must be True\nif x < 5 or x > 10:\n print(\"x is outside 5–10 range\") # → prints\n\n# not — inverts the truth value\nif not x == 0:\n print(\"x is non-zero\") # → prints\n\n# Short-circuit evaluation\nresult = None or \"default\"\nprint(result) # \"default\" — None is falsy, so \"default\" is returned\n"}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"16. Conditional Function Example — Weather Advice"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"A practical nested conditional function demonstrates how "},{"type":"text","marks":[{"type":"code"}],"text":"and"},{"type":"text","text":", "},{"type":"text","marks":[{"type":"code"}],"text":"or"},{"type":"text","text":", and "},{"type":"text","marks":[{"type":"code"}],"text":"if/elif/else"},{"type":"text","text":" combine in real code."}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"def get_weather_advice(temp, is_raining):\n \"\"\"Return weather advice based on temperature and rain status.\"\"\"\n\n # Hot conditions (temp > 30)\n if temp > 30:\n if is_raining:\n return \"It's hot and rainy – don't forget an umbrella and stay hydrated.\"\n else:\n return \"It's a hot day – remember to drink plenty of water.\"\n\n # Mild conditions (20 ≤ temp ≤ 30)\n elif 20 <= temp <= 30:\n if is_raining:\n return \"It's mild but rainy – an umbrella might be handy.\"\n else:\n return \"It's a pleasant day – enjoy the weather.\"\n\n # Cold conditions (temp < 20)\n else:\n if is_raining:\n return \"It's cold and rainy – take a warm jacket and an umbrella.\"\n else:\n return \"It's a cold day – bundle up.\"\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Decision table"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Temperature Range"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Rain?"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Returned Message"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"temp > 30"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"True"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Hot & rainy — umbrella + hydration"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"temp > 30"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"False"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Hot day — stay hydrated"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"20 ≤ temp ≤ 30"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"True"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Mild but rainy — umbrella handy"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"20 ≤ temp ≤ 30"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"False"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Pleasant day"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"temp < 20"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"True"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Cold & rainy — jacket + umbrella"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"code"}],"text":"temp < 20"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"False"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Cold day — bundle up"}]}]}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Test calls"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"print(get_weather_advice(40, False))\n# \"It's a hot day – remember to drink plenty of water.\"\n\nprint(get_weather_advice(26, True))\n# \"It's mild but rainy – an umbrella might be handy.\"\n\nprint(get_weather_advice(16, True))\n# \"It's cold and rainy – take a warm jacket and an umbrella.\"\n\nprint(get_weather_advice(10, False))\n# \"It's a cold day – bundle up.\"\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Decision flow"}]},{"type":"codeBlock","attrs":{"language":"javascript"},"content":[{"type":"text","text":"get_weather_advice(temp, is_raining)\n │\n temp > 30?\n ├── YES → is_raining?\n │ ├── YES → \"Hot & rainy\"\n │ └── NO → \"Hot day\"\n │\n 20 ≤ temp ≤ 30?\n ├── YES → is_raining?\n │ ├── YES → \"Mild & rainy\"\n │ └── NO → \"Pleasant day\"\n │\n ELSE (temp < 20) → is_raining?\n ├── YES → \"Cold & rainy\"\n └── NO → \"Cold day\"\n"}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"17. Imbalanced Data — Overview"}]},{"type":"paragraph","attrs":{"textAlign":null}},{"type":"customImage","attrs":{"src":"https://miro.medium.com/v2/resize:fit:1400/1*7xf9e1EaoK5n05izIFBouA.png","alt":"SMOTE Imbalanced Data","width":600,"height":176}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Imbalanced data"},{"type":"text","text":" — when one class dominates the others, leading to a biased model that performs well on the majority class but poorly on the minority class."}]}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Example:"},{"type":"text","text":" Fraud detection — 99% legitimate transactions, 1% fraudulent. A naive model that predicts \"legitimate\" for every transaction achieves 99% accuracy but catches 0% of fraud."}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Common treatment techniques"}]},{"type":"table","content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Technique"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Goal"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Oversampling"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Duplicate minority class samples to balance the dataset"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Undersampling"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Remove majority class samples to achieve parity"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"SMOTE"},{"type":"text","text":" (Synthetic Minority Over-sampling Technique)"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Generate new synthetic minority examples using KNN interpolation"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"bold"}],"text":"Class weighting"}]}]},{"type":"tableCell","attrs":{"colspan":1,"rowspan":1,"colwidth":null},"content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Assign higher penalty to misclassifying the minority class"}]}]}]}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Where it fits in the pipeline"}]},{"type":"codeBlock","attrs":{"language":"javascript"},"content":[{"type":"text","text":"raw data\n ↓\nmissing value treatment\n ↓\nencoding\n ↓\noutlier handling\n ↓\nfeature scaling\n ↓\ntrain-test split\n ↓\nimbalance treatment (on training set ONLY) ← here\n ↓\nmodel training\n"}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"⚠️ "},{"type":"text","marks":[{"type":"bold"}],"text":"Never apply SMOTE or oversampling before the train-test split."},{"type":"text","text":" Synthetic samples generated from training data must not appear in the test set, or evaluation metrics will be artificially inflated."}]}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"# Example: SMOTE after splitting\nfrom imblearn.over_sampling import SMOTE\nfrom sklearn.model_selection import train_test_split\n\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=0)\n\nsmote = SMOTE(random_state=42)\nX_train_resampled, y_train_resampled = smote.fit_resample(X_train, y_train)\n\nprint(f\"Before SMOTE: {dict(zip(*np.unique(y_train, return_counts=True)))}\")\nprint(f\"After SMOTE: {dict(zip(*np.unique(y_train_resampled, return_counts=True)))}\")\n"}]},{"type":"horizontalRule"},{"type":"heading","attrs":{"textAlign":null,"level":2},"content":[{"type":"text","text":"18. Full Pre-Processing Checklist"}]},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","text":"Here is the complete, ordered pre-processing pipeline:"}]},{"type":"codeBlock","attrs":{"language":"javascript"},"content":[{"type":"text","text":"✅ 1. Load & inspect data\n df.info() | df.isnull().sum() | df.describe()\n\n✅ 2. Handle missing values\n - Calculate missing rate per column\n - < 25% → impute (median/mean/mode or KNN)\n - ≥ 25% → drop (with stakeholder approval)\n\n✅ 3. Encode categorical variables\n - Ordinal → LabelEncoder / OrdinalEncoder\n - Nominal → OneHotEncoder / pd.get_dummies(drop_first=True)\n - High-cardinality → drop or engineer\n\n✅ 4. Detect and treat outliers\n - Visualise with box plot\n - IQR method: upper = Q3 + 1.5×IQR, lower = Q1 − 1.5×IQR\n - Choose: trimming / capping / Gaussian\n - Default to median for imputation if outliers are present\n\n✅ 5. Train-test split (BEFORE scaling)\n train_test_split(X, y, test_size=0.25, random_state=0)\n\n✅ 6. Apply feature scaling (fit on train, transform both)\n - No/few outliers → MinMaxScaler\n - Outliers present → StandardScaler\n - Custom range needed → MinMaxScaler(feature_range=(a,b))\n\n✅ 7. Handle class imbalance (on training set only)\n - SMOTE / oversampling / undersampling / class_weight\n\n✅ 8. Build and validate model\n"}]},{"type":"heading","attrs":{"textAlign":null,"level":3},"content":[{"type":"text","text":"Full pipeline in code"}]},{"type":"codeBlock","attrs":{"language":"python"},"content":[{"type":"text","text":"import pandas as pd\nimport numpy as np\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.impute import SimpleImputer\nfrom sklearn.preprocessing import StandardScaler, OneHotEncoder\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.compose import ColumnTransformer\n\n# Load\ndf = pd.read_csv('data.csv')\n\n# Outlier capping (IQR)\nfor col in ['age', 'salary']:\n q1, q3 = df[col].quantile([0.25, 0.75])\n iqr = q3 - q1\n df[col] = df[col].clip(q1 - 1.5*iqr, q3 + 1.5*iqr)\n\n# Split\nX = df.drop('target', axis=1)\ny = df['target']\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=0)\n\n# Pipeline\nnum_features = ['age', 'salary']\ncat_features = ['gender', 'city']\n\npreprocessor = ColumnTransformer([\n ('num', Pipeline([\n ('imp', SimpleImputer(strategy='median')),\n ('scale', StandardScaler())\n ]), num_features),\n ('cat', Pipeline([\n ('imp', SimpleImputer(strategy='most_frequent')),\n ('encode', OneHotEncoder(drop='first'))\n ]), cat_features)\n])\n\nX_train_proc = preprocessor.fit_transform(X_train)\nX_test_proc = preprocessor.transform(X_test)\n"}]},{"type":"horizontalRule"},{"type":"paragraph","attrs":{"textAlign":null},"content":[{"type":"text","marks":[{"type":"italic"}],"text":"Outlier treatment and feature scaling are not optional — they directly determine whether your model learns meaningful patterns or is dominated by noise and magnitude differences. Master these techniques and your preprocessing pipeline will consistently produce model-ready data."}]}]}